diff --git a/.gitignore b/.gitignore index 5d8891d..499c1ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,11 @@ reference/ -design/*.png .venv/ *.egg-info/ mechababs/_version.py __pycache__/ .pytest_cache/ .ruff_cache/ -processing/ local-notes/ -repronim-containers/ # setuptools' build tree, left behind by `uv build` when checking what ships. build/ build.stale_*/ diff --git a/bootstrap.sh b/bootstrap.sh deleted file mode 100755 index abff9f2..0000000 --- a/bootstrap.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env bash -# -# bootstrap.sh — build a mechababs campaign's environment (venv + vendored -# babs/mechababs pins + datalad dataset). See usage() for options. -set -euo pipefail - -BABS_DEFAULT="https://github.com/PennLINC/babs.git@main" -MECHABABS_DEFAULT="https://github.com/con/mechababs.git@main" - -usage() { - cat <<'EOF' -Usage: ./bootstrap.sh [--babs URL@REF] [--mechababs URL@REF] - [--system-site-packages] - -Build a mechababs campaign's environment: - - build .venv (optionally reusing the system site-packages) - - clone the pinned babs + mechababs - - make the campaign a datalad dataset, register subdatasets - - --babs URL@REF babs pin (default: PennLINC/babs main) - --mechababs URL@REF mechababs pin (default: con/mechababs main) - -URL is anything `git clone` accepts, including a local path — so -`--babs /path/to/babs-checkout@my-branch` pins a branch that exists only on -disk, and a change can be exercised end to end before it is pushed anywhere. -REF must be a branch or tag name, not a commit sha, since the pin is set by -`git clone --branch`. - --system-site-packages build the venv with access to the ambient Python's - installed packages, reusing a pre-built heavy stack - instead of rebuilding it (the e2e fixture uses this on - the slurm-docker-ci container, whose 2015-era compiler - can't build the newest wheels from source). -EOF -} - -run() { - printf '+ %s\n' "$*" >&2 - "$@" -} - -save_pin() { - # -d "$CAMPAIGN" is load-bearing: without it datalad saves inside the nested - # clone (a no-op) instead of registering it as a campaign subdataset. - local name="$1" ref="$2" head - head="$(git -C "code/$name" rev-parse --short HEAD)" - run "$VENV_DATALAD" save -d "$CAMPAIGN" -m "Vendor $name at $ref ($head)" "code/$name" -} - -require_url_ref() { - case "$1" in - *@*) : ;; - *) echo "expected URL@REF, got: $1" >&2; exit 1 ;; - esac -} - -CAMPAIGN="" -BABS_SPEC="$BABS_DEFAULT" -MECHABABS_SPEC="$MECHABABS_DEFAULT" -SSP="" -while [ $# -gt 0 ]; do - case "$1" in - --babs) BABS_SPEC="$2"; shift 2 ;; - --mechababs) MECHABABS_SPEC="$2"; shift 2 ;; - --system-site-packages) SSP=1; shift ;; - -h|--help) usage; exit 0 ;; - -*) echo "unknown option: $1" >&2; exit 1 ;; - *) - [ -n "$CAMPAIGN" ] && { echo "unexpected extra argument: $1" >&2; exit 1; } - CAMPAIGN="$1"; shift ;; - esac -done - -[ -n "$CAMPAIGN" ] || { echo "missing for the campaign" >&2; exit 1; } -require_url_ref "$BABS_SPEC" -BABS_URL="${BABS_SPEC%@*}"; BABS_REF="${BABS_SPEC##*@}" -require_url_ref "$MECHABABS_SPEC" -MECHA_URL="${MECHABABS_SPEC%@*}"; MECHA_REF="${MECHABABS_SPEC##*@}" - -for tool in git uv git-annex; do - command -v "$tool" >/dev/null 2>&1 || { echo "required tool not on PATH: $tool" >&2; exit 1; } -done -[ -e "$CAMPAIGN" ] && { echo "path already exists (fresh campaign only): $CAMPAIGN" >&2; exit 1; } - -run mkdir -p "$CAMPAIGN" -CAMPAIGN="$(cd "$CAMPAIGN" && pwd)" -cd "$CAMPAIGN" - -VENV="$CAMPAIGN/.venv" -VENV_PY="$VENV/bin/python" -VENV_DATALAD="$VENV/bin/datalad" - -run git clone --branch "$MECHA_REF" "$MECHA_URL" code/mechababs -run git clone --branch "$BABS_REF" "$BABS_URL" code/babs - -# venv + the mechababs pin, whose deps bring datalad (needed for `datalad create` -# below). babs is installed after registration — it is not a dep of mechababs, -# just a CLI it shells out to, and the campaign pins its version independently. -# --system-site-packages (opt-in) lets the venv reuse an ambient pre-built stack. -run uv venv ${SSP:+--system-site-packages} "$VENV" -run uv pip install --python "$VENV_PY" -e code/mechababs - -# gitignore .venv before create/save so it's never captured. The ledger lock is -# added by `mechababs configure`, which owns the ledger filename constant. -printf '%s\n' '.venv/' > .gitignore - -# make the campaign a datalad dataset over the populated dir, then register pins -run "$VENV_DATALAD" create --force -c text2git . -run "$VENV_DATALAD" save -d "$CAMPAIGN" -m "Ignore .venv" .gitignore -save_pin mechababs "$MECHA_REF" -save_pin babs "$BABS_REF" - -# The campaign owns its cluster/pipeline configs (not the vendored tool), so the -# config that produced a run is committed in the campaign and reproduces from it -# alone. Create the dirs now, versioned from the start; git can't track an empty -# dir, so seed each with a README pointing at the starter set. -run mkdir -p clusters pipelines -cat > clusters/README.md <<'EOF' -# Campaign cluster configs - -Cluster profiles for this campaign live here, so the config that produced a run is -committed in the campaign. `mechababs configure --cluster ` copies the config -in and resolves it by name; a `` already here resolves in place. Starter -profiles to copy from (or point `--cluster` straight at) live in -`code/mechababs/examples/clusters/`. -EOF -cat > pipelines/README.md <<'EOF' -# Campaign pipeline configs - -Pipeline configs for this campaign live here, so the config that produced a run is -committed in the campaign. `mechababs configure --pipelines ,...` copies each -config in and resolves it by name; a `` already here resolves in place. -Starters live in `code/mechababs/examples/pipelines/`. -EOF -run "$VENV_DATALAD" save -d "$CAMPAIGN" -m "Add campaign clusters/ + pipelines/ config dirs" clusters pipelines - -# editable-install the pinned babs + the campaign extras into the venv. -# Under --system-site-packages, install babs WITHOUT its deps: they come from the -# ambient env (--system-site-packages made them importable), and reinstalling -# them into the venv would rebuild babs's heavy stack from source — which fails on -# a host whose compiler can't build the newest wheels (the e2e slurm-docker-ci -# container). Trusts the ambient env to satisfy babs's deps. -run uv pip install --python "$VENV_PY" ${SSP:+--no-deps} -e code/babs -if [ -f code/mechababs/requirements-campaign.txt ]; then - run uv pip install --python "$VENV_PY" -r code/mechababs/requirements-campaign.txt -fi - -echo >&2 -echo "Environment ready at $CAMPAIGN" >&2 -echo "Next, from the campaign venv, configure the campaign:" >&2 -echo " source .venv/bin/activate" >&2 -echo " mechababs configure --pipelines --cluster " >&2 diff --git a/design/openneuro-findings.md b/design/openneuro-findings.md deleted file mode 100644 index f825ab0..0000000 --- a/design/openneuro-findings.md +++ /dev/null @@ -1,65 +0,0 @@ -# OpenNeuroStudies Exploration — Findings - -Date: 2026-03-23 - -## Structure - -Three GitHub orgs work together: - -| Org | Role | Example | -|---|---|---| -| **OpenNeuroDatasets** | Raw BIDS data | `ds005256` | -| **OpenNeuroDerivatives** | Processing outputs | `ds000001-mriqc` | -| **OpenNeuroStudies** | Glue — links raw to derivatives | `study-ds000001` | - -**OpenNeuroStudies/OpenNeuroStudies** is a datalad superdataset. Each -`study-dsXXXXXX/` is a subdataset containing: - -``` -study-dsXXXXXX/ - sourcedata/dsXXXXXX → github.com/OpenNeuroDatasets/dsXXXXXX - derivatives/ → github.com/OpenNeuroDerivatives/dsXXXXXX- -``` - -**`studies.tsv`** is the authoritative index (maintained by Yarik). Key -columns: `study_id`, `raw_version`, `derivative_count`, `derivative_ids`. - -## Candidate selection - -Filter: `raw_version != n/a` AND no MRIQC in `derivative_ids`. - -- `raw_version = n/a` means derivative-only datasets (e.g., ds006189–ds006192 - are a family of processing outputs derived from ds006131 and others) -- ds006192 lives in OpenNeuroDatasets but is semantically a derivative - (xcp_d output from 4 source datasets) - -**16 candidates** identified (15 excluding ds002685 which already failed, -see mechababs/issues/5). - -## Scripts created - -- **`update_candidates.py`** — Reads `studies.tsv`, generates/updates - `candidates.tsv` (dataset_id, status, issue, notes). Preserves - hand-edited rows. Appends "DONE UPSTREAM " to notes when a - dataset gains mriqc upstream. - -- **`preflight.py `** — Pre-flight check before running - mriqc. Verifies no mriqc derivative exists (`git ls-remote`) and - dataset is in `studies.tsv` with `raw_version`. Prints dataset info - (subjects, sessions, scans, sizes). Exit 0 = go, exit 1 = stop. - -## Environment - -- venv at `.venv/` created with `uv`, has `datalad 1.3.4` -- OpenNeuroStudies cloned into `./OpenNeuroStudies/` -- Global gitconfig rewrites HTTPS→SSH (`url.git@github.com:.insteadOf`); - use `GIT_CONFIG_GLOBAL=/dev/null` to bypass in this container - -## Next steps - -- Run mechababs across all candidates, 1 subject each, to find failure - modes (per Yarik: "try to run your tool ACROSS all of those listed - there on 1 subject and see how it fails") -- Consider adding preflight check for raw dataset existence - (`git ls-remote` against OpenNeuroDatasets) -- Subject vs session detection and resource estimation (mechababs/issues/3) diff --git a/mechababs/select.py b/mechababs/select.py index d237e07..c127990 100644 --- a/mechababs/select.py +++ b/mechababs/select.py @@ -19,8 +19,8 @@ TSV disambiguation — several "tsv/csv" artifacts are in play; this one is the per-study metadata (per subject/session: ``datatypes``, ``t1w_num``, ``bold_num``), -NOT OpenNeuro's all-studies ``studies.tsv``, our ledger ``desc-mechababs_datasets.tsv``, or -babs's in-project ``processing_inclusion.csv``. +NOT OpenNeuro's all-studies ``studies.tsv``, the campaign statefile +``sourcedata+derivatives.tsv``, or babs's in-project ``processing_inclusion.csv``. """ import csv diff --git a/mechababs/testing/e2e/test_spine.py b/mechababs/testing/e2e/test_spine.py index 85aaf02..466bfc3 100644 --- a/mechababs/testing/e2e/test_spine.py +++ b/mechababs/testing/e2e/test_spine.py @@ -248,8 +248,8 @@ def _run_record(study): def _iterate(study, *args): """One iterate, run the way a user runs it: sourced env.sh, then `iterate`. - No `--campaign-path` and no cell named: where you stand is the study, the env var - is the campaign, and which cell moves is the reconciler's decision, not ours. That + No path and no cell named: where you stand is the study, the env var is the + campaign, and which cell moves is the reconciler's decision, not ours. That is the whole difference between this and `_dispatch`. """ return _in_campaign(study, LABEL, "iterate", *args) diff --git a/mechababs/validate.py b/mechababs/validate.py index 31ca827..c7639b9 100644 --- a/mechababs/validate.py +++ b/mechababs/validate.py @@ -42,40 +42,6 @@ WORKDIR_ENV = "MECHABABS_E2E_WORKDIR" -class RefError(Exception): - """A checkout is not on anything ``git clone --branch`` could re-clone.""" - - -def clone_ref(clone): - """The branch or tag a checkout is on, as something re-clonable. - - A code pin is set with `git clone --branch `, which takes a branch or a tag, - so the ref handed back has to be one of those. A tag pin leaves the clone on a - detached HEAD, where `rev-parse --abbrev-ref HEAD` reports the literal "HEAD"; - passing that on fails later with "Remote branch HEAD not found". So fall back to - the exact tag, and raise when it is neither. - """ - ref = subprocess.run( - ["git", "-C", str(clone), "rev-parse", "--abbrev-ref", "HEAD"], - check=True, - text=True, - capture_output=True, - ).stdout.strip() - if ref != "HEAD": - return ref - tag = subprocess.run( - ["git", "-C", str(clone), "describe", "--tags", "--exact-match"], - text=True, - capture_output=True, - ) - if tag.returncode == 0 and tag.stdout.strip(): - return tag.stdout.strip() - raise RefError( - f"{clone} is on a detached HEAD with no exact tag, so there is no branch or " - f"tag to re-clone; check out a branch or tag first" - ) - - def resolve_cluster(arg): """The cluster config to validate, as a path that exists. diff --git a/merge_config.py b/merge_config.py deleted file mode 100644 index 10c4535..0000000 --- a/merge_config.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python3 -"""Merge pipeline + cluster + dataset configs into a babs container-config YAML. - -Usage: - python3 merge_config.py \\ - --pipeline examples/pipelines/MRIQC-24.0.2.yaml \\ - --cluster examples/clusters/dartmouth.yaml \\ - --dataset-url https://github.com/OpenNeuroDatasets/ds000003.git - -Writes merged YAML to stdout. - -TODO: pyyaml round-trip mangles multiline block scalars (customized_text, -script_preamble get extra blank lines). Doesn't break babs (also uses -pyyaml to read) but looks ugly. See reference/babs_demo/babs_walkthrough.sh:128 -for Dorota's heredoc+sed approach. -""" - -import argparse -import sys - -import yaml - - -def merge_babs_config( - pipeline_config, cluster_config, dataset_url, input_origins=None, campaign_venv=None -): - """Merge pipeline and cluster configs with dataset URL into a babs config dict.""" - # Pipeline config (bids_app_args, singularity_args, zip_foldernames, etc.) - # Strip 'mechababs' — the mechababs-only namespace (container = babs-init CLI - # args; selection = eligibility rule), consumed by mechababs, never babs config. - merged = {k: v for k, v in pipeline_config.items() if k != "mechababs"} - - # Cluster config (cluster_resources, script_preamble, job_compute_space) - for k, v in cluster_config.items(): - merged[k] = v - - # Resolve the venv placeholder in the preamble with the campaign venv abspath - # (the mechababs config keeps it relative; the caller resolves against the campaign root). - if campaign_venv and "script_preamble" in merged: - merged["script_preamble"] = merged["script_preamble"].replace( - "{{MECHABABS_VENV}}", campaign_venv - ) - - # Preserve any input_datasets already declared in the pipeline YAML - # (e.g. chained-pipeline anat-input for fmriprep-full). Always add - # or overwrite the BIDS entry from --dataset-url, and force BIDS - # to be FIRST in the dict — babs uses input_datasets[0] as the - # bids_dir positional arg to the BIDS app - # (generate_bidsapp_runscript.py:244 on add-containers-run-v2). - yaml_input_datasets = merged.get("input_datasets") or {} - input_datasets = { - "BIDS": { - "is_zipped": False, - "origin_url": dataset_url, - "path_in_babs": "sourcedata/raw", - } - } - for k, v in yaml_input_datasets.items(): - if k != "BIDS": - input_datasets[k] = v - - # Chained-input origin_urls: a downstream YAML declares the upstream stage's - # output as an input but leaves origin_url out — the upstream RIA doesn't - # exist until it has run + merged, so the key (== the producing pipeline's - # short_name) is the only thing knowable at author time. iterate resolves each - # such upstream's output-RIA URL at scaffold time and passes it as - # --input-origin =; nothing pipeline-specific is hardcoded here. - for key, url in (input_origins or {}).items(): - if key not in input_datasets: - sys.exit( - f"merge_config: --input-origin {key!r} given but pipeline declares " - f"no {key!r} input_dataset" - ) - input_datasets[key]["origin_url"] = url - - merged["input_datasets"] = input_datasets - - return merged - - -def main(): - parser = argparse.ArgumentParser( - description="Merge babs config from pipeline + cluster + dataset" - ) - parser.add_argument( - "--pipeline", required=True, help="Path to pipeline YAML config" - ) - parser.add_argument("--cluster", required=True, help="Path to cluster YAML config") - parser.add_argument( - "--dataset-url", required=True, help="URL or path to input BIDS dataset" - ) - parser.add_argument( - "--input-origin", - action="append", - default=[], - metavar="KEY=URL", - help="Set origin_url for the named input_datasets entry (repeatable). " - "Used for chained inputs, whose upstream RIA URL is only known at " - "run time; KEY is the input_datasets key (== the producing " - "pipeline's short_name).", - ) - parser.add_argument( - "--campaign-venv", - default=None, - help="abspath of the campaign venv; substitutes {{MECHABABS_VENV}} in the preamble", - ) - args = parser.parse_args() - - input_origins = {} - for item in args.input_origin: - key, sep, url = item.partition("=") - if not sep: - parser.error(f"--input-origin expects KEY=URL, got {item!r}") - input_origins[key] = url - - with open(args.pipeline) as f: - pipeline_config = yaml.safe_load(f) - with open(args.cluster) as f: - cluster_config = yaml.safe_load(f) - - merged = merge_babs_config( - pipeline_config, - cluster_config, - args.dataset_url, - input_origins, - campaign_venv=args.campaign_venv, - ) - yaml.dump(merged, sys.stdout, default_flow_style=False, sort_keys=False) - - -if __name__ == "__main__": - main() diff --git a/preflight.py b/preflight.py deleted file mode 100755 index 9033080..0000000 --- a/preflight.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 -"""Pre-flight checks before running mriqc on a dataset. - -Usage: preflight.py - e.g. preflight.py ds005256 - -Runs checks (fail fast if any fail). -Exit 0 = good to go, exit 1 = should not proceed. -""" - -import subprocess -import sys - - -def check_no_mriqc_repo(dataset_id): - """Check that no mriqc derivative repo exists on GitHub. - - TODO: false-passes on `git ls-remote` failures. Returncode != 0 + - empty stdout currently returns "no derivative" — but that case - includes auth failures (no ssh-agent, encrypted-key with no - passphrase entered), network errors, and any other git failure. - Should distinguish "Repository not found" in stderr (legit - pass) from auth/network errors (fail preflight). Also: only - checks mriqc; with multiple pipelines should be pipeline-aware. - """ - url = f"git@github.com:OpenNeuroDerivatives/{dataset_id}-mriqc.git" - result = subprocess.run( - ["git", "ls-remote", url, "HEAD"], - capture_output=True, - text=True, - ) - if result.returncode == 0 and result.stdout.strip(): - return False, f"mriqc derivative already exists: {url}" - return True, "no mriqc derivative repo found" - - -def check_terminal_multiplexer(dataset_id): - """Check that we're running inside screen or tmux.""" - import os - - if os.environ.get("TMUX") or os.environ.get("STY"): - return True, "running in tmux/screen" - if os.environ.get("TERM", "").startswith("screen"): - return True, "running in screen" - return ( - False, - "not running in screen or tmux — long-running jobs may be killed on disconnect", - ) - - -CHECKS = [ - ("terminal multiplexer", check_terminal_multiplexer), - ("mriqc derivative absent", check_no_mriqc_repo), -] - - -def main(): - if len(sys.argv) != 2: - print(f"Usage: {sys.argv[0]} ", file=sys.stderr) - print(f" e.g. {sys.argv[0]} ds005256", file=sys.stderr) - sys.exit(1) - - dataset_id = sys.argv[1].removeprefix("study-") - - print(f"Preflight checks for {dataset_id}\n") - - all_passed = True - for name, check_fn in CHECKS: - passed, msg = check_fn(dataset_id) - status = "PASS" if passed else "FAIL" - print(f" [{status}] {name}: {msg}") - if not passed: - all_passed = False - - if not all_passed: - print("\nPreflight FAILED — do not proceed.", file=sys.stderr) - sys.exit(1) - - print("\nAll checks passed.") - - -if __name__ == "__main__": - main() diff --git a/priority-openneuro-datasets.csv b/priority-openneuro-datasets.csv deleted file mode 100644 index 9be28b9..0000000 --- a/priority-openneuro-datasets.csv +++ /dev/null @@ -1,43 +0,0 @@ -Available on OpenNeuro,openneuro_id,openneuro_url,publication_url,first author,name,n_subjects,Adults / Children,Task or Rest,DC comments -yes,ds002685,https://openneuro.org/datasets/ds002685,https://www.nature.com/articles/s41597-024-03390-1,Pinho,Individual Brain Charting (IBC) - 3rd release,13,adults,Task & Rest, -yes,ds004636,https://openneuro.org/datasets/ds004636,https://www.ncbi.nlm.nih.gov/pmc/articles/PMC10557703/,Bisset,"Cognitive tasks, anatomical MRI, and functional MRI data evaluating the construct of self-regulation",110,adults,, -yes,ds005256,https://openneuro.org/datasets/ds005256,https://www.biorxiv.org/content/10.1101/2024.06.21.599974v1,Jung,Spacetop - multimodal fMRI dataset unifying naturalistic processes with a rich array of experimental tasks,118,adults,, -yes,ds004169,https://openneuro.org/datasets/ds004169,https://onlinelibrary.wiley.com/doi/full/10.1002/hbm.26717,Strike,Queensland Twin IMaging (QTIM),1202,adults,Task & Rest, -yes,ds003097,https://openneuro.org/datasets/ds003097,https://www.nature.com/articles/s41597-021-00870-6,Snoek,AOMIC-ID1000,928,adults,Task, -yes,ds004146,https://openneuro.org/datasets/ds004146,https://www.nature.com/articles/s41597-023-02038-w,Strike,Queensland Twin Adolescent Brain (QTAB),417,adolescent,Task & Rest, -yes,ds004746,https://openneuro.org/datasets/ds004746,https://www.nature.com/articles/s41467-024-50103-8,Botvinik-Nezer,Paingen_placebo,395,adults,Task, -yes,ds005165,https://openneuro.org/datasets/ds005165,https://www.nature.com/articles/s41467-024-50310-3,Lahner,Modeling short visual events through the BOLD Moments video fMRI dataset and metadata.,10,adults,Task & Rest, -yes,ds002345,https://openneuro.org/datasets/ds002345,https://www.nature.com/articles/s41597-021-01033-3,Nastase,Narratives,345,adults,Task, -yes,ds003604,https://openneuro.org/datasets/ds003604,https://www.nature.com/articles/s41597-021-01106-3,Wang,"A longitudinal neuroimaging dataset on language processing in children ages 5, 7, and 9 years old",322,children,Task, -yes,ds000221,https://openneuro.org/datasets/ds000221,https://www.nature.com/articles/sdata2018308,Babayan,MPI-Leipzig_Mind-Brain-Body,318,adults,Rest, -yes,ds003592,https://openneuro.org/datasets/ds003592,https://www.nature.com/articles/s41597-022-01231-7,Spreng,"Neurocognitive aging data release with behavioral, structural, and multi-echo functional MRI measures",301,adults,Rest, -yes,ds006623,https://openneuro.org/datasets/ds006623,https://www.nature.com/articles/s41597-025-06442-2,Huang,Michigan Human Anesthesia fMRI Dataset-1,26,adults,Task & Rest, -yes,ds000030,https://openneuro.org/datasets/ds000030,https://www.nature.com/articles/sdata2016110,Poldrack,UCLA Consortium for Neuropsychiatric Phenomics LA5c Study,272,adults,Task & Rest, -yes,ds004078,https://openneuro.org/datasets/ds004078,https://www.nature.com/articles/s41597-022-01708-5,Wang,A synchronized multimodal neuroimaging dataset to study brain language processing,12,adults,Task & Rest, -yes,ds005896,https://openneuro.org/datasets/ds005896,https://www.icpsr.umich.edu/web/NAHDAP/studies/39045/versions/V1,Way,"Structural and functional MRI dataset from the Adolescent Health and Development in Context (AHDC) study in Columbus, Ohio",262,adults,Task & Rest, -yes,ds005752,https://openneuro.org/datasets/ds005752,https://www.nature.com/articles/s41597-022-01623-9,Nugent,The NIMH Healthy Research Volunteer Dataset,251,adults,Task & Rest, -yes,ds005237,https://openneuro.org/datasets/ds005237,https://www.nature.com/articles/s41597-025-04895-z,Chopra,Transdiagnostic Connectome Project,241,adults,Task & Rest, -yes,ds005747,https://openneuro.org/datasets/ds005747,https://www.nature.com/articles/s41593-025-02087-x,Zhang,7T fMRI Resting-state Dataset,90,adults,Rest, -yes,ds004884,https://openneuro.org/datasets/ds004884,https://www.nature.com/articles/s41597-024-03819-7,Gibson,Aphasia Recovery Cohort (ARC) Dataset,230,adults,Task & Rest, -yes,ds002790,https://openneuro.org/datasets/ds002790,https://www.nature.com/articles/s41597-021-00870-6,Snoek,AOMIC-PIOP2,226,adults,Task & Rest, -yes,ds002785,https://openneuro.org/datasets/ds002785,https://www.nature.com/articles/s41597-021-00870-6,Snoek,AOMIC-PIOP1,216,adults,Task & Rest, -yes,ds001894,https://openneuro.org/datasets/ds001894,https://www.nature.com/articles/s41597-019-0338-5,Lytle,Longitudinal Brain Correlates of Multisensory Lexical Processing in Children,188,children,Task, -yes,ds004496,https://openneuro.org/datasets/ds004496,https://openneuro.org/datasets/ds004496/versions/2.1.2,Gong,A large-scale fMRI dataset for the visual processing of naturalistic scenes - NOD,30,adults,Task, -yes,ds002843,https://openneuro.org/datasets/ds002843,not available,Lee,Cognitive Training,166,adults,Task & Rest, -yes,ds004044,https://openneuro.org/datasets/ds004044,https://www.nature.com/articles/s41597-022-01644-4,Ma,An fMRI dataset for whole-body somatotopic mapping in humans,62,adults,Task, -yes,ds002766,https://openneuro.org/datasets/ds002766,https://pmc.ncbi.nlm.nih.gov/articles/PMC7419711/,Newbold,Cast-induced plasticity,3,adults,Task & Rest, -yes,ds007116,https://openneuro.org/datasets/ds007116,https://www.biorxiv.org/content/10.1101/2025.11.10.687633v1,Sevchick,Penn LEAD: Penn Longitudinal Executive functioning in Adolescent Development,132,adults,Task & Rest,renamed from ds006688 -yes,ds003798,https://openneuro.org/datasets/ds003798,https://www.nature.com/articles/s41597-022-01171-2,Kliemann,Caltech Conte Center - A multimodal data resource for exploring social cognition and decision-making.,117,adults,Task & Rest, -yes,ds004488,https://openneuro.org/datasets/ds004488,https://www.nature.com/articles/s41597-023-02325-6,Zhou,HAD (Human Action Dataset) - A large-scale fMRI dataset for human action recognition,30,adults,Task, -yes,ds003020,https://openneuro.org/datasets/ds003020/versions/2.2.0,https://www.nature.com/articles/s41597-023-02437-z,Lebel,"3T fMRI recordings from 8 participants listening to 27 complete, natural, narrative stories (~6 hours)",8,adults,Task, -yes,ds003643,https://openneuro.org/datasets/ds003643/versions/2.0.0,https://www.nature.com/articles/s41597-022-01625-7,Li,Le Petit Prince Multilingual,112,adults,Task, -yes,ds001506,https://openneuro.org/datasets/ds001506,https://journals.plos.org/ploscompbiol/article/file?id=10.1371%2Fjournal.pcbi.1006633&type=printable,Shen,"DeepImageRecon -(DeepRecon)",3,adults,Task, -yes,ds000113,https://openneuro.org/datasets/ds000113/versions/1.3.0,https://www.nature.com/articles/sdata20143,Hanke,StudyForrest,20,adults,Task, -yes,ds001499,https://openneuro.org/datasets/ds001499/versions/1.3.0,https://www.nature.com/articles/s41597-019-0052-3,Chang,"bold5000 - 3T fMRI recordings from 4 participants viewing ~5,000 natural images",4,adults,task, -,,,,,,7180,,, -no,n/a,https://plus.figshare.com/ndownloader/files/36789690&sa=D&source=editors&ust=1770396343900865&usg=AOvVaw306NjrLQoEcIO1sbycCdBN,https://elifesciences.org/articles/82580,Hebart,THINGS-fMRI,3,adults,Task, -no,n/a,https://github.com/courtois-neuromod/cneuromod-things,https://arxiv.org/abs/2507.09024,StLaurent,CNeuroMod-THINGS,4,adults,Task,"Should be all avialable via DataLad already, under CC0. Anything in particular for this effort?" -no,n/a,https://www.humanconnectome.org/study/hcp-young-adult/data-releases,https://pmc.ncbi.nlm.nih.gov/articles/PMC3724347/pdf/nihms482605.pdf,VanEssen,HCP - movie watching experiment; resting state; task fMRI,1200,adults,Task & Rest, -no,n/a,https://naturalscenesdataset.org/,https://www.nature.com/articles/s41593-021-00962-x,Allen,NSD,8,adults,Task,TODO: review terms / access -no,n/a,https://www.ukbiobank.ac.uk/,https://www.nature.com/articles/s41586-018-0579-z,Bycroft,UKB,100000,adults,Task & Rest,Needs dedicate DUA etc. See https://www.nature.com/articles/s41597-022-01163-2 for automating preprocessing with DataLad and condor cluster \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index ee6bb9a..a34fb64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,6 @@ log_cli_level = "INFO" log_cli_format = "%(asctime)s %(message)s" log_cli_date_format = "%H:%M:%S" -# Only the mechababs package is installed; the root bootstrap.sh is run directly. # `mechababs.testing` carries the e2e scenario, so `test-cluster` can run it from a # campaign wherever mechababs is installed — see mechababs/testing/__init__.py. [tool.setuptools] diff --git a/requirements-campaign.txt b/requirements-campaign.txt deleted file mode 100644 index a79f5dc..0000000 --- a/requirements-campaign.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Extra runtime tools a campaign needs beyond the babs + mechababs packages. -# Installed by bootstrap.sh into the campaign venv. Pin as needed. -con-duct -visidata # interactive TSV viewer for the state-file ledger -pytest # runs the packaged e2e scenario behind `mechababs test-cluster` diff --git a/tests/test_validate.py b/tests/test_validate.py index a48b336..3c27246 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -179,71 +179,3 @@ def test_cli_requires_a_scratch_path(monkeypatch, capsys): with pytest.raises(SystemExit): cli.main() assert "--scratch-path" in capsys.readouterr().err - - -# --- clone_ref: what `git clone --branch` can actually re-clone --------------------- -# The riskiest logic here (a pin that resolves to something unclonable fails far from -# its cause), so it gets real git repos rather than mocks. - - -def _repo(path, tag=None): - import subprocess - - path.mkdir(parents=True) - - def run(*a): - return subprocess.run( - ["git", "-C", str(path), *a], check=True, capture_output=True - ) - - run("init", "-q", "-b", "main") - run( - "-c", - "user.email=t@t", - "-c", - "user.name=t", - "commit", - "-q", - "--allow-empty", - "-m", - "c", - ) - if tag: - run("tag", tag) - return path - - -def test_clone_ref_returns_the_branch(tmp_path): - assert validate.clone_ref(_repo(tmp_path / "b")) == "main" - - -def test_clone_ref_falls_back_to_the_tag_when_detached(tmp_path): - """A tag pin leaves a detached HEAD, where `--abbrev-ref` says the literal - "HEAD" — which `git clone --branch` cannot resolve.""" - import subprocess - - src = _repo(tmp_path / "src", tag="v1.2.3") - clone = tmp_path / "clone" - subprocess.run( - ["git", "clone", "-q", "--branch", "v1.2.3", str(src), str(clone)], - check=True, - capture_output=True, - ) - assert validate.clone_ref(clone) == "v1.2.3", "must recover something re-clonable" - - -def test_clone_ref_refuses_a_detached_commit_with_no_tag(tmp_path): - import subprocess - - repo = _repo(tmp_path / "d") - sha = subprocess.run( - ["git", "-C", str(repo), "rev-parse", "HEAD"], - check=True, - text=True, - capture_output=True, - ).stdout.strip() - subprocess.run( - ["git", "-C", str(repo), "checkout", "-q", sha], check=True, capture_output=True - ) - with pytest.raises(validate.RefError, match="detached HEAD"): - validate.clone_ref(repo) diff --git a/tmp-repronim-container-shim.sh b/tmp-repronim-container-shim.sh deleted file mode 100755 index acb761a..0000000 --- a/tmp-repronim-container-shim.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash -# tmp-repronim-container-shim.sh — make a ReproNim/containers clone usable by -# *vanilla* babs. -# -# WHY: vanilla/PR babs reads the container image at the HARDCODED path -# containers/.datalad/environments//image -# (babs/container.py). ReproNim/containers stores images at -# images/bids/--.sif and registers them pointing there — a layout only -# the mechababs working branch resolves. This clones ReproNim once and -# `datalad containers-add`s the wanted containers AT THE DEFAULT (babs-expected) -# location, so `babs init --container-ds --container-name ` works -# against vanilla babs — the point being to e2e-test vanilla babs + PRs. -# -# Build once, OUTSIDE any campaign — as a SIBLING of your campaigns, since the -# pipeline configs reference it relatively as `../repronim-containers-shim` -# (iterate resolves a relative local source against the campaign root). So clone -# campaigns and this shim under the same parent dir. -# -# TEMPORARY: drop this (script + clone) once PennLINC/babs#383 lands (babs -# resolves the container image path via datalad-containers, i.e. understands -# ReproNim's layout natively) — then point container.source straight at ReproNim. -# -# Usage: -# tmp-repronim-container-shim.sh [ ...] # default: bids-mriqc bids-fmriprep -# REPRONIM=/other/path tmp-repronim-container-shim.sh bids-mriqc -# The clone path defaults below; override with the REPRONIM env var. -# -# `bids-simbids` is special: simbids isn't in upstream ReproNim/containers, so -# it's BUILT from Docker Hub (apptainer/singularity) once, not fetched — the fast -# stand-in app for the mechababs e2e harness. All other names are fetched from -# ReproNim. - -export PS4='> ' -set -x -set -eu - -REPRONIM="${REPRONIM:-/dartfs/rc/lab/D/DBIC/DBIC/CON/asmacdo/repronim-containers-shim}" -NAMES=("$@") -[ "${#NAMES[@]}" -eq 0 ] && NAMES=(bids-mriqc bids-fmriprep) - -# Clone ReproNim once; reuse thereafter. -[ -e "$REPRONIM/.datalad" ] || \ - datalad clone https://github.com/ReproNim/containers.git "$REPRONIM" - -# simbids has no upstream ReproNim home yet, so it can't be fetched — it's built -# from Docker Hub instead. Prefer apptainer, fall back to singularity (the -# slurm-docker-ci container ships the latter). -BUILDER=$(command -v apptainer || command -v singularity || true) - -for NAME in "${NAMES[@]}"; do - if [ "$NAME" = "bids-simbids" ]; then - # TODO(#62): add simbids upstream to ReproNim/containers so it's fetched - # like the others and this build branch can go away (its prerequisite - # for shim removal). Until then: - # No upstream SIF to `datalad get` — build it once into ReproNim's own - # images/bids/ layout (so it drops in cleanly when simbids lands - # upstream) and save it, so the shared --url add below has a local file - # to copy in. - [ -n "$BUILDER" ] || { echo "need apptainer or singularity to build simbids" >&2; exit 1; } - SIF_REL="images/bids/bids-simbids--0.0.3.sif" - if [ ! -e "$REPRONIM/$SIF_REL" ]; then - # Force an ANONYMOUS pull: apptainer reads ~/.docker/config.json and a - # stale credential there gets sent as bad auth (Docker Hub answers 401 - # "incorrect username or password"). apptainer >=1.5 ignores DOCKER_CONFIG, - # so point HOME at a throwaway dir for the build instead — with no - # ~/.docker/config.json to find, the pull is anonymous. Only the build - # subshell gets the fake HOME; the datalad steps keep the real one (they - # need ~/.gitconfig). simbids is public, so no-auth is correct. - HOME=$(mktemp -d) "$BUILDER" build "$REPRONIM/$SIF_REL" docker://pennlinc/simbids:0.0.3 - datalad -C "$REPRONIM" save -m "Build simbids SIF (no upstream ReproNim home)" "$SIF_REL" - fi - else - # ReproNim already registers pointing at its images/bids/...sif; - # read that path from containers-list, then fetch the SIF. - SIF_REL=$(datalad -C "$REPRONIM" containers-list | awk -v n="$NAME" '$1==n {print $3}') - [ -n "$SIF_REL" ] || { echo "no container '$NAME' in ReproNim containers-list" >&2; exit 1; } - datalad -C "$REPRONIM" get "$SIF_REL" - fi - # Re-register the image at babs's default location so vanilla babs finds it. - # --url with a BARE local path: datalad copies the file in (a file:// URL - # routes through `git annex addurl`, which annex.security.allowed-url-schemes - # blocks by default). - datalad containers-add -d "$REPRONIM" "$NAME" --update \ - -i ".datalad/environments/$NAME/image" \ - --url "$REPRONIM/$SIF_REL" \ - --call-fmt 'singularity exec --cleanenv {img} {cmd}' -done - -set +x -echo -echo "shim clone ready: $REPRONIM" -for NAME in "${NAMES[@]}"; do - echo " $NAME -> $REPRONIM/.datalad/environments/$NAME/image" -done -echo "point pipelines at it: container: {source: $REPRONIM, name: }"