Flexible pseudobulk designs, robust annotations and improved visualization

.github/workflows/tests.yml

@@ -27,7 +27,7 @@ concurrency:
 jobs:
   lint-and-dryrun:
     # Fast smoke test: lints the workflow and builds the full DAG on a tiny synthetic
-    # decoupled dataset (.tests/integration). No Conda envs or Space Ranger needed — a
+    # decoupled dataset (.test). No Conda envs or Space Ranger needed — a
     # dry-run validates the graph and (via the Snakefile) the config against the schema.
     runs-on: ubuntu-latest
     steps:
@@ -41,5 +41,7 @@ jobs:
         # (or an example config pointing at generated paths) cannot break CI.
       - name: Lint workflow
         run: snakemake -s workflow/Snakefile -d .test --lint --workflow-profile none
+      - name: Catalogue rule graph
+        run: snakemake -s workflow/Snakefile -c 1 -d .test --forceall --rulegraph > /dev/null
       - name: Dry-run (decoupled smoke test)
         run: snakemake -s workflow/Snakefile -d .test --dry-run --workflow-profile none

.gitignore

@@ -23,7 +23,7 @@ __pycache__/
 Thumbs.db
 
 # Test outputs
-.test/output/
+.test/test_output/
 
 # Doc build
 site/

.snakemake-workflow-catalog.yml

@@ -4,8 +4,9 @@
 usage:
   mandatory-flags:
     desc: >
-      Configure config/config.yaml and the required sample sheets before
-      execution. Space Ranger must be installed separately when required.
+      Copy config/config.yaml.template to config/config.yaml, then configure it
+      and the required sample sheets. Space Ranger must be installed separately
+      when required.
     flags: []
 
   software-stack-deployment:

.test/config/config.yaml

@@ -1,17 +1,17 @@
 # Canonical catalogue/CI fixture (mode: decoupled). Paths are relative to .test/.
 mode: decoupled
-post_processing_outdir: output
-logdir: logs
-spaceranger_processing_outdir: sr
-geojson_path: geojson
-contract_dir: data
+base_dir: test_output
+post_processing_outdir: "{base_dir}/output"
+logdir: "{base_dir}/logs"
+spaceranger_processing_outdir: "{base_dir}/sr"
+geojson_path: "{base_dir}/geojson"
+contract_dir: "data"
 random_seed: 0
 use_precomputed_clusters: false
 precomputed_metadata_dir: ''
 core_samples: config/core_samples.tsv
 xenium5k:
-  xenium_dir: /scratch/CBiB/soterino/xenium/{sample}
-  zarr_dir: ''
+  xenium_dir: "{base_dir}/xenium/{sample}"
   qupath_pyramid_level: 3
   hires_pyramid_level: 3
   pixel_size_um: 0.2125
@@ -106,13 +106,13 @@ spatial_niches:
   niche_dir: ''
   lambda: 0.8
   num_neighbours: 18
-  cluster_n_neighbours: 15
-  n_iterations: 2
+  cluster_n_neighbours: 50
+  n_iterations: -1
   max_m: 1
   nbr_weight_decay: scaled_gaussian
   use_hvg: true
   n_top_genes: 2000
-  niche_resolution: 0.3
+  niche_resolution: 0.1
   refine: false
   refine_num_neigh: 6
   refine_iterations: 1
@@ -266,7 +266,7 @@ resources:
     mem_mb: 65536
     runtime: 240
     threads: 4
-  explore_genes_spatial:
+  explore_genes_sample:
     mem_mb: 32768
     runtime: 360
     threads: 4

CONTRIBUTING.md

@@ -20,16 +20,19 @@ The practical consequence: **to support a new platform you write a new HeadBlock
 ## Repository layout
 
 ```
+.test/                tiny synthetic decoupled fixture for CI
+config/               config.yaml, README.md (config reference), sample sheets
+demos/                public-data worked examples
+docs/                 the MkDocs site
+images/               rulegraph and main image
+profiles/             Snakemake profile configuration
+tools/                scripts to extract external annotations and color the repo images/
 workflow/
 ├── Snakefile         globals, mode selection, named targets
 ├── rules/*.smk       one file per rule (+ common.smk for shared helpers)
 ├── scripts/*.py,*.R  rule implementations
 ├── envs/*.yaml       one Conda env per rule group (+ *_linux-64.lock)
 └── schemas/*.yaml    config + sample-sheet validation
-config/               config.yaml, README.md (config reference), sample sheets
-docs/                 the MkDocs site
-.test/                tiny synthetic decoupled fixture for CI
-reproduction/         public-data worked examples
 ```
 
 ## Development setup
@@ -41,9 +44,9 @@ standards.
 ```bash
 git clone https://github.com/cbib/SpaceBlocks && cd SpaceBlocks
 # Install the development environment
-conda create -f workflow/envs/dev.yaml
+conda env create -f workflow/envs/dev.yaml
 conda activate spaceblocks_dev
-snakemake -n --sdm conda      # dry-run: builds the DAG, validates the config, provisions envs
+snakemake -n --sdm conda      # dry-run: builds the DAG and validates the config
 snakemake -s workflow/Snakefile -d .test -n --workflow-profile none  # decoupled smoke test
 ```
 
@@ -73,7 +76,7 @@ These are load-bearing, and most past bugs we experienced during development cam
 - **Rules pass params explicitly.** A `.smk` rule reads from `config` and passes values through
   `params:`; the script reads `snakemake.params`, **never `config` directly**. After any change,
   cross-check that every param name matches between the `.smk` and its script.
-- **Resources scale with retries.** Don't   hardcode resources in a rule. Every compute rule draws `mem_mb`/`runtime`/`threads` from
+- **Resources scale with retries.** Don't hardcode resources in a rule. Every compute rule draws `mem_mb`/`runtime`/`threads` from
   `config["resources"]` (with a `default`), and `mem_mb` grows with the attempt number.
 - **The contract convention.** `obs["cell_id"]` must equal `obs_names` (as strings); downstream
   joins key on it. Head-produced contracts live at `SAMPLES_DIR/{sample}/{sample}_unfiltered.h5ad`
@@ -99,30 +102,35 @@ changes. See the `xenium5k` head as a reference implementation.
 
 ## Validating a change
 
-Run the applicable checks locally before pushing. The first two mirror CI and use the committed
-`mode: decoupled` fixture under `.test/`:
+Run the applicable checks locally before pushing. The lint and dry-run commands mirror CI and use
+the committed `mode: decoupled` fixture under `.test/`:
 
 ```bash
-# 1. Workflow parses + lints (against the committed test fixture)
-snakemake -s workflow/Snakefile -d .test --lint
+# 1. Formatting and file-hygiene hooks pass
+pre-commit run --all-files
+
+# 2. Workflow parses + lints (against the committed test fixture)
+snakemake -s workflow/Snakefile -d .test --lint --workflow-profile none
 
-# 2. The decoupled CoreBlock DAG builds
+# 3. The decoupled CoreBlock DAG builds
 snakemake -s workflow/Snakefile -d .test -n --workflow-profile none
 
-# 3. Config still validates against the schema
+# 4. Catalog fixture and user template still validate against the schema
 python -c "import yaml,jsonschema; jsonschema.validate(yaml.safe_load(open('config/config.yaml')), yaml.safe_load(open('workflow/schemas/config.schema.yaml'))); print('Configuration schema validation passed')"
+python -c "import yaml,jsonschema; jsonschema.validate(yaml.safe_load(open('config/config.yaml.template')), yaml.safe_load(open('workflow/schemas/config.schema.yaml'))); print('Configuration template schema validation passed')"
 
-# 4. Docs build cleanly (only if you touched docs/)
+# 5. Docs build cleanly (only if you touched docs/)
 # Activate the development environment
 conda activate spaceblocks_dev
 # Build the docs
 mkdocs build --strict
 ```
 
-Before a catalogue release, generate the decoupled rule graph once manually:
+Before a Catalog release, run the two checks used by the Catalog:
 
 ```bash
-snakemake -s workflow/Snakefile -d .test --forceall --rulegraph --workflow-profile none > pipeline_rulegraph.dot
+snakemake --lint
+snakemake -s workflow/Snakefile -c 1 -d .test --forceall --rulegraph > pipeline_rulegraph.dot
 ```
 
 Rendering the DOT file to SVG is optional and requires Graphviz:
@@ -131,7 +139,7 @@ Rendering the DOT file to SVG is optional and requires Graphviz:
 dot -Tsvg pipeline_rulegraph.dot > images/rulegraph.svg
 ```
 
-If you modify a HeadBlock, also dry-run the affected mode using a complete platform-specific configuration and representative mock or real inputs. Changing only `mode` in the generic `config/config.yaml` is not sufficient because its input paths are placeholders. Supported modes are `visiumhd`, `xenium5k`, `atera`, `merscope`, and `decoupled`.
+If you modify a HeadBlock, also dry-run the affected mode using a complete platform-specific configuration and representative mock or real inputs. Changing only `mode` in `config/config.yaml.template` is not sufficient because its input paths are placeholders. Supported modes are `visiumhd`, `xenium5k`, `atera`, `merscope`, and `decoupled`.
 
 Quick per-file sanity checks are cheap and worth it: `python -c "import ast; ast.parse(open('file.py').read())"`
 for Python, and `yaml.safe_load` for any YAML you edit.
@@ -155,7 +163,7 @@ Two workflows run on every PR and must pass:
 
 ## Submitting a pull request
 
-1. Branch from `main`, keep the change focused, and make sure the validation commands above pass.
+1. Branch from `dev`, keep the change focused, and make sure the validation commands above pass.
 2. Write a clear PR description: what changed and why. If behaviour changed, update the affected
    page under `docs/`.
 3. For bug fixes, a one-line note of the root cause in the PR helps reviewers.

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img src="docs/img/spaceblocks_logo.svg" alt="SpaceBlocks" width="640">
+  <img src="docs/img/spaceblocks_logo.svg" alt="SpaceBlocks" width="570">
 </p>
 <p align="center">
   <a href="https://cbib.github.io/SpaceBlocks/"><img src="https://img.shields.io/badge/docs-online-blue" alt="Documentation"></a>
@@ -16,9 +16,9 @@
 
 ## Streamlined, reproducible, spatial transcriptomics analyses with Snakemake
 
-SpaceBlocks is a technology-agnostic Snakemake workflow for **semi-automated analysis of single-cell-resolution spatial transcriptomics (ST)**.
+SpaceBlocks is a technology-agnostic Snakemake workflow for **expert-supervised cohort-level analysis of single cell resolution spatial transcriptomics (ST)**.
 
-It supports scalable and reproducible quality control, clustering, annotation, integration, differential expression, and spatial neighbourhood or niche analysis.
+It supports scalable and reproducible quality control, clustering, annotation, integration, differential expression, and spatial neighbourhood and niche analysis.
 
 Its modular architecture supports both public and in-house ST datasets while remaining maintainable and extensible.
 
@@ -31,21 +31,31 @@ Its modular architecture supports both public and in-house ST datasets while rem
 
 ## Overview
 
-The workflow is divided into (optional) technology-specific **HeadBlocks** and common **CoreBlocks** that streamline pre-, post-processing, and informative exploration of results.
+The workflow is divided into (optional) technology-specific **HeadBlocks** and common **CoreBlocks** that streamline preprocessing, postprocessing, and informative exploration of results.
 
-<p align="center"><img src="images/main.png" alt="SpaceBlocks workflow overview"><figcaption>Workflow overview and partial output from Xenium 5K demo example.</figcaption><width="700"></p>
+<br/>
 
-A full SpaceBlocks run takes three inputs:
+<p align="center">
+  <img src="images/main.png"
+       alt="SpaceBlocks workflow overview"
+       width="700">
+</p>
 
-1. **ST formatted AnnData objects**. Either generated from the HeadBlocks, or manually formatted as a standardized h5ad AnnData object. For brevity, **we refer to each of these objects as THE CONTRACT**. Their structure (count matrix, spatial coordinates and optional region annotations) is validated (`validate_input`) before downstream analyses.
+> **Overview of the SpaceBlocks workflow**. **(a)** Snakevision tubemap scheme showcasing the workflow rules, coloured by technology and processing step. The AnnData contract validation entry point and user-supervised steps are displayed in **bold font**. **(b)** Schematic representation of the outputs produced by the CoreBlocks subworkflow and main design features of the pipeline. **(c)** Runtime benchmarking across Visium HD, Xenium 5K and MERSCOPE datasets. β denotes the fitted scaling exponent; doubling the cell count multiplies the estimated wall-clock time by 2^β. **(d)** SpaceBlocks processing of the multiple sclerosis MERSCOPE dataset from Feng et al. (2025). Left, BANKSY niches annotated in the publication, shown on a UMAP. Right, the number of unique domain-specific upregulated genes identified by one-vs-rest Wald tests (Benjamini–Hochberg adjusted p-value < 0.05 and |log2FC| > 0.25).
 
-2. **Region annotations**. Provided in GeoJSON format.
+<br/>
+
+SpaceBlocks consumes ST data and can additionally use region and cell-type annotations:
+
+1. **ST formatted AnnData objects**. Either generated from the HeadBlocks, or manually formatted as a standardized h5ad AnnData object. For brevity, **we refer to each of these objects as THE CONTRACT**. Their structure (count matrix, spatial coordinates and optional region annotations) is validated (`validate_input`) before downstream analyses.
 
-3. **Cell type annotations**. Automatically generated through externally annotated references (`ingest`), manually annotating clusters (default) or providing externally generated annotations.
+2. **Region annotations (optional)**. Provided as GeoJSON files in HeadBlock modes, or already
+   stored in `obs["region_annotation"]` in a decoupled contract.
 
-The only mandatory input for SpaceBlocks is the actual ST data (1).
+3. **Cell type annotations**. Automatically transferred from a reference (`ingest`),
+   manually assigned from clusters (default), or supplied as external labels.
 
-SpaceBlocks provides tools to ease region annotation via QuPath (ideal if collaborating with pathologists) and cell type annotation (either manual, semi-automatic or external).
+SpaceBlocks provides tools to ease region annotation via QuPath (ideal if collaborating with pathologists) and cell type annotation (either manual, ingested or external).
 
 ## Quickstart
 
@@ -54,15 +64,15 @@ SpaceBlocks requires [Snakemake](https://snakemake.readthedocs.io) and Conda/Mam
 If you are not familiar with Snakemake, there are three key files to configure:
 
 1. `workflow/Snakefile` — loads the config file/s (normally, at `config/config.yaml`), and has the instructions to execute the workflow. It is normally static, so you should not modify it.
-2. The `config.yaml` — defines the parameters for your workflow run (i.e. where the input and output should be found, whether to use an external reference for cell type annotation, etc.). **It needs to be configured.** The config may require other files needed for the run (in the case of SpaceBlocks, `config/core_samples.tsv`).
-3. The `profiles/config.yaml` — Snakemake is a workflow manager that allows parallelization in systems with job schedulers (i.e. `slurm`), or local execution. **The profile needs to be adapted to your system.** We provide a default profile that runs locally and a slurm profile we used on our HPC.
+2. The `config.yaml` — defines the parameters for your workflow run (i.e. where the input and output should be found, whether to use an external reference for cell type annotation, etc.). Copy `config/config.yaml.template` to `config/config.yaml`, then configure it for your project. The config may require other files needed for the run (in the case of SpaceBlocks, `config/core_samples.tsv`).
+3. The `profiles/<profile>/config.yaml` — Snakemake is a workflow manager that allows parallelization in systems with job schedulers (i.e. `Slurm`), or local execution. **The profile needs to be adapted to your system.** We provide a default profile that runs the `test` pipeline locally and a slurm profile we used on our HPC.
 
 The full [configuration reference](https://cbib.github.io/SpaceBlocks/configuration/) presents a detailed explanation on how to set `config.yaml`.
 
 We recommend reading the [configuration reference](https://cbib.github.io/SpaceBlocks/configuration/) first, and then starting with the tutorial on [how to streamline a full run](https://cbib.github.io/SpaceBlocks/getting-started), and an [example with public data](https://cbib.github.io/SpaceBlocks/demos/).
 
 > [!TIP]
-> Any run can be fully reproduced end-to-end only with the experimental design and cell type annotations.
+> Once the input data is available, a run can be reproduced from its configuration, sample sheet, and any manual or external annotations.
 >
 > Color palettes for sample metadata, cell type annotations and spatial niches are fully customizable from `config/config.yaml`.
 
@@ -72,23 +82,27 @@ In short, to run the workflow:
 # 1. Get the workflow  (or: snakedeploy deploy-workflow cbib/SpaceBlocks spaceblocks --tag <version>)
 git clone https://github.com/cbib/SpaceBlocks && cd SpaceBlocks
 
-# 2. Configure
+# 2. Configure the config/config.yaml and the Snakemake profile in profiles/
+cp config/config.yaml.template config/config.yaml
 #    config/visiumhd_samples.csv   — if running mode : visiumhd
 #    config/core_samples.tsv       — one row per sample (+ any design columns)
-#    config/config.yaml            — mode: visiumhd | xenium5k | decoupled, and paths
-#    profiles/**/config.yaml        — profiles/default provides an example for slurm
+#    config/config.yaml            — mode: visiumhd | xenium5k | atera | merscope | decoupled
+#    profiles/**/config.yaml       — local and Slurm execution examples
 #
 #    Note: Invoke profile manually (e.g. --profile profiles/default) or
 #          set environment variable (e.g. export SNAKEMAKE_PROFILE=profiles/default )
 
-# 3. Check the plan
+# 3. Dry-run your SpaceBlocks configuration
 snakemake -n --sdm conda
 
-# 4. Make the QuPath images, annotate regions in QuPath, then run the core
-snakemake qupath_images qc_sweep  --sdm conda   # → annotate, export GeoJSON to config["geojson_path"] + sweep QC thresholds
-snakemake run_preprocessing       --sdm conda   # validate → QC/normalise → cluster
-snakemake run_postprocessing      --sdm conda   # annotate → integrate → DE → reports + subclusters
-snakemake run_exploration         --sdm conda   # explore genes (integrated and per sample)
+# 4. Make the QuPath images (only when using the HeadBlocks)
+snakemake qupath_images       --sdm conda
+# 5. Annotate regions in QuPath and screen the QCs
+snakemake  qc_sweep_all       --sdm conda    # sweep QC thresholds
+# 6. Commit the QCs and run the CoreBlocks
+snakemake run_preprocessing   --sdm conda    # validate → QC/normalise → cluster
+snakemake run_postprocessing  --sdm conda    # annotate → integrate → DE → reports + subclusters
+snakemake run_exploration     --sdm conda    # explore genes (integrated and per sample)
 ```
 
 ## Documentation
@@ -118,4 +132,4 @@ Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md) for the arc
 
 ## License
 
-Released under the MIT License — see [LICENSE](LICENSE).
+Released under the MIT License — see [LICENSE](LICENSE.md).