diff --git a/.test/config/config.yaml b/.test/config/config.yaml index 6e75510..9112652 100644 --- a/.test/config/config.yaml +++ b/.test/config/config.yaml @@ -1,16 +1,17 @@ # Canonical catalogue/CI fixture (mode: decoupled). Paths are relative to .test/. mode: decoupled -post_processing_outdir: output -logdir: logs -spaceranger_processing_outdir: sr -geojson_path: geojson -contract_dir: data +base_dir: test_output +post_processing_outdir: "{base_dir}/output" +logdir: "{base_dir}/logs" +spaceranger_processing_outdir: "{base_dir}/sr" +geojson_path: "{base_dir}/geojson" +contract_dir: "data" random_seed: 0 use_precomputed_clusters: false precomputed_metadata_dir: '' core_samples: config/core_samples.tsv xenium5k: - xenium_dir: /scratch/CBiB/soterino/xenium/{sample} + xenium_dir: "{base_dir}/xenium/{sample}" zarr_dir: '' qupath_pyramid_level: 3 hires_pyramid_level: 3 @@ -266,7 +267,7 @@ resources: mem_mb: 65536 runtime: 240 threads: 4 - explore_genes_spatial: + explore_genes_sample: mem_mb: 32768 runtime: 360 threads: 4 diff --git a/README.md b/README.md index 72e1706..3ea8e3e 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ If you are not familiar with Snakemake, there are three key files to configure: 1. `workflow/Snakefile` — loads the config file/s (normally, at `config/config.yaml`), and has the instructions to execute the workflow. It is normally static, so you should not modify it. 2. The `config.yaml` — defines the parameters for your workflow run (i.e. where the input and output should be found, whether to use an external reference for cell type annotation, etc.). **It needs to be configured.** The config may require other files needed for the run (in the case of SpaceBlocks, `config/core_samples.tsv`). -3. The `profiles/config.yaml` — Snakemake is a workflow manager that allows parallelization in systems with job schedulers (i.e. `slurm`), or local execution. **The profile needs to be adapted to your system.** We provide a default profile that runs locally and a slurm profile we used on our HPC. +3. The `profiles/config.yaml` — Snakemake is a workflow manager that allows parallelization in systems with job schedulers (i.e. `slurm`), or local execution. **The profile needs to be adapted to your system.** We provide a default profile that runs the `test` pipeline locally and a slurm profile we used on our HPC. The full [configuration reference](https://cbib.github.io/SpaceBlocks/configuration/) presents a detailed explanation on how to set `config.yaml`. diff --git a/config/README.md b/config/README.md index d85d143..a279db1 100644 --- a/config/README.md +++ b/config/README.md @@ -31,21 +31,55 @@ The analysis **CoreBlocks** are identical in all five cases. Atera support is cu The SpaceBlocks `config/config.yaml` sets the input and output directories, and gets the sample information and metadata (for integration and plotting) from one or two TSV files, depending on the `config["mode"]` set. +!!! tip "Setting a base directory" + `base_dir` is the project-root prefix for generated output directories. In the shipped template, most output folders are written relative to it, but several required inputs are intentionally left as explicit filesystem paths because they point to external software or reference data. + +Paths in SpaceBlocks fall into three categories: + +- Project output paths: typically relative to `base_dir` or kept under the config directory for metadata files. +- Project input paths: configuration or pre-computed data used by the pipeline. These can be defined relative to `config/` or as absolute paths. +- External absolute paths: installed tools, references, and other files that must be supplied explicitly. + +### Project output paths + +These are the paths that are normally defined relative to `base_dir`. + | Path / File | Key | Condition | Meaning | | --- | --- | --- | --- | -| Path | `spaceranger_processing_outdir` | Head modes | Directory where the head writes heavy intermediates such as Space Ranger outputs or SpatialData Zarr stores. | -| File | `spaceranger` | `mode: visiumhd` | Path to the Space Ranger installation. | -| File | `probe_set` | `mode: visiumhd` | [Probe set CSV](https://www.10xgenomics.com/support/spatial-gene-expression-hd/documentation/steps/probe-sets) needed to run Space Ranger. | -| File | `transcriptome` | `mode: visiumhd` | [Reference transcriptome](https://www.10xgenomics.com/support/software/space-ranger/downloads) needed to run Space Ranger. | | Path | `post_processing_outdir` | **Mandatory** | Output directory root; it will contain the result folders for the run. | | Path | `logdir` | **Mandatory** | Directory for per-rule logs and benchmarks. | -| Path | `geojson_path` | **Optional, recommended** | Directory containing the GeoJSONs that annotate spatial regions for each sample (see the [QuPath tutorial](https://cbib.github.io/SpaceBlocks/qupath-tutorial/)). | +| Path | `spaceranger_processing_outdir` | Head modes | Directory where the head writes heavy intermediates such as Space Ranger outputs or SpatialData Zarr stores. | + +### Project input paths + +These paths contain metadata, configuration or pre-processed results that are ingested by the pipeline. We provide some defaults under the `config/` directory. Others are defined as placeholders `/path/to/dir/` and `/path/to/file.ext`, so make sure to configure them if needed. + +| Path / File | Key | Condition | Meaning | +| --- | --- | --- | --- | +| File | `core_samples` | **Mandatory** | Sample metadata sheet. The shipped template keeps this in `config/` as project metadata. | | Path | `contract_dir` | `mode: decoupled` | Directory holding the pre-existing contract h5ads to analyse. | +| Path | `xenium5k.xenium_dir` | `mode: xenium5k` | Directory pattern for each Xenium bundle. | +| Path | `spatial_niches.niche_dir` | `spatial_niches` | Directory for precomputed niche TSVs when `use_precomputed` is enabled. | +| Path | `geojson_path` | **Optional, recommended** | Directory containing the GeoJSONs that annotate spatial regions for each sample (see the [QuPath tutorial](https://cbib.github.io/SpaceBlocks/qupath-tutorial/)). | +| Path | `precomputed_metadata_dir` | **Optional** (reproducibility) | Directory of precomputed per-sample metadata TSVs. The metadata can contain precomputed clusters and/or external annotations. | | File | `per_sample_qc` | **Optional** (per-sample filters) | TSV of sample-specific QC filters. If empty, the same `analysis.*` thresholds apply to every sample (*default*). | -| File | `snakemake_cell_markers` | **Optional** (pre-annotation) | TSV of canonical marker genes drawn on the Leiden diagnostic plots (`leiden_analysis`) to guide manual annotation. | -| File | `gene_exploration.queries` (`config/gene_queries.tsv`) | Exploration Coreblock | Genes / gene sets to score (AUCell) and plot in the exploration block. | +| File | `snakemake_cell_markers` | **Optional** (pre-annotation) | TSV of canonical marker genes drawn on the Leiden diagnostic plots (`leiden_analysis`) to guide manual annotation. If not set, default markers are used (`config/snakemake_cell_markers.tsv`) | +| File | `gene_exploration.queries` | Exploration Coreblock | Genes / gene sets to score (AUCell) and plot in the exploration block. If not set, test gene queries are used (`config/gene_queries.tsv`). | +| File | `cluster_annotations` | **Optional** | TSV of cluster-to-cell-type labels used during annotation. Template ships with a template of placeholder annotations (`config/cluster_annotations.tsv`). If not set, cell will be labeled `Unannotated` | + +### External absolute paths + +These are not derived from `base_dir` and must be set explicitly to the correct filesystem location. + +| Path / File | Key | Condition | Meaning | +| --- | --- | --- | --- | +| File | `spaceranger` | `mode: visiumhd` | Path to the external Space Ranger installation. | +| File | `probe_set` | `mode: visiumhd` | [Probe set CSV](https://www.10xgenomics.com/support/spatial-gene-expression-hd/documentation/steps/probe-sets) needed to run Space Ranger. | +| File | `transcriptome` | `mode: visiumhd` | [Reference transcriptome](https://www.10xgenomics.com/support/software/space-ranger/downloads) needed to run Space Ranger. | | File | `ingest_ref` | **Optional** (auto-annotation) | Annotated scRNA-seq reference h5ad for `ingest_ref`. | -| Path | `precomputed_metadata_dir` | **Optional** (reproducibility) | Directory of precomputed per-sample metadata TSVs. The metadata can contain precomputed clusters and/or external annotations. | +| File | `samples` | `mode: visiumhd` | Sample sheet with fastq directories, slide, and capture area per sample. | + +The template uses placeholder values such as `/path/to/...` for these external inputs to make it clear they must be replaced before running the workflow. `core_samples.tsv` is the technology-agnostic sample sheet used by the CoreBlocks in every use case. Visium HD additionally uses `visiumhd_samples.csv` for fastq, slide, and capture-area information. @@ -153,7 +187,7 @@ samples: "config/visiumhd_samples.csv" # fastq dirs / slide / area per spaceranger: "/path/to/spaceranger" probe_set: "/path/to/probe_set.csv" transcriptome: "/path/to/refdata-gex" -spaceranger_processing_outdir: "/path/to/sr_out" +spaceranger_processing_outdir: "{base_dir}/sr_out" ``` **Xenium 5K** (`mode: xenium5k`) @@ -197,7 +231,7 @@ mode: "decoupled" contract_dir: "/path/to/contract_h5ads" # one .h5ad file per sample ``` -All modes additionally set `core_samples`, `post_processing_outdir`, `logdir`, and the other required common keys in `config/config.yaml`; **`geojson_path` is optional but strongly recommended**. +All modes additionally set `core_samples`, `post_processing_outdir`, `logdir`, and the other required common keys in `config/config.yaml`; **`geojson_path` is optional but strongly recommended** and may be left as a project-local directory or an explicit path. ## 8. Minimal example @@ -205,8 +239,9 @@ All modes additionally set `core_samples`, `post_processing_outdir`, `logdir`, a mode: "visiumhd" samples: "config/visiumhd_samples.csv" core_samples: "config/core_samples.tsv" -geojson_path: "/path/to/geojson" -post_processing_outdir: "/path/to/results" +geojson_path: "path/to/geojson" +precomputed_metadata_dir: "/path/to/metadata_visiumhd/" +post_processing_outdir: "{base_dir}/results" # ... see config/config.yaml for the full, commented template. ``` diff --git a/config/config.yaml b/config/config.yaml index 9ebef73..c383732 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -5,17 +5,29 @@ mode: decoupled # ─── input/output directories ──────────────────────────────────────────────── -post_processing_outdir: /scratch/CBiB/soterino/smk_visiumhd_results -logdir: /scratch/CBiB/soterino/smk_logs -spaceranger_processing_outdir: /scratch/CBiB/soterino/smk_spaceranger_results +## BASE DIRECTORY ############################################################# +## +## You may modify 'base_dir' to point to your project results directory. +## Paths relative to base_dir can be left as is, +## or be overridden with absolute paths if desired. + +base_dir: SpaceBlocksProject + +## Paths that are not relative to base_dir and must be set explicitly below: +# spaceranger, probe_set, transcriptome, ingest_ref +############################################################################### + +post_processing_outdir: "{base_dir}/output" +logdir: "{base_dir}/logs" +spaceranger_processing_outdir: "{base_dir}/spaceranger_results" # dir with region annotation GeoJSONs (optional, recommended) -geojson_path: /scratch/CBiB/soterino/geojson +geojson_path: "/path/to/geojson" # only when using "mode: decoupled" — dir of pre-built contract h5ads contract_dir: '' # ─── Reproducibility & reuse (docs → Configuration) ──────────────────────── random_seed: 0 -precomputed_metadata_dir: /scratch/CBiB/soterino/metadata_visiumhd/ +precomputed_metadata_dir: "/path/to/metadata_visiumhd/" # reuse clusters from precomputed_metadata_dir use_precomputed_clusters: true @@ -25,14 +37,15 @@ core_samples: config/core_samples.tsv samples: config/visiumhd_samples.csv # ─── Visium HD head (mode: visiumhd) ─────────────────────────────────────── -spaceranger: /home/soterinosogo/spaceranger-4.0.1/spaceranger # external install; path to the executable -probe_set: /scratch/CBiB/soterino/Visium_Human_Transcriptome_Probe_Set_v2.1.0_GRCh38-2024-A.csv -transcriptome: /mnt/cbib/bank/cellranger/refdata-gex-GRCh38-2024-A +spaceranger: /path/to/spaceranger-4.0.1/spaceranger # external install; path to the executable +probe_set: /path/to/Visium_Human_Transcriptome_Probe_Set_v2.1.0_GRCh38-2024-A.csv +transcriptome: /path/to/cellranger/refdata-gex-GRCh38-2024-A # ─── Xenium 5K head (mode: xenium5k) ─────────────────────────────────────── xenium5k: - xenium_dir: /scratch/CBiB/soterino/xenium/{sample} - zarr_dir: '' + # xenium_dir is the directory containing the Xenium 5K samples. + # Make sure to keep the {sample} placeholder in the path. + xenium_dir: "/path/to/xenium/{sample}" qupath_pyramid_level: 3 hires_pyramid_level: 3 pixel_size_um: 0.2125 @@ -136,7 +149,7 @@ external_annotation: enabled: false column: manual_celltype keep_unannotated: true -ingest_ref: /scratch/CBiB/soterino/ingest_ref_data/bcc_and_normal_Garnier_PNAS_2024.h5ad # optional annotated scRNA-seq reference (ingest_ref) +ingest_ref: "/path/to/ingest_ref_data/bcc_and_normal_Garnier_PNAS_2024.h5ad" # optional annotated scRNA-seq reference (ingest_ref) ingest_ref_label_key: 04_celltypes integration: integrate_key: sample @@ -151,7 +164,7 @@ spatial_niches: enabled: true # use niche_dir precomputed .tsv use_precomputed: true - niche_dir: /scratch/CBiB/soterino/niches_visiumhd/ + niche_dir: "/path/to/niches_visiumhd/" lambda: 0.8 num_neighbours: 18 cluster_n_neighbours: 15 diff --git a/docs/getting-started.md b/docs/getting-started.md index 40ae2e4..7789f0f 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -14,6 +14,8 @@ If you want to see an example run on public data, see the [public data end-to-en - [QuPath](https://qupath.github.io/) (desktop), recommended to include manual region annotations for the downstream analyses (see [QuPath annotation tutorial](qupath-tutorial.md)). - Configure your Snakemake profile in `--profile profiles/default`. An example for HPCs with Slurm scheduler is provided (tested with version 24.11.5). +!!! tip "Setting project directory" The default profile is configured to run the `test` project. Change the path to `directory: .` to run the project on the root of the SpaceBlocks repository. + We recommend to always run snakemake with `--sdm conda`, so each rule is executed within its pinned environment. ## 2. Configure diff --git a/images/rulegraph.svg b/images/rulegraph.svg index 21e46b4..11b93e7 100644 --- a/images/rulegraph.svg +++ b/images/rulegraph.svg @@ -1 +1,265 @@ -spaceranger_count_vhdgenerate_annotation_templateconvert_zarr_x5kconvert_zarr_ateprepare_input_mergenerate_qupath_merprepare_input_vhdgenerate_qupath_vhdprepare_input_x5kgenerate_qupath_x5kprepare_input_ategenerate_qupath_atevalidate_inputpreprocess_umapleiden_analysisspatial_nichesingest_refannotate_cellsneighbourhood_analysisintegrate_samplessample_reportpseudobulk_aggregateexplore_genes_integratedsubclusterpseudobulk_deexplore_genes_sampleHeadblocksCoreblocksVisium HDXenium 5KAteraMERSCOPEPreprocessingPostprocessingExploration \ No newline at end of file + + + + + + +snakemake_dag + + + +0 + +all + + + +1 + +preprocess_umap + + + +1->0 + + + + + +3 + +leiden_analysis + + + +1->3 + + + + + +4 + +spatial_niches + + + +1->4 + + + + + +5 + +annotate_cells + + + +1->5 + + + + + +2 + +validate_input + + + +2->1 + + + + + +3->0 + + + + + +4->0 + + + + + +4->5 + + + + + +5->0 + + + + + +6 + +neighbourhood_analysis + + + +5->6 + + + + + +8 + +integrate_samples + + + +5->8 + + + + + +10 + +sample_report + + + +5->10 + + + + + +12 + +explore_genes_sample + + + +5->12 + + + + + +6->0 + + + + + +7 + +pseudobulk_aggregate + + + +7->0 + + + + + +9 + +pseudobulk_de + + + +7->9 + + + + + +8->0 + + + + + +8->7 + + + + + +11 + +explore_genes_integrated + + + +8->11 + + + + + +13 + +subcluster + + + +8->13 + + + + + +9->0 + + + + + +10->0 + + + + + +11->0 + + + + + +11->12 + + + + + +12->0 + + + + + +13->0 + + + + + +14 + +generate_annotation_template + + + +14->0 + + + + + diff --git a/profiles/default/config.yaml b/profiles/default/config.yaml index cd838dd..4a84d53 100644 --- a/profiles/default/config.yaml +++ b/profiles/default/config.yaml @@ -3,7 +3,15 @@ # ═══════════════════════════════════════════════════════════════════════════ # Invoke with: snakemake --profile profiles/default # Set as default for a shell session with: export SNAKEMAKE_PROFILE=profiles/default - +# IMPORTANT +# "directory: .test" is necessary for Snakemake catalog's linting +# Switch from "directory: .test" to "directory: ." when running from the repository root +# +# IMPORTANT +# "directory: .test" is necessary for Snakemake catalog's linting +# Switch from "directory: .test" to "directory: ." when running from the repository root, +# or define a custom directory "directory: /path/to/your/working/directory" to run from anywhere. +directory: .test executor: local jobs: 4 diff --git a/profiles/slurm/config.yaml b/profiles/slurm/config.yaml index 24246fb..e161e44 100644 --- a/profiles/slurm/config.yaml +++ b/profiles/slurm/config.yaml @@ -3,7 +3,12 @@ # ═══════════════════════════════════════════════════════════════════════════ # Invoke with: snakemake --profile profiles/slurm # Set as default for a shell session with: export SNAKEMAKE_PROFILE=profiles/slurm - +# +# IMPORTANT +# "directory: .test" is necessary for Snakemake catalog's linting +# Switch from "directory: .test" to "directory: ." when running from the repository root, +# or define a custom directory "directory: /path/to/your/working/directory" to run from anywhere. +directory: .test executor: slurm jobs: 30 diff --git a/workflow/Snakefile b/workflow/Snakefile index b529e9d..43e9600 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -27,13 +27,18 @@ configfile: "config/config.yaml" # configfile: "demos/atera/atera_config.yaml" # alpha: Atera HeadBlock test # configfile: "demos/merscope/merscope_config.yaml" -validate(config, schema="schemas/config.schema.yaml") - # Helper functions include: "rules/common.smk" +# Expand config paths recursively +config = _expand_base_dir(config) + +# Validate config against schema after path expansion +validate(config, schema="schemas/config.schema.yaml") + + # ── Globals ────────────────────────────────────────────────────────────────── ANALYSIS = config.get("analysis", {}) _region_colors = ANALYSIS.get("region_colors", {}) @@ -47,7 +52,7 @@ RANDOM_SEED = config.get("random_seed", 42) # Xenium 5K head specifics XENIUM5K = config.get("xenium5k", {}) or {} -XENIUM_ZARR_DIR = XENIUM5K.get("zarr_dir", "") or OUTDIR_SR +XENIUM_ZARR_DIR = OUTDIR_SR # Atera head specifics (ALPHA — the platform ships H2 2026 and its output format is # still a preview; see docs/rules.md). diff --git a/workflow/envs/dev.yaml b/workflow/envs/dev.yaml index 77d8837..25df53f 100644 --- a/workflow/envs/dev.yaml +++ b/workflow/envs/dev.yaml @@ -15,3 +15,6 @@ dependencies: - mkdocs - pymdown-extensions - mkdocs-material + + # Graphviz (for dot) + - graphviz diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 2db86c5..f501c5d 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -33,6 +33,28 @@ def mem_mb_attempt(rule_name, cap_factor=None): return _mem +# ── Expand {base_dir} in config paths ──────────────────────────────────────── +BASE_DIR = config.get("base_dir", "") + + +def _expand_base_dir(node, root="config"): + """ + Expand config paths (recursively), replacing only {base_dir} + leaving other placeholders (for example, the wildcard {sample}) intact. + """ + if isinstance(node, str): + if "{base_dir}" in node: + expanded = node.replace("{base_dir}", BASE_DIR) + print(f"[config] expanded {root}: {expanded}", file=sys.stderr) + return expanded + return node + if isinstance(node, dict): + return {k: _expand_base_dir(v, f"{root}.{k}") for k, v in node.items()} + if isinstance(node, list): + return [_expand_base_dir(v, f"{root}[{i}]") for i, v in enumerate(node)] + return node + + # ── Small derivations called from the Snakefile globals block ──────────────── def check_external_annotation(): """Fail fast (at parse time, before any job) when external_annotation is enabled diff --git a/workflow/rules/explore_genes.smk b/workflow/rules/explore_genes.smk index 8915d73..72c9264 100644 --- a/workflow/rules/explore_genes.smk +++ b/workflow/rules/explore_genes.smk @@ -8,7 +8,7 @@ rule explore_genes_integrated: """ input: integrated=rules.integrate_samples.output.harmony, - queries=GENE_EXPLORATION.get("queries", "") or [], + queries=GENE_EXPLORATION.get("queries", "config/gene_queries.tsv"), output: ranges=f"{OUTDIR_PP}/gene_exploration/expression_ranges.tsv", done=touch(f"{OUTDIR_PP}/gene_exploration/.integrated_done"), diff --git a/workflow/rules/leiden_analysis.smk b/workflow/rules/leiden_analysis.smk index 8081868..79581d7 100644 --- a/workflow/rules/leiden_analysis.smk +++ b/workflow/rules/leiden_analysis.smk @@ -8,7 +8,9 @@ rule leiden_analysis: """ input: adata=rules.preprocess_umap.output.adata, - cell_markers=config["snakemake_cell_markers"], + cell_markers=config.get( + "snakemake_cell_markers", "config/snakemake_cell_markers.tsv" + ), output: res_dir=directory(f"{SAMPLES_DIR}/{{sample}}/leiden_resolution_{{resolution}}"), log: diff --git a/workflow/schemas/config.schema.yaml b/workflow/schemas/config.schema.yaml index a8f9cce..6e5bbba 100644 --- a/workflow/schemas/config.schema.yaml +++ b/workflow/schemas/config.schema.yaml @@ -157,6 +157,12 @@ properties: override the analysis.* defaults per sample, key-by-key; missing samples/columns fall back to config. Empty string disables overrides. + base_dir: + type: string + description: >- + Base project directory used by config path templates (for example, + "{base_dir}/output"). + mode: type: string enum: ["visiumhd", "xenium5k", "atera", "merscope", "decoupled"]