diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 53d31e8..ddf802f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,7 +27,7 @@ concurrency: jobs: lint-and-dryrun: # Fast smoke test: lints the workflow and builds the full DAG on a tiny synthetic - # decoupled dataset (.tests/integration). No Conda envs or Space Ranger needed — a + # decoupled dataset (.test). No Conda envs or Space Ranger needed — a # dry-run validates the graph and (via the Snakefile) the config against the schema. runs-on: ubuntu-latest steps: @@ -41,5 +41,7 @@ jobs: # (or an example config pointing at generated paths) cannot break CI. - name: Lint workflow run: snakemake -s workflow/Snakefile -d .test --lint --workflow-profile none + - name: Catalogue rule graph + run: snakemake -s workflow/Snakefile -c 1 -d .test --forceall --rulegraph > /dev/null - name: Dry-run (decoupled smoke test) run: snakemake -s workflow/Snakefile -d .test --dry-run --workflow-profile none diff --git a/.gitignore b/.gitignore index 468aab2..046e9df 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,7 @@ __pycache__/ Thumbs.db # Test outputs -.test/output/ +.test/test_output/ # Doc build site/ diff --git a/.snakemake-workflow-catalog.yml b/.snakemake-workflow-catalog.yml index d5aaa13..703629f 100644 --- a/.snakemake-workflow-catalog.yml +++ b/.snakemake-workflow-catalog.yml @@ -4,8 +4,9 @@ usage: mandatory-flags: desc: > - Configure config/config.yaml and the required sample sheets before - execution. Space Ranger must be installed separately when required. + Copy config/config.yaml.template to config/config.yaml, then configure it + and the required sample sheets. Space Ranger must be installed separately + when required. flags: [] software-stack-deployment: diff --git a/.test/config/config.yaml b/.test/config/config.yaml index 4ac7fd6..715a1ec 100644 --- a/.test/config/config.yaml +++ b/.test/config/config.yaml @@ -12,7 +12,6 @@ precomputed_metadata_dir: '' core_samples: config/core_samples.tsv xenium5k: xenium_dir: "{base_dir}/xenium/{sample}" - zarr_dir: '' qupath_pyramid_level: 3 hires_pyramid_level: 3 pixel_size_um: 0.2125 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cdb3578..13f0592 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -112,8 +112,9 @@ snakemake -s workflow/Snakefile -d .test --lint --workflow-profile none # 3. The decoupled CoreBlock DAG builds snakemake -s workflow/Snakefile -d .test -n --workflow-profile none -# 4. Config still validates against the schema +# 4. Catalog fixture and user template still validate against the schema python -c "import yaml,jsonschema; jsonschema.validate(yaml.safe_load(open('config/config.yaml')), yaml.safe_load(open('workflow/schemas/config.schema.yaml'))); print('Configuration schema validation passed')" +python -c "import yaml,jsonschema; jsonschema.validate(yaml.safe_load(open('config/config.yaml.template')), yaml.safe_load(open('workflow/schemas/config.schema.yaml'))); print('Configuration template schema validation passed')" # 5. Docs build cleanly (only if you touched docs/) # Activate the development environment @@ -122,10 +123,11 @@ conda activate spaceblocks_dev mkdocs build --strict ``` -Before a catalogue release, generate the decoupled rule graph once manually: +Before a Catalog release, run the two checks used by the Catalog: ```bash -snakemake -s workflow/Snakefile -d .test --forceall --rulegraph --workflow-profile none > pipeline_rulegraph.dot +snakemake --lint +snakemake -s workflow/Snakefile -c 1 -d .test --forceall --rulegraph > pipeline_rulegraph.dot ``` Rendering the DOT file to SVG is optional and requires Graphviz: @@ -134,7 +136,7 @@ Rendering the DOT file to SVG is optional and requires Graphviz: dot -Tsvg pipeline_rulegraph.dot > images/rulegraph.svg ``` -If you modify a HeadBlock, also dry-run the affected mode using a complete platform-specific configuration and representative mock or real inputs. Changing only `mode` in the generic `config/config.yaml` is not sufficient because its input paths are placeholders. Supported modes are `visiumhd`, `xenium5k`, `atera`, `merscope`, and `decoupled`. +If you modify a HeadBlock, also dry-run the affected mode using a complete platform-specific configuration and representative mock or real inputs. Changing only `mode` in `config/config.yaml.template` is not sufficient because its input paths are placeholders. Supported modes are `visiumhd`, `xenium5k`, `atera`, `merscope`, and `decoupled`. Quick per-file sanity checks are cheap and worth it: `python -c "import ast; ast.parse(open('file.py').read())"` for Python, and `yaml.safe_load` for any YAML you edit. diff --git a/README.md b/README.md index 3ea8e3e..e701832 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ SpaceBlocks requires [Snakemake](https://snakemake.readthedocs.io) and Conda/Mam If you are not familiar with Snakemake, there are three key files to configure: 1. `workflow/Snakefile` — loads the config file/s (normally, at `config/config.yaml`), and has the instructions to execute the workflow. It is normally static, so you should not modify it. -2. The `config.yaml` — defines the parameters for your workflow run (i.e. where the input and output should be found, whether to use an external reference for cell type annotation, etc.). **It needs to be configured.** The config may require other files needed for the run (in the case of SpaceBlocks, `config/core_samples.tsv`). +2. The `config.yaml` — defines the parameters for your workflow run (i.e. where the input and output should be found, whether to use an external reference for cell type annotation, etc.). Copy `config/config.yaml.template` to `config/config.yaml`, then configure it for your project. The config may require other files needed for the run (in the case of SpaceBlocks, `config/core_samples.tsv`). 3. The `profiles/config.yaml` — Snakemake is a workflow manager that allows parallelization in systems with job schedulers (i.e. `slurm`), or local execution. **The profile needs to be adapted to your system.** We provide a default profile that runs the `test` pipeline locally and a slurm profile we used on our HPC. The full [configuration reference](https://cbib.github.io/SpaceBlocks/configuration/) presents a detailed explanation on how to set `config.yaml`. @@ -73,6 +73,7 @@ In short, to run the workflow: git clone https://github.com/cbib/SpaceBlocks && cd SpaceBlocks # 2. Configure +cp config/config.yaml.template config/config.yaml # config/visiumhd_samples.csv — if running mode : visiumhd # config/core_samples.tsv — one row per sample (+ any design columns) # config/config.yaml — mode: visiumhd | xenium5k | decoupled, and paths diff --git a/config/README.md b/config/README.md index 89fbb4f..9f2b3d0 100644 --- a/config/README.md +++ b/config/README.md @@ -1,6 +1,6 @@ This page explains how to fully set up the config files to run SpaceBlocks. Full documentation can be found at https://cbib.github.io/SpaceBlocks/configuration/. -SpaceBlocks is configured through **`config/config.yaml`**, which contains all the parameters needed for the run, plus one or two **sample sheets**. +Copy **`config/config.yaml.template`** to **`config/config.yaml`** before configuring a run. The template contains all parameters; the committed `config/config.yaml` is only the runnable fixture used by the Snakemake Workflow Catalog. The explanations on this page are divided by type (directory, parameter, color, and so on). In the config file, parameters are divided by Block and function. @@ -124,6 +124,9 @@ sample_colors: patient: "Patient 1": "#000000" "Patient 2": "#E69F00" + condition: + "Control": "#0072B2" + "Treatment": "#D55E00" batch: "Batch 1": "#8E44AD" @@ -153,7 +156,7 @@ The rest of the configuration lives in nested blocks. Files and single parameter | Section | Condition | What it configures | | --- | --- | --- | -| `xenium5k` | `mode: xenium5k` | Xenium head settings: `xenium_dir`, `zarr_dir`, pyramid levels, and `pixel_size_um`. | +| `xenium5k` | `mode: xenium5k` | Xenium head settings: `xenium_dir`, pyramid levels, and `pixel_size_um`. | | `atera` | `mode: atera` | Atera head settings: `atera_dir`, Zarr and pyramid options, plus optional registered H&E image, alignment, and keypoint patterns. | | `merscope` | `mode: merscope` | MERSCOPE head settings: `merscope_dir`, selected z-plane, embedded-image resolution, and image channels. | | `contract` | **Mandatory** | Semantic keys of the hand-off object: `sample_key`, `spatial_key`, `require_region`, `require_raw_counts`, `mito_prefix`. | @@ -340,7 +343,7 @@ These files are generated during the run, and can be shared with minimum effort ## 7. Example use case configurations -The commented `config/config.yaml` is the full template. The mode-specific keys that differ are: +The commented `config/config.yaml.template` is the full template. The mode-specific keys that differ are: **Visium HD** (`mode: visiumhd`) ```yaml @@ -357,7 +360,6 @@ spaceranger_processing_outdir: "{base_dir}/sr_out" mode: "xenium5k" xenium5k: xenium_dir: "/path/to/xenium/{sample}" # {sample} pattern to each bundle - zarr_dir: "" # "" → spaceranger_processing_outdir qupath_pyramid_level: 3 hires_pyramid_level: 3 pixel_size_um: 0.2125 @@ -368,7 +370,6 @@ xenium5k: mode: "atera" atera: atera_dir: "/path/to/atera/{sample}/outs" - zarr_dir: "" qupath_pyramid_level: 3 hires_pyramid_level: 3 pixel_size_um: 0.2125 @@ -398,15 +399,15 @@ All modes additionally set `core_samples`, `post_processing_outdir`, `logdir`, a ## 8. Minimal example ```yaml -mode: "visiumhd" -samples: "config/visiumhd_samples.csv" +mode: "decoupled" core_samples: "config/core_samples.tsv" -geojson_path: "path/to/geojson" -precomputed_metadata_dir: "/path/to/metadata_visiumhd/" +contract_dir: "/path/to/contract_h5ads" +geojson_path: "/path/to/geojson" +precomputed_metadata_dir: "/path/to/precomputed_metadata/" post_processing_outdir: "{base_dir}/results" -# ... see config/config.yaml for the full, commented template. +# ... see config/config.yaml.template for the full, commented template. ``` -Start from the commented `config/config.yaml` shipped with the workflow and adjust the paths and sample sheets to your data. +Copy `config/config.yaml.template` to `config/config.yaml`, then adjust the paths and sample sheets to your data. You may next read the [get started](https://cbib.github.io/SpaceBlocks/getting-started/) documentation and the [public data end-to-end example runs](https://cbib.github.io/SpaceBlocks/demos/). diff --git a/config/config.yaml b/config/config.yaml index 4bcaa2c..d1724ab 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,56 +1,21 @@ -# SpaceBlocks configuration. Full explanations: config/README.md (or the online docs https://cbib.github.io/SpaceBlocks/configuration/). -# Every key is validated against workflow/schemas/config.schema.yaml before the run starts. - -# ─── Run mode — which headblock builds the contract h5ad ───────────────────── +# Canonical Catalog/CI fixture (mode: decoupled). Paths are relative to the +# repository root; users should copy config/config.yaml.template over this file. mode: decoupled - -# ─── input/output directories ──────────────────────────────────────────────── -## BASE DIRECTORY ############################################################# -## -## You may modify 'base_dir' to point to your project results directory. -## Paths relative to base_dir can be left as is, -## or be overridden with absolute paths if desired. - -base_dir: SpaceBlocksProject - -## Paths that are not relative to base_dir and must be set explicitly below: -# spaceranger, probe_set, transcriptome, ingest_ref -############################################################################### - +base_dir: .test/test_output post_processing_outdir: "{base_dir}/output" logdir: "{base_dir}/logs" -spaceranger_processing_outdir: "{base_dir}/spaceranger_results" -# dir with region annotation GeoJSONs (optional, recommended) -geojson_path: "/path/to/geojson" -# only when using "mode: decoupled" — dir of pre-built contract h5ads -contract_dir: '' - -# ─── Reproducibility & reuse (docs → Configuration) ──────────────────────── +spaceranger_processing_outdir: "{base_dir}/sr" +geojson_path: "{base_dir}/geojson" +contract_dir: ".test/data" random_seed: 0 -precomputed_metadata_dir: "/path/to/metadata_visiumhd/" -# reuse clusters from precomputed_metadata_dir -use_precomputed_clusters: true - -# ─── Sample sheets ─────────────────────────────────────────────────────────── -core_samples: config/core_samples.tsv -# only when using "mode: visiumhd" — fastq/slide/area per sample -samples: config/visiumhd_samples.csv - -# ─── Visium HD head (mode: visiumhd) ─────────────────────────────────────── -spaceranger: /path/to/spaceranger-4.0.1/spaceranger # external install; path to the executable -probe_set: /path/to/Visium_Human_Transcriptome_Probe_Set_v2.1.0_GRCh38-2024-A.csv -transcriptome: /path/to/cellranger/refdata-gex-GRCh38-2024-A - -# ─── Xenium 5K head (mode: xenium5k) ─────────────────────────────────────── +use_precomputed_clusters: false +precomputed_metadata_dir: '' +core_samples: .test/config/core_samples.tsv xenium5k: - # xenium_dir is the directory containing the Xenium 5K samples. - # Make sure to keep the {sample} placeholder in the path. - xenium_dir: "/path/to/xenium/{sample}" + xenium_dir: "{base_dir}/xenium/{sample}" qupath_pyramid_level: 3 hires_pyramid_level: 3 pixel_size_um: 0.2125 - -# ─── Contract — Head → Core hand-off ───────────────────────────────────────── contract: sample_key: sample spatial_key: spatial @@ -59,58 +24,22 @@ contract: mito_prefix: - MT- - mt- - -# ─── QC & preprocessing ────────────────────────────────────────────────────── -# qc thresholds to screen in qc_sweep_all -qc_sweep: - # use ingest_ref - ingest_enabled: true - ingest_key: cell_type_ingest - cell_id_key: cell_id - dpi: 300 - # thresholds to screen - thresholds: - min_genes: - - 50 - - 100 - - 200 - min_counts: - - 1 - - 10 - - 50 - max_counts: - - 3000 - - 5000 - max_pct_mt: - - 10 - - 15 - - 20 - -# optional: per-sample QC thresholds TSV (example at config/per_sample_qc.tsv) per_sample_qc: '' -# default: global thresholds analysis: - # qc thresholds plot_dpi: 300 - # marker sizes for UMAP embeddings and spatial maps, respectively umap_point_size: 2 spatial_point_size: 20 min_counts: 1 min_cells: 3 min_genes: 100 - # umap n_neighbors: 10 n_pcs: 30 - # n marker genes; n DE genes in pseudobulk heatmaps de_n_genes: 25 - # leiden resolution_scan_min: 0.6 resolution_scan_max: 0.8 resolution_scan_step: 0.2 min_cells_per_type: 15 - # integrate sketch_fraction: 0.25 - # pseudobulk run_pseudobulk_de: true run_leiden_analysis: true min_cells_per_pseudobulk: 10 @@ -118,9 +47,6 @@ analysis: min_counts_per_pseudobulk: 1000 padj_threshold: 0.05 lfc_threshold: 0.25 - # Each named analysis defines one aggregation and one DE model. `cell_type` is - # resolved to the annotation selected by annotation_types. Multiple group_by - # columns form one combined comparison group; interactions are not fitted. pseudobulk: analyses: - name: regions_all_cells @@ -131,17 +57,14 @@ analysis: exclude_levels: region_annotation: - Unlabeled - - Bubble contrasts: - name: tumor_vs_healthy numerator: region_annotation: Tumor area denominator: region_annotation: Healthy area - # Omnibus test across every sufficiently replicated, non-excluded region. lrt: enabled: true - - name: regions_by_celltype aggregation: by_celltype_region group_by: @@ -150,7 +73,6 @@ analysis: exclude_levels: region_annotation: - Unlabeled - - Bubble cell_type: - Unannotated contrasts: @@ -161,188 +83,94 @@ analysis: region_annotation: Healthy area lrt: enabled: true - - # Preferred region order for QC-sweep, subcluster, and region-only pseudobulk split - # heatmaps, first-to-last (typically left-to-right). This does not filter pseudobulk - # levels; use each analysis' exclude_levels for filtering. region_levels: - Healthy area - - Healthy over tumor - - Dysplastic area - - AK area - - AK over tumor - - In situ area + - Premalignant area - Tumor area - - Invasive area - # Palette used when region_annotation is the single comparison variable. region_colors: Healthy area: '#FDE725FF' - Healthy over tumor: '#9FDA3AFF' - Dysplastic area: '#4AC16DFF' - AK area: '#1FA187FF' - AK over tumor: '#277F8EFF' - In situ area: '#365C8DFF' + Premalignant area: '#1FA187FF' Tumor area: '#46337EFF' - Invasive area: '#440154FF' - -# ─── Annotation ────────────────────────────────────────────────────────────── -# (optional, recommended) predefined cell type markers, screened during leiden to ease annotation -snakemake_cell_markers: config/snakemake_cell_markers.tsv -# (default) cluster → cell-type TSV (fill template from generate_annotation_template) -cluster_annotations: config/cluster_annotations.tsv -# (optional) external labels from metadata TSVs, or from contract h5ad obs in decoupled mode +qc_sweep: + ingest_enabled: false + ingest_key: cell_type_ingest + cell_id_key: cell_id + dpi: 300 + thresholds: + min_genes: + - 50 + - 100 + - 200 + min_counts: + - 1 + - 10 + - 50 + max_counts: + - 3000 + - 5000 + max_pct_mt: + - 10 + - 15 + - 20 +cluster_annotations: .test/config/cluster_annotations.tsv +snakemake_cell_markers: .test/config/snakemake_cell_markers.tsv external_annotation: enabled: false - column: manual_celltype # arbitrary source name; normalized to cell_type_external + column: manual_celltype keep_unannotated: true -ingest_ref: "/path/to/ingest_ref_data/bcc_and_normal_Garnier_PNAS_2024.h5ad" # optional annotated scRNA-seq reference (ingest_ref) -ingest_ref_label_key: 04_celltypes +ingest_ref: '' +ingest_ref_label_key: cell_type_annotation_column integration: integrate_key: sample -# (optional) include annotations for downstream plots from core_samples.tsv extra_annotations: columns: - patient - batch - -# ─── Spatial niches (optional) ───────────────────────────────────────────── spatial_niches: enabled: true - # use niche_dir precomputed .tsv - use_precomputed: true - niche_dir: "/path/to/niches_visiumhd/" + use_precomputed: false + niche_dir: '' lambda: 0.8 num_neighbours: 18 cluster_n_neighbours: 50 n_iterations: -1 max_m: 1 nbr_weight_decay: scaled_gaussian - # (recommended) use only HVGs for niche clustering use_hvg: true n_top_genes: 2000 niche_resolution: 0.1 - # (optional) BANKSY refine niches refine: false refine_num_neigh: 6 refine_iterations: 1 refine_auto: false compute_umap: false - -# ─── Subcompartments (optional) ──────────────────────────────────────────── subcompartments: - # subcompartment name - Fibroblasts: - # exact-string match of cell type names to pool together - strings: - - Fibroblasts - - Fibroblasts_Macro - - Fibroblasts_Endothelial - - Fibroblasts_Keratinocytes - - Fibroblasts_Plasma - # resolution range to screen in leiden clustering - resolution_min: 0.6 - resolution_max: 1.2 - resolution_step: 0.2 Keratinocytes: strings: - Keratinocytes - - Keratinocytes_Mal - - Keratinocytes_Basal - - Keratinocytes_Suprabasal - - Fibroblasts_Keratinocytes - resolution_min: 0.6 - resolution_max: 1.2 - resolution_step: 0.2 - MonoMacro_Tcells: - strings: - - Mono_Macro - - Macro_Tcells - - Macro_Plasma - - Tcells - - Fibroblasts_Macro - resolution_min: 0.6 - resolution_max: 1.2 - resolution_step: 0.2 - Plasma_Bcells: - strings: - - Plasma_cells - - Macro_Plasma - - Fibroblasts_Plasma resolution_min: 0.6 - resolution_max: 1.2 + resolution_max: 0.8 resolution_step: 0.2 - -# ─── Gene / signature exploration ──────────────────────────────────────────── gene_exploration: - # tsv where col name is gene set name, values are genes (one or more) - queries: config/gene_queries.tsv + queries: .test/config/gene_queries.tsv aucell_max_rank_fraction: 0.05 niche_column: spatial_niche dpi: 300 - -# ─── Colour palettes (docs → Configuration) ──────────────────────────────── -# region color palettes is found under analysis: region_colors sample_colors: - # extra column in core_samples.tsv patient: - # levels and HEX code; unfound levels fall back to grey Patient 1: '#000000' Patient 2: '#E69F00' - Patient 3: '#56B4E9' - Patient 4: '#009E73' - Patient 5: '#F0E442' - Patient 6: '#CC79A7' - Patient 7: '#0072B2' - Patient 8: '#D55E00' batch: Batch 1: '#8E44AD' - Batch 2: '#C0392B' - Batch 3: '#7F7F7F' annotation_colors: - # levels in cluster_annotations.tsv cell_type_tsv: Keratinocytes: '#d62728' - Keratinocytes_Basal: '#8c564b' - Keratinocytes_Suprabasal: '#a0522d' - Keratinocytes_Mal: '#000000' Tcells: '#1f77b4' - Plasma_cells: '#9467bd' - Mono_Macro: '#08519c' - Macro_Tcells: '#756bb1' - Macro_Plasma: '#9e9ac8' Fibroblasts: '#2ca02c' - Fibroblasts_Macro: '#bcbd22' - Fibroblasts_Endothelial: '#17becf' - Fibroblasts_Keratinocytes: '#ffbb78' - Fibroblasts_Plasma: '#fdae6b' - Endothelial: '#98df8a' - Adipocytes: '#dbdb8d' - Gland_cells: '#ff7f0e' - Smooth_muscle: '#d9d9d9' - Skeletal_muscle: '#b9b9c9' - # levels when external_annotation: enabled: true cell_type_external: Keratinocytes: '#d62728' - Keratinocytes_Basal: '#8c564b' - Keratinocytes_Suprabasal: '#a0522d' - Keratinocytes_Mal: '#000000' Tcells: '#1f77b4' - Plasma_cells: '#9467bd' - Mono_Macro: '#08519c' - Macro_Tcells: '#756bb1' - Macro_Plasma: '#9e9ac8' Fibroblasts: '#2ca02c' - Fibroblasts_Macro: '#bcbd22' - Fibroblasts_Endothelial: '#17becf' - Fibroblasts_Keratinocytes: '#ffbb78' - Fibroblasts_Plasma: '#fdae6b' - Endothelial: '#98df8a' - Adipocytes: '#dbdb8d' - Gland_cells: '#ff7f0e' - Smooth_muscle: '#d9d9d9' - Skeletal_muscle: '#b9b9c9' - -# ─── Per-rule resources — mem_mb scales with the retry attempt ─────────────── resources: spaceranger_count_vhd: mem_mb: 65536 @@ -429,7 +257,7 @@ resources: runtime: 240 threads: 4 explore_genes_sample: - mem_mb: 16192 + mem_mb: 32768 runtime: 360 threads: 4 default: diff --git a/config/config.yaml.template b/config/config.yaml.template new file mode 100644 index 0000000..88cef5d --- /dev/null +++ b/config/config.yaml.template @@ -0,0 +1,362 @@ +# SpaceBlocks configuration. Full explanations: config/README.md (or the online docs https://cbib.github.io/SpaceBlocks/configuration/). +# Every key is validated against workflow/schemas/config.schema.yaml before the run starts. + +# ─── Run mode — which headblock builds the contract h5ad ───────────────────── +mode: decoupled + +# ─── input/output directories ──────────────────────────────────────────────── +## BASE DIRECTORY ############################################################# +## +## You may modify 'base_dir' to point to your project results directory. +## This is useful to keep results in a fast filesystem separate from the source code. +## Paths relative to base_dir can be left as is, +## or be overridden with absolute paths if desired. + +base_dir: /path/to/SpaceBlocks/results + +## Paths that are not relative to base_dir and must be set explicitly below: +# spaceranger, probe_set, transcriptome, ingest_ref +############################################################################### + +post_processing_outdir: "{base_dir}/output" +logdir: "{base_dir}/logs" +spaceranger_processing_outdir: "{base_dir}/spaceranger_results" +# dir with region annotation GeoJSONs (optional, recommended) +geojson_path: "/path/to/geojson" +# only when using "mode: decoupled" — dir of pre-built contract h5ads +contract_dir: '/path/to/preformatted_contract_h5ads' + +# ─── Reproducibility & reuse (docs → Configuration) ──────────────────────── +random_seed: 0 +precomputed_metadata_dir: '/path/to/precomputed_metadata' +# reuse clusters from precomputed_metadata_dir +use_precomputed_clusters: false + +# ─── Sample sheets ─────────────────────────────────────────────────────────── +core_samples: config/core_samples.tsv +# only when using "mode: visiumhd" — fastq/slide/area per sample +samples: config/visiumhd_samples.csv + +# ─── Visium HD head (mode: visiumhd) ─────────────────────────────────────── +spaceranger: /path/to/spaceranger-4.0.1/spaceranger # external install; path to the executable +probe_set: /path/to/Visium_Human_Transcriptome_Probe_Set_v2.1.0_GRCh38-2024-A.csv +transcriptome: /path/to/cellranger/refdata-gex-GRCh38-2024-A + +# ─── Xenium 5K head (mode: xenium5k) ─────────────────────────────────────── +xenium5k: + # xenium_dir is the directory containing the Xenium 5K samples. + # Make sure to keep the {sample} placeholder in the path. + xenium_dir: "/path/to/xenium/{sample}" + qupath_pyramid_level: 3 + hires_pyramid_level: 3 + pixel_size_um: 0.2125 + +# ─── Contract — Head → Core hand-off ───────────────────────────────────────── +contract: + sample_key: sample + spatial_key: spatial + require_region: false + require_raw_counts: true + mito_prefix: + - MT- + - mt- + +# ─── QC & preprocessing ────────────────────────────────────────────────────── +# qc thresholds to screen in qc_sweep_all +qc_sweep: + # use ingest_ref + ingest_enabled: false + ingest_key: cell_type_ingest + cell_id_key: cell_id + dpi: 300 + # thresholds to screen + thresholds: + min_genes: + - 50 + - 100 + - 200 + min_counts: + - 1 + - 10 + - 50 + max_counts: + - 3000 + - 5000 + max_pct_mt: + - 10 + - 15 + - 20 + +# optional: per-sample QC thresholds TSV (example at config/per_sample_qc.tsv) +per_sample_qc: '' +# default: global thresholds +analysis: + # qc thresholds + plot_dpi: 300 + # marker sizes for UMAP embeddings and spatial maps, respectively + umap_point_size: 2 + spatial_point_size: 20 + min_counts: 1 + min_cells: 3 + min_genes: 100 + # umap + n_neighbors: 10 + n_pcs: 30 + # n marker genes; n DE genes in pseudobulk heatmaps + de_n_genes: 25 + # leiden + resolution_scan_min: 0.6 + resolution_scan_max: 0.8 + resolution_scan_step: 0.2 + min_cells_per_type: 15 + # integrate + sketch_fraction: 0.25 + # pseudobulk + run_pseudobulk_de: true + run_leiden_analysis: true + min_cells_per_pseudobulk: 10 + min_replicates: 3 + min_counts_per_pseudobulk: 1000 + padj_threshold: 0.05 + lfc_threshold: 0.25 + # Each named analysis defines one aggregation and one DE model. `cell_type` is + # resolved to the annotation selected by annotation_types. Multiple group_by + # columns form one combined comparison group; interactions are not fitted. + pseudobulk: + analyses: + - name: regions_all_cells + aggregation: by_region + group_by: + - region_annotation + paired_by: sample + exclude_levels: + region_annotation: + - Unlabeled + contrasts: + - name: tumor_vs_healthy + numerator: + region_annotation: Tumor area + denominator: + region_annotation: Healthy area + # Omnibus test across every sufficiently replicated, non-excluded region. + # Note: LRT only runs when more than >= 3 levels are present after filtering by exclude_levels and min_replicates. + lrt: + enabled: true + + - name: regions_by_celltype + aggregation: by_celltype_region + group_by: + - region_annotation + paired_by: sample + exclude_levels: + region_annotation: + - Unlabeled + cell_type: + - Unannotated + contrasts: + - name: tumor_vs_healthy + numerator: + region_annotation: Tumor area + denominator: + region_annotation: Healthy area + # Note: LRT only runs when more than >= 3 levels are present after filtering by exclude_levels and min_replicates. + lrt: + enabled: true + + # Preferred region order for QC-sweep, subcluster, and region-only pseudobulk split + # heatmaps, first-to-last (typically left-to-right). This does not filter pseudobulk + # levels; use each analysis' exclude_levels for filtering. + region_levels: + - Healthy area + - Premalignant area + - Tumor area + + # Palette used when region_annotation is the single comparison variable. + region_colors: + Healthy area: '#FDE725FF' + Premalignant area: '#1FA187FF' + Tumor area: '#46337EFF' + + +# ─── Annotation ────────────────────────────────────────────────────────────── +# (optional, recommended) predefined cell type markers, screened during leiden to ease annotation +snakemake_cell_markers: config/snakemake_cell_markers.tsv +# (default) cluster → cell-type TSV (fill template from generate_annotation_template) +cluster_annotations: config/cluster_annotations.tsv +# (optional) external labels from metadata TSVs, or from contract h5ad obs in decoupled mode +external_annotation: + enabled: false + column: manual_celltype # arbitrary source name; normalized to cell_type_external + keep_unannotated: true +ingest_ref: '/path/to/ingest_ref_data/ref.h5ad' # optional annotated scRNA-seq reference (ingest_ref) +ingest_ref_label_key: cell_type_annotation_column +integration: + integrate_key: sample +# (optional) include annotations for downstream plots from core_samples.tsv +extra_annotations: + columns: + - condition + - batch + +# ─── Spatial niches (optional) ───────────────────────────────────────────── +spatial_niches: + enabled: true + # use niche_dir precomputed .tsv + use_precomputed: false + niche_dir: '/path/to/precomputed_niches' + lambda: 0.8 + num_neighbours: 18 + cluster_n_neighbours: 50 + n_iterations: -1 + max_m: 1 + nbr_weight_decay: scaled_gaussian + # (recommended) use only HVGs for niche clustering + use_hvg: true + n_top_genes: 2000 + niche_resolution: 0.1 + # (optional) BANKSY refine niches + refine: false + refine_num_neigh: 6 + refine_iterations: 1 + refine_auto: false + compute_umap: false + +# ─── Subcompartments (optional) ──────────────────────────────────────────── +subcompartments: + # subcompartment name (example) + Fibroblasts: + # exact-string match of cell type names to pool together + strings: + - Fibroblasts + - Fibroblasts_COLA1 + - Fibroblasts_COLA2 + # resolution range to screen in leiden clustering + resolution_min: 0.6 + resolution_max: 1.2 + resolution_step: 0.2 + +# ─── Gene / signature exploration ──────────────────────────────────────────── +gene_exploration: + # tsv where col name is gene set name, values are genes (one or more) + queries: config/gene_queries.tsv + aucell_max_rank_fraction: 0.05 + niche_column: spatial_niche + dpi: 300 + +# ─── Colour palettes (docs → Configuration) ──────────────────────────────── +# region color palettes is found under analysis: region_colors +sample_colors: + # extra column in core_samples.tsv + condition: + # levels and HEX code; unfound levels fall back to grey + Control: '#0072B2' + Treatment: '#D55E00' + batch: + Batch 1: '#8E44AD' + Batch 2: '#C0392B' +annotation_colors: + # levels in cluster_annotations.tsv + cell_type_tsv: + Keratinocytes: '#d62728' + Tcells: '#1f77b4' + Fibroblasts: '#2ca02c' + # levels when external_annotation: enabled: true + cell_type_external: + Keratinocytes: '#d62728' + Tcells: '#1f77b4' + Fibroblasts: '#2ca02c' + +# ─── Per-rule resources — mem_mb scales with the retry attempt ─────────────── +resources: + spaceranger_count_vhd: + mem_mb: 65536 + runtime: 1200 + threads: 16 + generate_qupath_vhd: + mem_mb: 4096 + runtime: 30 + threads: 1 + prepare_input_vhd: + mem_mb: 32768 + runtime: 240 + threads: 4 + convert_zarr_x5k: + mem_mb: 65536 + runtime: 240 + threads: 4 + generate_qupath_x5k: + mem_mb: 32768 + runtime: 60 + threads: 2 + prepare_input_x5k: + mem_mb: 49152 + runtime: 120 + threads: 4 + validate_input: + mem_mb: 32768 + runtime: 30 + threads: 2 + process_geojson: + mem_mb: 4096 + runtime: 30 + threads: 2 + qc_sweep: + mem_mb: 49152 + runtime: 180 + threads: 4 + preprocess_umap: + mem_mb: 32768 + runtime: 240 + threads: 8 + ingest_ref: + mem_mb: 32768 + runtime: 240 + threads: 8 + leiden_analysis: + mem_mb: 16384 + runtime: 120 + threads: 4 + pseudobulk_aggregate: + mem_mb: 124768 + runtime: 60 + threads: 4 + pseudobulk_de: + mem_mb: 24384 + runtime: 120 + threads: 4 + neighbourhood_analysis: + mem_mb: 32768 + runtime: 120 + threads: 8 + annotate_cells: + mem_mb: 32768 + runtime: 60 + threads: 4 + integrate_samples: + mem_mb: 65536 + runtime: 360 + threads: 8 + spatial_niches: + mem_mb: 124768 + runtime: 720 + threads: 16 + subcluster: + mem_mb: 120768 + runtime: 1440 + threads: 16 + sample_report: + mem_mb: 32768 + runtime: 60 + threads: 4 + explore_genes_integrated: + mem_mb: 65536 + runtime: 240 + threads: 4 + explore_genes_sample: + mem_mb: 16192 + runtime: 360 + threads: 4 + default: + mem_mb: 8192 + runtime: 60 + threads: 1 diff --git a/config/core_samples.tsv b/config/core_samples.tsv index b72769c..32689bf 100644 --- a/config/core_samples.tsv +++ b/config/core_samples.tsv @@ -1,11 +1,9 @@ -# CORE sample sheet — technology-agnostic anchor (see config.yaml: core_samples). -# Required column: sample. Optional columns (here: patient, type) are stamped into -# obs by preprocess_umap and surfaced downstream via config.extra_annotations. -# `sample` values match config/samples.csv sample_id (visiumhd head run). -# IMPORTANT: patients 1 3 5 and 7 are shared with the HTGEdgeSeq study and match -# their patient id there; the remaining patients are not shared between studies. -# Lines starting with '#' are ignored (no commas inside so no quoting). -sample patient batch -sample1 Patient 1 Batch 1 -sample2 Patient 2 Batch 2 -sample3 Patient 1 Batch 1 +# CORE sample sheet — technology-agnostic anchor (see config.yaml: core_samples). +# Required column: sample. Optional columns (here: condition, batch) are stamped into +# obs by preprocess_umap and surfaced downstream via config.extra_annotations. +# `sample` values match config/samples.csv sample_id (visiumhd head run). +# Lines starting with '#' are ignored (no commas inside so no quoting). +sample condition batch +sample1 Control Batch 1 +sample2 Treatment Batch 1 +sample3 Treatment Batch 2 diff --git a/demos/atera/atera_config.yaml b/demos/atera/atera_config.yaml index ae233eb..77d2a4f 100644 --- a/demos/atera/atera_config.yaml +++ b/demos/atera/atera_config.yaml @@ -53,7 +53,6 @@ atera: # The preview ships as ONE sample, so atera_dir carries no {sample} placeholder. # For a real multi-sample run use a pattern, e.g. /path/to/atera/{sample}/outs. atera_dir: /scratch/CBiB/soterino/atera/outs - zarr_dir: '' # '' -> spaceranger_processing_outdir # Level 2 = 0.85 um/px. Both QuPath images (fluorescence composite and registered # H&E) are exported on this morphology grid, so they overlay exactly and share one # pixel->micron scalar. Level 3 halves the resolution and the file size. diff --git a/demos/visiumhd/visiumhd_config.yaml b/demos/visiumhd/visiumhd_config.yaml index 5e2d831..9cafdf2 100644 --- a/demos/visiumhd/visiumhd_config.yaml +++ b/demos/visiumhd/visiumhd_config.yaml @@ -13,7 +13,6 @@ precomputed_metadata_dir: 'demos/visiumhd/metadata' core_samples: demos/visiumhd/contracts/core_samples.tsv xenium5k: xenium_dir: /scratch/CBiB/soterino/xenium/{sample} - zarr_dir: '' qupath_pyramid_level: 3 hires_pyramid_level: 3 pixel_size_um: 0.2125 diff --git a/demos/xenium5k/xenium5k_config.yaml b/demos/xenium5k/xenium5k_config.yaml index 0912844..fac8bce 100644 --- a/demos/xenium5k/xenium5k_config.yaml +++ b/demos/xenium5k/xenium5k_config.yaml @@ -13,7 +13,6 @@ precomputed_metadata_dir: 'demos/xenium5k/metadata' core_samples: demos/xenium5k/contracts/core_samples.tsv xenium5k: xenium_dir: /scratch/CBiB/soterino/demos/xenium5k/results/{sample} - zarr_dir: '' qupath_pyramid_level: 3 hires_pyramid_level: 3 pixel_size_um: 0.2125 diff --git a/docs/getting-started.md b/docs/getting-started.md index c4119c5..55673bb 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -20,7 +20,11 @@ We recommend to always run snakemake with `--sdm conda`, so each rule is execute ## 2. Configure -In the `config/config.yaml` pick a `mode` and fill two files: +Create your run configuration, then pick a `mode` and fill the sample sheet: + +```bash +cp config/config.yaml.template config/config.yaml +``` 1. **`config/core_samples.tsv`** — one row per sample; extra columns (e.g. `patient`, `condition`) become design metadata, and can be used for downstream analyses and visualization. 2. **`config/config.yaml`** — set `mode`, the input and output paths, and (for Visium HD) the head files. See [Configuration](configuration.md) for full details. diff --git a/profiles/default/config.yaml b/profiles/default/config.yaml index 4a84d53..b1fbdea 100644 --- a/profiles/default/config.yaml +++ b/profiles/default/config.yaml @@ -3,15 +3,6 @@ # ═══════════════════════════════════════════════════════════════════════════ # Invoke with: snakemake --profile profiles/default # Set as default for a shell session with: export SNAKEMAKE_PROFILE=profiles/default -# IMPORTANT -# "directory: .test" is necessary for Snakemake catalog's linting -# Switch from "directory: .test" to "directory: ." when running from the repository root -# -# IMPORTANT -# "directory: .test" is necessary for Snakemake catalog's linting -# Switch from "directory: .test" to "directory: ." when running from the repository root, -# or define a custom directory "directory: /path/to/your/working/directory" to run from anywhere. -directory: .test executor: local jobs: 4 diff --git a/profiles/slurm/config.yaml b/profiles/slurm/config.yaml index e161e44..bf205e8 100644 --- a/profiles/slurm/config.yaml +++ b/profiles/slurm/config.yaml @@ -4,11 +4,6 @@ # Invoke with: snakemake --profile profiles/slurm # Set as default for a shell session with: export SNAKEMAKE_PROFILE=profiles/slurm # -# IMPORTANT -# "directory: .test" is necessary for Snakemake catalog's linting -# Switch from "directory: .test" to "directory: ." when running from the repository root, -# or define a custom directory "directory: /path/to/your/working/directory" to run from anywhere. -directory: .test executor: slurm jobs: 30 diff --git a/workflow/Snakefile b/workflow/Snakefile index 1b2627f..3f91156 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -59,7 +59,7 @@ XENIUM_ZARR_DIR = OUTDIR_SR # Atera head specifics (ALPHA — the platform ships H2 2026 and its output format is # still a preview; see docs/rules.md). ATERA = config.get("atera", {}) or {} -ATERA_ZARR_DIR = ATERA.get("zarr_dir", "") or OUTDIR_SR +ATERA_ZARR_DIR = OUTDIR_SR # The registered H&E is OPTIONAL and ships separately from the outs/ bundle. Both the # image and its alignment matrix are needed for the transform to be defined, so the # H&E QuPath rule is included only when both are configured. diff --git a/workflow/schemas/config.schema.yaml b/workflow/schemas/config.schema.yaml index fced940..32f9701 100644 --- a/workflow/schemas/config.schema.yaml +++ b/workflow/schemas/config.schema.yaml @@ -37,9 +37,6 @@ properties: xenium_dir: type: string description: "{sample} pattern to each Xenium output bundle." - zarr_dir: - type: string - description: Where {sample}.zarr stores are written ("" → spaceranger_processing_outdir). qupath_pyramid_level: type: integer minimum: 0 @@ -66,9 +63,6 @@ properties: atera_dir: type: string description: "{sample} pattern to each Atera outs/ bundle." - zarr_dir: - type: string - description: Where {sample}.zarr stores are written ("" -> spaceranger_processing_outdir). qupath_pyramid_level: type: integer minimum: 0 @@ -166,14 +160,14 @@ properties: mode: type: string enum: ["visiumhd", "xenium5k", "atera", "merscope", "decoupled"] - default: "visiumhd" + default: "decoupled" description: >- A HEAD technology name (that head builds the contract h5ads in this run) or "decoupled" (the core consumes pre-existing contracts from contract_dir; head - rules skipped; validation aborts if any core_samples sample is missing its - contract). Extend the enum as new heads are added (e.g. xenium5k). "atera" is - ALPHA support: the platform ships in H2 2026 and the public data is a preview - whose output format 10x state will change at commercial release. + rules are skipped; Snakemake reports any missing configured contracts while + building the DAG). Extend the enum as new heads are added (e.g. xenium5k). + "atera" is ALPHA support: the platform ships in H2 2026 and the public data is + a preview whose output format 10x state will change at commercial release. core_samples: type: string