diff --git a/scripts/create_resources/combine/process_datasets_bruker_nebius.sh b/scripts/create_resources/combine/process_datasets_bruker_nebius.sh index dbc103819..a55151794 100644 --- a/scripts/create_resources/combine/process_datasets_bruker_nebius.sh +++ b/scripts/create_resources/combine/process_datasets_bruker_nebius.sh @@ -12,7 +12,11 @@ cd "$REPO_ROOT" set -e -input_dir="s3://openproblems-data/resources/datasets" +# Read raw inputs from the scratch PVC (where the bruker spatial scripts publish via +# publish_dir='/scratch/task_ist_preprocessing/raw'), not S3. Both input_sp (bruker_cosmx/…) +# and input_sc (scrnaseq refs) resolve under here; the raw/ layout mirrors the S3 subpaths. +input_dir="/scratch/task_ist_preprocessing/raw" +#input_dir="s3://openproblems-data/resources/datasets" #publish_dir="s3://openproblems-data/resources/task_ist_preprocessing/datasets" publish_dir='/scratch/task_ist_preprocessing/datasets' @@ -41,17 +45,6 @@ param_list: dataset_description: "Bruker CosMx Human Liver + 2022 Andrews scRNAseq" dataset_organism: "homo_sapiens" - - id: "bruker_human_liver_cancer_cosmx_combined" - input_sp: "$input_dir/bruker_cosmx/bruker_human_liver_cancer_cosmx/dataset.zarr" - input_sc: "$input_dir/scrnaseq_for_ist/2022Lu_human_liver_cancer_sc/dataset.h5ad" - dataset_id: "bruker_human_liver_cancer_cosmx_combined" - dataset_name: "Human liver cancer combined Bruker CosMx 2022 Lu scRNAseq" - dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/human-liver-rna-ffpe-dataset/" - dataset_reference: "https://doi.org/10.1038/s41467-022-32283-3" - dataset_summary: "Bruker CosMx Human Liver Cancer + 2022 Lu scRNAseq" - dataset_description: "Bruker CosMx Human Liver Cancer + 2022 Lu scRNAseq" - dataset_organism: "homo_sapiens" - - id: "bruker_human_lung_cancer_cosmx_combined/lung5_rep1" input_sp: "$input_dir/bruker_cosmx/bruker_human_lung_cancer_cosmx/lung5_rep1/dataset.zarr" input_sc: "$input_dir/zuani_human_nsclc_sc/2024Zuani_human_nsclc_sc/dataset.h5ad" diff --git a/src/base/setup_spatialdata_partial.yaml b/src/base/setup_spatialdata_partial.yaml index 9c34899a3..9155a7ec8 100644 --- a/src/base/setup_spatialdata_partial.yaml +++ b/src/base/setup_spatialdata_partial.yaml @@ -1,6 +1,13 @@ setup: - type: python - # Pinned exactly: an unpinned >=0.7.3 silently pulled 0.8.0 on a rebuild, whose - # bounding_box/filter_table regressions transposed/emptied transcripts in the crop. - # process_dataset's transform-agnostic crop is validated against 0.8.0, so pin here. - pypi: ["spatialdata==0.8.0", "anndata>=0.12.0,<0.13", "zarr>=3.0.0"] + # Pin exactly per python version: spatialdata 0.8.0 requires python>=3.12, but this + # partial is shared by components on older python (e.g. clustermap) that can only take + # 0.7.3. Pinning stops the unpinned >=0.7.3 from silently drifting again (that 0.7.3->0.8.0 + # drift is what transposed/emptied transcripts in process_dataset's crop). py>=3.12 + # components (process_dataset, vizgen loader) get 0.8.0, where the transform-agnostic crop + # is validated; older-python components stay on 0.7.3 (what they already resolved to). + pypi: + - "spatialdata==0.8.0; python_version>='3.12'" + - "spatialdata>=0.7.3; python_version<'3.12'" + - "anndata>=0.12.0,<0.13" + - "zarr>=3.0.0" diff --git a/src/datasets/loaders/allen_brain_cell_atlas/config.vsh.yaml b/src/datasets/loaders/allen_brain_cell_atlas/config.vsh.yaml index e3c57636c..d5d71c2e3 100644 --- a/src/datasets/loaders/allen_brain_cell_atlas/config.vsh.yaml +++ b/src/datasets/loaders/allen_brain_cell_atlas/config.vsh.yaml @@ -100,8 +100,11 @@ engines: image: openproblems/base_python:1 setup: - type: python - packages: - - git+https://github.com/alleninstitute/abc_atlas_access + packages: + # Pin the tag: an unpinned git install drifted to an API where AbcProjectCache + # dropped list_data_files (split into list_expression_matrix_files etc.), breaking + # the loader. v.3.3.0 has list_expression_matrix_files + get_data_path (deprecated). + - git+https://github.com/alleninstitute/abc_atlas_access@v.3.3.0 - type: native runners: diff --git a/src/datasets/loaders/allen_brain_cell_atlas/script.py b/src/datasets/loaders/allen_brain_cell_atlas/script.py index 8a01bcf42..c6c9b1fc8 100644 --- a/src/datasets/loaders/allen_brain_cell_atlas/script.py +++ b/src/datasets/loaders/allen_brain_cell_atlas/script.py @@ -73,14 +73,14 @@ obs = obs.sample(n=par["sample_n_obs"]) -# From abc_cache.list_data_files("WMB-10Xv2") +# From abc_cache.list_expression_matrix_files("WMB-10Xv2") # TODO: potentially also load other chemistries (currently only 10Xv2) print("Downloading and reading expression matrices", flush=True) abca_data_subdir="WMB-10Xv2" abca_region_files = [ [file_name, region] - for file_name in abc_cache.list_data_files(abca_data_subdir) + for file_name in abc_cache.list_expression_matrix_files(abca_data_subdir) for region in REGIONS if re.match(f"WMB-10Xv2-{region}[\\-0-9]*/raw", file_name) ] @@ -116,6 +116,14 @@ adata = ad.concat(adatas, merge="first") del adatas +# Drop genes expressed in <3 cells. The ABCA panel carries ~32k genes, most unexpressed in the +# sampled brain cells; their ~zero log-mean collapses scanpy HVG's mean-quantile bins to +# duplicate edges ("Bin edges must be unique"). Filtering keeps HVG well-defined downstream. +print("Filtering genes (>=3 cells)", flush=True) +gene_ncells = np.asarray((adata.layers["counts"] > 0).sum(axis=0)).ravel() +adata = adata[:, gene_ncells >= 3].copy() +print(f"Kept {adata.n_vars} genes", flush=True) + print("Processing .obs") adata.obs = obs.loc[adata.obs.index] diff --git a/src/datasets/processors/subsample/config.vsh.yaml b/src/datasets/processors/subsample/config.vsh.yaml new file mode 100644 index 000000000..f9e7fac68 --- /dev/null +++ b/src/datasets/processors/subsample/config.vsh.yaml @@ -0,0 +1,90 @@ +name: subsample +namespace: datasets/processors +description: | + Subsample an h5ad file to create a smaller (test) dataset. + + Local copy of openproblems-bio/datasets `processors/subsample` carrying a + single-modality fix: the mod2 output write is guarded on `--input_mod2` + (whether a second modality was actually read) instead of `--output_mod2`. + Under the Nextflow runner the optional `--output_mod2` path is auto-populated, + so the upstream guard tried to write an `adata_output_mod2` that is only + defined for multimodal input, crashing single-modality runs (Wu, Zuani) with + `NameError: name 'adata_output_mod2' is not defined`. + + Retire this component and switch the run scripts back to the `datasets` + dependency once the upstream fix propagates to openproblems-bio/datasets@build/main. + +arguments: + - name: --input + type: file + required: true + direction: input + description: Input h5ad dataset (must contain a `counts` layer). + - name: --input_mod2 + type: file + required: false + direction: input + description: Optional second-modality h5ad input. + - name: --output + type: file + required: true + direction: output + description: Output subsampled h5ad. + example: output.h5ad + - name: --output_mod2 + type: file + required: false + direction: output + description: Optional second-modality h5ad output (only written when --input_mod2 is given). + example: output_mod2.h5ad + - name: --output_compression + type: string + choices: [gzip, lzf] + required: false + default: gzip + - name: --n_obs + type: integer + default: 500 + description: Maximum number of observations to keep. May end up fewer because empty cells/genes are removed. + - name: --n_vars + type: integer + default: 500 + description: Maximum number of variables to keep. May end up fewer because empty cells/genes are removed. + - name: --keep_features + type: string + multiple: true + required: false + description: A list of genes to keep. + - name: --keep_cell_type_categories + type: string + multiple: true + required: false + description: Cell type categories to be selected. + - name: --keep_batch_categories + type: string + multiple: true + required: false + description: Batch categories to be selected. + - name: --even + type: boolean_true + description: Subsample evenly from different batches. + - name: --seed + type: integer + required: false + description: A seed for the subsampling. + example: 123 + +resources: + - type: python_script + path: script.py + +engines: + - type: docker + image: openproblems/base_python:1 + - type: native + +runners: + - type: executable + - type: nextflow + directives: + label: [midtime, highmem, midcpu] diff --git a/src/datasets/processors/subsample/script.py b/src/datasets/processors/subsample/script.py new file mode 100644 index 000000000..d188550f3 --- /dev/null +++ b/src/datasets/processors/subsample/script.py @@ -0,0 +1,146 @@ +import scanpy as sc +import random +import numpy as np + +### VIASH START +par = { + "input": "resources_test/common/scicar_cell_lines/temp_mod1_full.h5ad", + "input_mod2": "resources_test/common/scicar_cell_lines/temp_mod2_full.h5ad", + "n_obs": 600, + "n_vars": 1500, + "keep_cell_type_categories": None, + "keep_batch_categories": None, + "keep_features": None, + "even": False, + "output": "subsample_mod1.h5ad", + "output_mod2": "subsample_mod2.h5ad", + "output_compression": "gzip", + "seed": 123 +} +### VIASH END + +if par["seed"]: + print(f">> Setting seed to {par['seed']}", flush=True) + random.seed(par["seed"]) + +print(">> Load data", flush=True) +adata_input = sc.read_h5ad(par["input"]) + +if par["input_mod2"] is not None: + adata_mod2 = sc.read_h5ad(par["input_mod2"]) + +# copy counts to .X because otherwise filter_genes and filter_cells won't work +adata_input.X = adata_input.layers["counts"] +if par["input_mod2"] is not None: + adata_mod2.X = adata_mod2.layers["counts"] + +print(">> Determining output shape", flush=True) +min_obs_list = [par["n_obs"], adata_input.shape[0]] +if par["input_mod2"] is not None: + min_obs_list.append(adata_mod2.shape[0]) +n_obs = min(min_obs_list) + +min_vars_list = [par["n_vars"], adata_input.shape[1]] +if par["input_mod2"] is not None: + min_vars_list.append(adata_mod2.shape[1]) +n_vars = min(min_vars_list) + +print(">> Subsampling the observations", flush=True) +obs_filt = np.ones(dtype=np.bool_, shape=adata_input.n_obs) + +# subset by cell_type +if par.get("keep_cell_type_categories"): + print(f">> Selecting cell_type_categories {par['keep_cell_type_categories']}") + obs_filt = obs_filt & adata_input.obs["cell_type"].isin(par["keep_cell_type_categories"]) + +# subset by batch +if par.get("keep_batch_categories"): + print(f">> Selecting cell_type_categories {par['keep_batch_categories']}") + obs_filt = obs_filt & adata_input.obs["batch"].isin(par["keep_batch_categories"]) + +# subsample evenly across batches or not +if par.get("even"): + obs_evenly = "batch" + choice_ix = np.where(obs_filt)[0] + choice_batch = adata_input[choice_ix].obs[obs_evenly] + names, counts = np.unique(choice_batch, return_counts=True) + probs = dict(zip(names, 1 / counts / len(names))) + + choice_probs = [ probs[batch] for batch in choice_batch ] + obs_index = np.random.choice(choice_ix, size=n_obs, replace=False, p=choice_probs) +else: + obs_index = np.random.choice(np.where(obs_filt)[0], n_obs, replace=False) + +# subsample obs +adata_output = adata_input[obs_index].copy() +if par["input_mod2"] is not None: + adata_output_mod2 = adata_mod2[obs_index].copy() + +# filter cells and genes +if par["input_mod2"] is not None: + n_cells = adata_output.X.sum(axis=1).A.flatten() + n_cells_mod2 = adata_output_mod2.X.sum(axis=1).A.flatten() + keep_cells = np.minimum(n_cells, n_cells_mod2) > 1 + adata_output = adata_output[keep_cells, :].copy() + adata_output_mod2 = adata_output_mod2[keep_cells, :].copy() + + sc.pp.filter_genes(adata_output, min_cells=1) + sc.pp.filter_genes(adata_output_mod2, min_cells=1) + +else: + # todo: this should not remove features in keep_features! + print(">> Remove empty observations and features", flush=True) + sc.pp.filter_genes(adata_output, min_cells=1) + sc.pp.filter_cells(adata_output, min_counts=2) + +print(">> Subsampling the features", flush=True) +if par.get("keep_features"): + initial_filt = adata_output.var_names.isin(par["keep_features"]) + initial_idx, *_ = initial_filt.nonzero() + remaining_idx, *_ = (~initial_filt).nonzero() + rest_idx = remaining_idx[np.random.choice(len(remaining_idx), n_vars - len(initial_idx), replace=False)] + var_ix = np.concatenate([initial_idx, rest_idx]) +else: + var_ix = np.random.choice(adata_output.shape[1], n_vars, replace=False) + if par["input_mod2"] is not None: + var_ix_mod2 = np.random.choice(adata_output_mod2.shape[1], n_vars, replace=False) + +# subsample vars +adata_output = adata_output[:, var_ix].copy() +if par["input_mod2"] is not None: + adata_output_mod2 = adata_output_mod2[:, var_ix_mod2].copy() + +# filter cells and genes +if par["input_mod2"] is not None: + n_cells = adata_output.X.sum(axis=1).A.flatten() + n_cells_mod2 = adata_output_mod2.X.sum(axis=1).A.flatten() + keep_cells = np.minimum(n_cells, n_cells_mod2) > 1 + adata_output = adata_output[keep_cells, :].copy() + adata_output_mod2 = adata_output_mod2[keep_cells, :].copy() + + sc.pp.filter_genes(adata_output, min_cells=1) + sc.pp.filter_genes(adata_output_mod2, min_cells=1) + + +else: + # todo: this should not remove features in keep_features! + print(">> Remove empty observations and features", flush=True) + sc.pp.filter_genes(adata_output, min_cells=1) + sc.pp.filter_cells(adata_output, min_counts=2) + +print(">> Update dataset_id", flush=True) +adata_output.uns["dataset_id"] = adata_output.uns["dataset_id"] + "_subsample" +if par["input_mod2"] is not None: + adata_output_mod2.uns["dataset_id"] = adata_output_mod2.uns["dataset_id"] + "_subsample" + +# remove previously copied .X +del adata_output.X +if par["input_mod2"] is not None: + del adata_output_mod2.X + +print(">> Writing data", flush=True) +adata_output.write_h5ad(par["output"], compression=par["output_compression"]) +# Guard on input_mod2 (not output_mod2): adata_output_mod2 only exists for multimodal +# input, but the Nextflow runner auto-populates the optional output_mod2 path. +if par["input_mod2"] is not None: + adata_output_mod2.write_h5ad(par["output_mod2"], compression=par["output_compression"]) diff --git a/src/methods_transcript_assignment/comseg/config.vsh.yaml b/src/methods_transcript_assignment/comseg/config.vsh.yaml index a92907f43..715df59ee 100644 --- a/src/methods_transcript_assignment/comseg/config.vsh.yaml +++ b/src/methods_transcript_assignment/comseg/config.vsh.yaml @@ -11,7 +11,10 @@ links: documentation: "https://github.com/openproblems-bio/task_ist_preprocessing" repository: "https://github.com/openproblems-bio/task_ist_preprocessing" references: - doi: "10.1038/s41592-020-01018-x" + # ComSeg: Defard et al., "A point cloud segmentation framework for image-based + # spatial transcriptomics", Commun Biol 7, 800 (2024). The previous DOI + # (10.1038/s41592-020-01018-x) was the Cellpose paper, not ComSeg. + doi: "10.1038/s42003-024-06480-3" arguments: diff --git a/src/methods_transcript_assignment/fastreseg/NOTES.md b/src/methods_transcript_assignment/fastreseg/NOTES.md index e57e325a8..cc64112d3 100644 --- a/src/methods_transcript_assignment/fastreseg/NOTES.md +++ b/src/methods_transcript_assignment/fastreseg/NOTES.md @@ -112,11 +112,24 @@ Why each non-obvious choice: ## Arguments -Only the standard API args (`--input_ist`, `--input_segmentation`, `--input_scrnaseq`, -`--sc_cell_type_key`, `--output`). Method-specific knobs (pixel/z-step size, distance cutoffs, -lrtest/svm cutoffs) are currently **hardcoded** in `script.R` and `um_per_pixel`/ -`sc_celltype_key` in `input.py` — not yet exposed as Viash arguments (the commented -`arguments:` block in the config is a placeholder). +Standard API args (`--input_ist`, `--input_segmentation`, `--input_scrnaseq`, +`--sc_cell_type_key`, `--output`) **plus four exposed FastReseg tuning knobs** (added for the +parameter sweep — see "Optimization / tuning" below): + +| Arg | Type | Default | Maps to (`fastReseg_full_pipeline`) | +|-----|------|---------|-------------------------------------| +| `--molecular_distance_cutoff` | double | 2.7 | `molecular_distance_cutoff` | +| `--flagCell_lrtest_cutoff` | double | 5 | `flagCell_lrtest_cutoff` | +| `--svmClass_score_cutoff` | double | -2 | `svmClass_score_cutoff` | +| `--cutoff_spatialMerge` | double | 0.5 | `cutoff_spatialMerge` | + +Each default equals FastReseg's TRUE package default (verified against the upstream function +signature), so the exposed-arg behaviour with defaults is byte-for-byte the old hardcoded +behaviour. **Still hardcoded** (not exposed): `pixel_size` (0.18) and `zstep_size` (0.8) in +`script.R` (Tier-0 dataset-physical constants); `groupTranscripts_method` ("dbscan") / +`spatialMergeCheck_method` ("leidenCut") (Tier-2/3 categoricals); the NULL auto-cutoffs +(`cellular_distance_cutoff`, `score_baseline`, `lower/higherCutoff_transNum`); and +`sc_celltype_key` ("cell_type") in `input.py`. ## Wiring @@ -137,3 +150,72 @@ lrtest/svm cutoffs) are currently **hardcoded** in `script.R` and `um_per_pixel` - **Validated**: `viash test` passes locally (amd64/emulated); all four runtime bugs and the downstream `basic_count_aggregation` compatibility were reproduced and fixed in a k8s pod on the `build_main` image. Not yet run through a full Nextflow benchmark end-to-end. + +## Optimization / tuning + +FastReseg corrects an image-based segmentation from the spatial profile of transcripts, so its +levers control **how aggressively transcript groups are formed, flagged as mis-segmented, split, +and re-merged**. Ranges below are grounded in the upstream `fastReseg_full_pipeline()` roxygen +docs (DOI 10.1038/s41598-025-08733-5). + +**Non-default audit (important):** every value that was hardcoded in `script.R` *exactly* +matched FastReseg's package default — `pixel_size=0.18`, `zstep_size=0.8`, +`molecular_distance_cutoff=2.7`, `flagCell_lrtest_cutoff=5`, `svmClass_score_cutoff=-2`, +`groupTranscripts_method="dbscan"`, `spatialMergeCheck_method="leidenCut"`, +`cutoff_spatialMerge=0.5`, and all NULL auto-cutoffs. So **nothing was a pre-tuned deviation**; +each exposed config default is set to that same package default and the sweep walks the knob +away from it in both directions. + +### Tiers + +- **Tier 0 (input / not swept):** `pixel_size` (0.18) and `zstep_size` (0.8) — dataset-physical + geometry that converts transcript pixel coords to microns; wrong values rescale *all* distance + cutoffs at once. Left hardcoded because they are properties of the data, not quality dials. + (Caveat worth revisiting: `input.py` feeds FastReseg coords already in the standardized global + space, so whether 0.18 is the right multiplier here is unverified — but that is a correctness + question, not a sweep axis.) +- **Tier 1 (swept — highest impact on output quality):** the four exposed knobs below. +- **Tier 2/3 (exposed candidates, NOT swept):** `groupTranscripts_method` + (`dbscan`↔`delaunay`) and `spatialMergeCheck_method` (`leidenCut`↔`geometryDiff`) are + categorical algorithm choices — worth a future 1-value-each flip, but left hardcoded to keep + this sweep focused on the continuous error-detection/correction dials. The NULL auto-cutoffs + (`cellular_distance_cutoff`, `score_baseline`, `lower/higherCutoff_transNum`) are + data-adaptively computed by FastReseg; pinning them to fixed values is a deeper study. + +### Swept knobs (Tier 1) — 11 variants (1 default + 10 sweep) + +| Arg | Default | Sweep | Rationale | +|-----|---------|-------|-----------| +| `--molecular_distance_cutoff` | 2.7 | `[1.5, 2.0, 3.5, 4.0]` | Sharpest lever: max molecule↔molecule µm distance for grouping transcripts into candidate cells. Smaller → tighter/fragmented groups; larger → looser groups that may merge neighbours. Widest sweep. | +| `--flagCell_lrtest_cutoff` | 5 | `[3, 8]` | `-log10 p` cutoff to flag mis-segmented cells. Lower → flags more cells (aggressive re-segmentation); higher → conservative. | +| `--svmClass_score_cutoff` | -2 | `[-3, -1]` | Transcript-score boundary between high/low SVM classes; straddles the default. | +| `--cutoff_spatialMerge` | 0.5 | `[0.3, 0.7]` | Fractional (0–1) spatial constraint for accepting a group-merge; lower stricter, higher more permissive. | + +Defaults omitted from each sweep list (the "default" variant already covers that point). Booleans +would get exactly one flip — none of these are boolean. + +### Arg-ordering contract (CRITICAL — this component is multi-script) + +The knobs are passed to `script.R` as **positional args appended AFTER the 6 file paths**, so the +order must be identical in three places or the run silently corrupts (a value lands on the wrong +parameter with no error): + +``` +config.vsh.yaml arguments: --molecular_distance_cutoff, --flagCell_lrtest_cutoff, --svmClass_score_cutoff, --cutoff_spatialMerge +orchestrator.sh Rscript: ... transcripts_out.csv $par_molecular_distance_cutoff $par_flagCell_lrtest_cutoff $par_svmClass_score_cutoff $par_cutoff_spatialMerge +script.R: args[7]=molecular_distance_cutoff args[8]=flagCell_lrtest_cutoff args[9]=svmClass_score_cutoff args[10]=cutoff_spatialMerge +``` + +`script.R` reads them via a `num_arg(i, default)` helper (`as.numeric`, with a fallback to the +FastReseg default if the arg is absent/empty, so an older orchestrator degrades to stock +behaviour instead of injecting `NA`). + +### NEEDS REBUILD (unlike the six sibling sweeps) + +These four args did **not** exist before this change — they were hardcoded literals inside the +`fastReseg_full_pipeline(...)` call. The `build/main` container run by +`run_test_fastreseg_nebius.sh` (`--revision build/main`) will **reject** them with "unknown +option" until the component is rebuilt. Before launching the sweep: commit + push the config / +orchestrator.sh / script.R edits, run `viash ns build`, rebuild the fastreseg container on ghcr +(`build_main` tag — see the `check-component` skill), then launch. Sweep files live at +`scripts/run_benchmark/param_sweep/fastreseg_params.yaml` (+ `run_test_fastreseg_nebius.sh`). diff --git a/src/methods_transcript_assignment/fastreseg/config.vsh.yaml b/src/methods_transcript_assignment/fastreseg/config.vsh.yaml index c1130a9fb..9542ea000 100644 --- a/src/methods_transcript_assignment/fastreseg/config.vsh.yaml +++ b/src/methods_transcript_assignment/fastreseg/config.vsh.yaml @@ -13,15 +13,45 @@ references: doi: "10.1038/s41598-025-08733-5" -#arguments: -# - name: --transcripts_key -# type: string -# default: "transcripts" -# description: "Key for transcripts in the points layer" -# - name: --coordinate_system -# type: string -# default: "global" -# description: "Coordinate system for the transcripts" +# Method-specific FastReseg tuning knobs. Each default == FastReseg's TRUE package +# default for fastReseg_full_pipeline() (verified against the upstream signature), i.e. +# the value that was previously hardcoded in script.R. These are Tier-1 error-detection / +# correction levers; see NOTES.md ("Optimization / tuning") for tiers and ranges. +# +# ARG-ORDERING CONTRACT: orchestrator.sh passes these to script.R as POSITIONAL args +# appended AFTER the 6 file-path args, in this exact order: +# args[7]=molecular_distance_cutoff, args[8]=flagCell_lrtest_cutoff, +# args[9]=svmClass_score_cutoff, args[10]=cutoff_spatialMerge. +# Keep config <-> orchestrator.sh <-> script.R in lockstep or the run silently corrupts. +arguments: + - name: --molecular_distance_cutoff + type: double + default: 2.7 + description: | + Maximum molecule-to-molecule distance (microns) within a connected transcript group + during neighborhood search. FastReseg default 2.7. Sharpest lever on how transcripts + are grouped into candidate cells: smaller => tighter / more fragmented groups, larger + => looser groups that may merge neighbouring cells. + - name: --flagCell_lrtest_cutoff + type: double + default: 5 + description: | + Cutoff on lrtest_nlog10P (-log10 p-value of the likelihood-ratio test) used to flag + putative mis-segmented cells with strong spatial dependency in their transcript-score + profile. FastReseg default 5. Lower => more cells flagged for re-segmentation (more + aggressive correction); higher => more conservative. + - name: --svmClass_score_cutoff + type: double + default: -2 + description: | + Transcript-score cutoff separating high- vs low-score transcript classes in the SVM + used during error detection/correction. FastReseg default -2. + - name: --cutoff_spatialMerge + type: double + default: 0.5 + description: | + Spatial-constraint cutoff (fraction in 0-1) for accepting a merge event between two + source transcript groups. FastReseg default 0.5 (a 50% cutoff). resources: diff --git a/src/methods_transcript_assignment/fastreseg/orchestrator.sh b/src/methods_transcript_assignment/fastreseg/orchestrator.sh index 34d1eff74..80ce99f52 100644 --- a/src/methods_transcript_assignment/fastreseg/orchestrator.sh +++ b/src/methods_transcript_assignment/fastreseg/orchestrator.sh @@ -56,13 +56,23 @@ head $par_intermediate_dir/cell_types.tsv # Step 2: RunFastReseg -##running the R script +## running the R script. +## ARG-ORDERING CONTRACT: script.R reads these POSITIONALLY. The first 6 are file paths +## (args[1..6]); the FastReseg tuning knobs are appended AFTER them in this EXACT order and +## must match script.R's args[7..10] and config.vsh.yaml's `arguments:` block: +## args[7]=molecular_distance_cutoff, args[8]=flagCell_lrtest_cutoff, +## args[9]=svmClass_score_cutoff, args[10]=cutoff_spatialMerge. +## A reordering here silently corrupts the run — do not touch one file without the others. Rscript "$meta_resources_dir/script.R" "$par_intermediate_dir/counts.tsv" \ "$par_intermediate_dir/transcripts.tsv" \ "$par_intermediate_dir/cell_types.tsv" \ "$par_intermediate_dir/cell_ids.csv" \ "$par_intermediate_dir/gene_names.csv" \ - "$par_intermediate_dir/transcripts_out.csv" + "$par_intermediate_dir/transcripts_out.csv" \ + "$par_molecular_distance_cutoff" \ + "$par_flagCell_lrtest_cutoff" \ + "$par_svmClass_score_cutoff" \ + "$par_cutoff_spatialMerge" ## python output python "$meta_resources_dir/output.py" \ diff --git a/src/methods_transcript_assignment/fastreseg/script.R b/src/methods_transcript_assignment/fastreseg/script.R index bc864d170..0df0a7f68 100644 --- a/src/methods_transcript_assignment/fastreseg/script.R +++ b/src/methods_transcript_assignment/fastreseg/script.R @@ -18,9 +18,27 @@ path_to_transcripts <- args[2] path_to_cell_annot <- args[3] path_to_cell_ids_out <- args[4] -path_to_gene_names_out <- args[5] +path_to_gene_names_out <- args[5] path_to_transcripts_out <- args[6] +# --- FastReseg tuning knobs (POSITIONAL, appended by orchestrator.sh AFTER the 6 file +# paths, in this FIXED order). This ordering is the contract between orchestrator.sh's +# Rscript call and this block AND config.vsh.yaml's `arguments:` — a mismatch silently +# corrupts the run, so keep all three in lockstep: +# args[7] = molecular_distance_cutoff (FastReseg default 2.7) +# args[8] = flagCell_lrtest_cutoff (FastReseg default 5) +# args[9] = svmClass_score_cutoff (FastReseg default -2) +# args[10] = cutoff_spatialMerge (FastReseg default 0.5) +# num_arg() falls back to the FastReseg package default if an arg is absent/empty, so an +# older orchestrator (or a missing value) degrades to stock behaviour instead of NA. +num_arg <- function(i, default) { + if (length(args) >= i && !is.na(args[i]) && nzchar(args[i])) as.numeric(args[i]) else default +} +molecular_distance_cutoff <- num_arg(7, 2.7) +flagCell_lrtest_cutoff <- num_arg(8, 5) +svmClass_score_cutoff <- num_arg(9, -2) +cutoff_spatialMerge <- num_arg(10, 0.5) + ### reading in data count_df <- as.matrix(read.csv(path_to_counts, row.names = 1)) @@ -54,8 +72,8 @@ refineAll_res_one_FOC <- fastReseg_full_pipeline( # Similar to `runPreprocess()`, one can set various cutoffs to NULL for automatic calculation from input data # distance cutoff for neighborhood searching at molecular and cellular levels, respectively - molecular_distance_cutoff = 2.7, - cellular_distance_cutoff = NULL, + molecular_distance_cutoff = molecular_distance_cutoff, # exposed Viash arg (default 2.7) + cellular_distance_cutoff = NULL, # cutoffs for transcript scores and number for cells under each cell type score_baseline = NULL, @@ -64,11 +82,11 @@ refineAll_res_one_FOC <- fastReseg_full_pipeline( imputeFlag_missingCTs = TRUE, # Settings for error detection and correction, refer to `runSegRefinement()` for more details - flagCell_lrtest_cutoff = 5, # cutoff to flag for cells with strong spatial dependcy in transcript score profiles - svmClass_score_cutoff = -2, # cutoff of transcript score to separate between high and low score classes + flagCell_lrtest_cutoff = flagCell_lrtest_cutoff, # exposed Viash arg (default 5): flag cells with strong spatial dependency in transcript score profiles + svmClass_score_cutoff = svmClass_score_cutoff, # exposed Viash arg (default -2): separates high vs low transcript-score classes groupTranscripts_method = "dbscan", - spatialMergeCheck_method = "leidenCut", - cutoff_spatialMerge = 0.5, # spatial constraint cutoff for a valid merge event + spatialMergeCheck_method = "leidenCut", + cutoff_spatialMerge = cutoff_spatialMerge, # exposed Viash arg (default 0.5): spatial constraint cutoff for a valid merge event path_to_output = "res2_multiFiles", save_intermediates = TRUE, # flag to return and write intermediate results to disk diff --git a/src/methods_transcript_assignment/proseg/config.vsh.yaml b/src/methods_transcript_assignment/proseg/config.vsh.yaml index 05e425a4c..4a4bd3374 100644 --- a/src/methods_transcript_assignment/proseg/config.vsh.yaml +++ b/src/methods_transcript_assignment/proseg/config.vsh.yaml @@ -53,9 +53,55 @@ arguments: type: double required: false description: "Larger voxel size to use for the burn-in phase. (This must be an integer multiple of the final voxel size)." - direction: input + direction: input default: 2.0 + # ---- Model knobs previously hardcoded in script.py, now exposed for tuning. ---- + # Defaults below == proseg's own CLI defaults (src/main.rs), EXCEPT that the previously + # hardcoded cell_compactness (0.03) is reset here to proseg's true default (0.04); the + # deviating 0.03 is covered by the parameter sweep instead. See NOTES.md. + - name: --cell_compactness + type: double + required: false + description: "Exponential prior on cell compactness; smaller values induce more compact cells (proseg --cell-compactness)." + direction: input + default: 0.04 + + - name: --nuclear_reassignment_prob + type: double + required: false + description: "Prior probability of reassigning a transcript away from its prior (nuclear) segmentation label (proseg --nuclear-reassignment-prob)." + direction: input + default: 0.2 + + - name: --diffusion_probability + type: double + required: false + description: "Prior probability that a transcript diffused from its cell of origin, controlling how aggressively transcripts are repositioned (proseg --diffusion-probability)." + direction: input + default: 0.2 + + - name: --ncomponents + type: integer + required: false + description: "Number of components in the mixture model of cellular gene expression (proseg --ncomponents)." + direction: input + default: 10 + + - name: --diffusion_sigma_far + type: double + required: false + description: "Stddev (microns) for repositioning of diffused transcripts (proseg --diffusion-sigma-far)." + direction: input + default: 4.0 + + - name: --diffusion_sigma_near + type: double + required: false + description: "Stddev (microns) for repositioning of un-diffused transcripts (proseg --diffusion-sigma-near)." + direction: input + default: 1.0 + resources: - type: python_script path: script.py diff --git a/src/methods_transcript_assignment/segger/NOTES.md b/src/methods_transcript_assignment/segger/NOTES.md index 685191f07..4fd957c9b 100644 --- a/src/methods_transcript_assignment/segger/NOTES.md +++ b/src/methods_transcript_assignment/segger/NOTES.md @@ -321,3 +321,77 @@ Nextflow runner labels: `[hightime, midcpu, highmem, gpuh100]`. label-image index — matches `basic_transcript_assignment`'s convention. - The Docker step ordering (esp. the final pandas pin) is fragile; changing the base image usually cascades into all the version pins above. + +## Optimization / tuning + +Parameter sweep for optimizing segger's transcript-assignment quality. The sweep files live +under `scripts/run_benchmark/param_sweep/` (`segger_params.yaml` + `run_test_segger_nebius.sh`). +`run_benchmark` expands the params file as a **star around the default** (one extra variant per +swept value, that ONE arg overridden — NOT a grid), so total variants = +1 default + Σ(sweep list lengths) = **8**. GPU runs are expensive, so the sweep is kept lean and +touches **only already-exposed args ⇒ SUBMITTABLE AS-IS (no container rebuild)**. + +### Non-default audit (vs segger 0.1.0 @ `0233cf62`) + +Every exposed quality knob's shipped default MATCHES segger 0.1.0's own default — there is **no** +pre-existing deviation to force onto a sweep axis: + +| arg | shipped | segger 0.1.0 default | source | +|-----|---------|----------------------|--------| +| `n_epochs` | 20 | 20 | `cli/segment.py` (`n_epochs=20` literal) | +| `prediction_graph_buffer_ratio` | 0.05 | 0.05 | `data/data_module.py:151` (`= 0.05`) | +| `prediction_mode` | `cell` | `cell` | `data/data_module.py:149` (`prediction_graph_mode="cell"`) | +| `node_representation_dim` | 128 | 128 | `data/data_module.py:138` (`cells_embedding_size=128`) | + +⚠️ **Correction to the arg-table note above:** the `0.5` mentioned there was the default of the +**OLD** `--prediction-expansion-ratio` flag. In 0.1.0 that flag was renamed to +`--prediction-graph-buffer-ratio` AND its default was lowered to **0.05** — exactly what we ship. +So 0.05 is *not* a deviation; it is segger's current default. (`transcripts_key` / +`coordinate_system` are our adapter's I/O keys, not segger knobs — fixed, never swept.) + +### Tiers + +- **Tier 0 — inputs, not parameters.** The biggest levers aren't in the sweep: the segmentation + prior fed in (which segmentation method, how good its boundaries) and the fraction of + transcripts that fall inside the label window (OOB transcripts are dropped — see the empty-`bd` + gotcha). Both are fixed by the upstream stage here. +- **Tier 1 — highest impact on *what* gets assigned:** + - **`prediction_mode`** {`nucleus`,`cell`,`uniform`} — which polygon set drives the prediction + graph. We write the SAME polygons to both cell/nucleus files, so `nucleus` vs `cell` differ + only in how segger treats them internally; `uniform` drops the prior-boundary identity. + Default `cell`; sweep the other two → `[nucleus, uniform]`. + - **`prediction_graph_buffer_ratio`** — fraction of each polygon's equivalent radius used to + buffer (expand) it when building the prediction graph. The direct **recall↔precision** dial: + a larger buffer expands each cell so it captures more surrounding transcripts (↑recall, + ↓precision). Default 0.05 (tight); sweep UP → `[0.1, 0.25, 0.5]` (0.5 == the old + expansion-ratio default — a natural upper anchor). +- **Tier 2 — quality/speed trade-off:** + - **`n_epochs`** — GNN training epochs. More epochs = better convergence at a linear GPU-time + cost (60 ≈ 3× the default's train time). Default 20; sweep UP → `[40, 60]`. +- **Tier 3 — not exposed (would need a config arg + a GPU/RAPIDS container rebuild — deliberately + NOT done, to keep the sweep submittable-as-is).** Highest-value candidates for a serious + follow-up: `prediction_max_k` (default 3 — k for the prediction kNN graph, another recall dial), + `cells_representation` (`pca`/`morphology`), `segmentation_loss` (`triplet`/`bce`), + `transcripts_max_dist`/`transcripts_max_k` (transcript-transcript graph radius), `learning_rate`. + +### Why `node_representation_dim` is NOT swept + +It maps to segger's `cells_embedding_size`/`in_channels` and sets the PCA `n_components` for the +gene embeddings. It is effectively an **upper bound**: on small panels/crops fewer genes survive +segger's count filters than 128, and `script.py` already auto-retries at the ceiling segger +reports. Sweeping it lower would only cap the embedding capacity (not improve quality); higher +would just re-hit the same ceiling. It is a *capacity/robustness* knob, not a quality axis → left +fixed at 128. + +### Sweep summary + +| tier | arg | default | swept values | +|------|-----|---------|--------------| +| 1 | `prediction_mode` | `cell` | `nucleus`, `uniform` | +| 1 | `prediction_graph_buffer_ratio` | 0.05 | 0.1, 0.25, 0.5 | +| 2 | `n_epochs` | 20 | 40, 60 | + +Total = 1 default + 2 + 3 + 2 = **8 variants**. All args already exposed ⇒ **SUBMITTABLE AS-IS**. +segger is GPU-only: the `gpuh100` label in the config + `src/base/labels_nebius.config` +(`runAsUser:0` + Memory-backed `/dev/shm`) pin it to the GPU node group — no GPU-specific change +is needed in the sweep script.