From 10eecf1c3e9947ceee983ff70ea95e2dc1347e49 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Mon, 21 Apr 2025 15:34:57 -0400 Subject: [PATCH 01/78] modules for assign and measure spots; changed spots container for post-rs-fish --- modules/local/assign_spots/main.nf | 55 +++++++++++++++++++++++ modules/local/measure_spots/main.nf | 57 ++++++++++++++++++++++++ modules/local/multiscale/pyramid/main.nf | 4 +- modules/local/post_rs_fish/main.nf | 18 ++++---- 4 files changed, 123 insertions(+), 11 deletions(-) create mode 100644 modules/local/assign_spots/main.nf create mode 100644 modules/local/measure_spots/main.nf diff --git a/modules/local/assign_spots/main.nf b/modules/local/assign_spots/main.nf new file mode 100644 index 00000000..d266cbd7 --- /dev/null +++ b/modules/local/assign_spots/main.nf @@ -0,0 +1,55 @@ +process ASSIGN_SPOTS { + tag { meta.id } + container { task.ext.container ?: 'ghcr.io/janeliascicomp/easifish-spots-utils:v1' } + cpus { ncpus } + memory { "${mem_in_gb}GB" } + + input: + tuple val(meta), + path(input_image_path), + val(input_dataset), + path(labels_path), + val(labels_dataset), + path(spots_input_dir), + val(input_pattern), + path(output_dir) + val(ncpus) + val(mem_in_gb) + + output: + tuple val(meta), + env(full_input_image_path), + val(input_dataset), + env(full_spots_input_dir), + env(output_csv_file), emit: results + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + """ + full_input_image_path=\$(readlink -e ${input_image_path}) + full_labels_path=\$(readlink -e ${labels_path}) + full_spots_input_dir=\$(readlink -e ${spots_input_dir}) + full_output_dir=\$(readlink ${output_dir}) + + mkdir -p \${full_output_dir} + + output_csv_file="\${full_output_dir}/count.csv" + + python /opt/scripts/spots-utils/assign-spots.py \ + --image-container \${full_input_image_path} \ + --image-subpath ${input_dataset} \ + --labels-container \${full_labels_path} \ + --labels-subpath ${labels_dataset} \ + --spots-pattern "\"\${full_spots_input_dir}/${input_pattern}\"" \ + --output \${output_csv_file} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + assign-spots: v1 + END_VERSIONS + """ + +} diff --git a/modules/local/measure_spots/main.nf b/modules/local/measure_spots/main.nf new file mode 100644 index 00000000..a72303a5 --- /dev/null +++ b/modules/local/measure_spots/main.nf @@ -0,0 +1,57 @@ +process MEASURE_SPOTS { + tag { meta.id } + container { task.ext.container ?: 'ghcr.io/janeliascicomp/easifish-spots-utils:v1' } + cpus { ncpus } + memory { "${mem_in_gb}GB" } + + input: + tuple val(meta), + path(input_image_path), + val(input_dataset), + path(labels_path), + val(labels_dataset), + val(dapi_dataset), + val(bleed_dataset), + path(output_dir), + val(output_name) + val(ncpus) + val(mem_in_gb) + + output: + tuple val(meta), + env(full_input_image_path), + val(input_dataset), + env(output_csv_file), emit: results + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + dapi_dataset_arg = dapi_dataset ? "--dapi-subpath ${dapi_dataset}" : "" + bleed_dataset_arg = bleed_dataset ? "--bleed-subpath ${bleed_dataset}" : "" + """ + full_input_image_path=\$(readlink -e ${input_image_path}) + full_labels_path=\$(readlink -e ${labels_path}) + full_output_dir=\$(readlink ${output_dir}) + + mkdir -p \${full_output_dir} + + output_csv_file="\${full_output_dir}/${output_name}" + + python /opt/scripts/spots-utils/measure-spots.py \ + --image-container \${full_input_container_path} \ + --image-subpath ${input_dataset} \ + --labels-container \${full_labels_path} \ + --labels-subpath ${labels_dataset} \ + ${dapi_dataset_arg} \ + ${bleed_dataset_arg} \ + --output \${output_csv_file} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + measure-spots: v1 + END_VERSIONS + """ + +} diff --git a/modules/local/multiscale/pyramid/main.nf b/modules/local/multiscale/pyramid/main.nf index 72c26c45..e2c99289 100644 --- a/modules/local/multiscale/pyramid/main.nf +++ b/modules/local/multiscale/pyramid/main.nf @@ -23,7 +23,7 @@ process MULTISCALE_PYRAMID { # Create command line parameters full_n5_container_path=\$(readlink -e ${n5_container}) echo "Generate pyramid for \${full_n5_container_path}:${fullscale_dataset}" - # move existing scale levels because the downsampler + # move existing scale levels because the downsampler # fails if a downsample folder already exists intermediate_data=\$(echo "$fullscale_dataset" | sed -e s/s0\$/intermediate-downsampling-XY/) renamed_intermediate_data=\$(echo "$fullscale_dataset" | sed -e s/s0\$/prev-intermediate-downsampling-XY/) @@ -61,4 +61,4 @@ process MULTISCALE_PYRAMID { END_VERSIONS """ -} \ No newline at end of file +} diff --git a/modules/local/post_rs_fish/main.nf b/modules/local/post_rs_fish/main.nf index a6379a07..07f91930 100644 --- a/modules/local/post_rs_fish/main.nf +++ b/modules/local/post_rs_fish/main.nf @@ -1,17 +1,17 @@ process POST_RS_FISH { tag { meta.id } - container { task.ext.container ?: 'ghcr.io/janeliascicomp/post-rs-fish:v1' } + container { task.ext.container ?: 'ghcr.io/janeliascicomp/easifish-spots-utils:v1' } input: tuple val(meta), - path(input_container), + path(input_path), val(input_dataset), val(voxel_spots_csv_file) output: - tuple val(meta), - path(input_container), - val(input_dataset), + tuple val(meta), + path(input_path), + val(input_dataset), env(coord_spots_csv_file), emit: results path "versions.yml" , emit: versions @@ -24,13 +24,13 @@ process POST_RS_FISH { """ # Create command line parameters - full_input_container_path=\$(readlink -e ${input_container}) + full_input_path=\$(readlink -e ${input_path}) voxel_spots_csv_file=\$(readlink -e ${voxel_spots_csv_file}) voxel_spots_csv_dir=\$(dirname \${voxel_spots_csv_file}) coord_spots_csv_file=\${voxel_spots_csv_dir}/${spots_filename} - python /opt/scripts/post-rs-fish/post-rs-fish.py \ - --image-container \${full_input_container_path} \ + python /opt/scripts/spots-utils/post-rs-fish.py \ + --image-container \${full_input_path} \ --image-subpath ${input_dataset} \ --input \${voxel_spots_csv_file} \ --output \${coord_spots_csv_file} @@ -41,4 +41,4 @@ process POST_RS_FISH { END_VERSIONS """ -} \ No newline at end of file +} From 1c5a32dc9c59aa9b672cdcacdc8546122ea31c5d Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Mon, 21 Apr 2025 17:26:06 -0400 Subject: [PATCH 02/78] changed warp_spots to return both fixed round spots - unchanged - and warped spots from moving rounds --- workflows/easifish.nf | 5 +++-- workflows/warp_spots.nf | 29 ++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/workflows/easifish.nf b/workflows/easifish.nf index c83d798e..69263a58 100644 --- a/workflows/easifish.nf +++ b/workflows/easifish.nf @@ -160,12 +160,13 @@ workflow EASIFISH { spot_extraction_results.subscribe { log.debug "Spot extraction result: $it " } - WARP_SPOTS( + def final_spot_results = WARP_SPOTS( registration_results, spot_extraction_results, outdir, - ) + ) // final_spot_results includes spots for fixed and warped spots from the moving rounds + final_spot_results.subscribe { log.debug "Final spot results: $it " } } workflow.onComplete { diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index 17cd91fd..5fffe4ab 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -30,6 +30,10 @@ workflow WARP_SPOTS { [ id, reg_meta, inv_transform_output, inv_transform_name, inv_transform_subpath ] } + def all_moving_rounds = transform + | map { it[0] /* id */ } + | toList() + def spots = spot_extraction_results | map { log.debug "Prepare warp spot inputs: $it" @@ -41,18 +45,33 @@ workflow WARP_SPOTS { ) = it def id = meta.id - [ + [ id, + meta, spots_file, image_container, image_dataset, ] } + def fixed_spots = spots + | filter { !(it]0] in all_moving_rounds) } + | map { + def (id, meta, spots_file) = it + def r = [ + meta, + spots_file, + spots, + ] + log.debug "Fixed spots: $id -> $r" + r + } + def spots_warp_input = spots | join(transform, by: 0) | map { def ( id, + meta, spots_file, image_container, image_dataset, reg_meta, @@ -65,7 +84,7 @@ workflow WARP_SPOTS { def spots_filename = file(spots_file).name [ [ - reg_meta, spots_file, warped_spots_output_dir, "warped-${spots_filename}", + meta, spots_file, warped_spots_output_dir, "warped-${spots_filename}", ], [ image_container, image_dataset, @@ -106,10 +125,10 @@ workflow WARP_SPOTS { } else { // skip warp spots spots_warp_results = spots_warp_input.map { - def (reg_meta, spots_file, warped_spots_output_dir, warped_spots_filename) = it[0] + def (meta, spots_file, warped_spots_output_dir, warped_spots_filename) = it[0] def r = [ - reg_meta, + meta, spots_file, "${warped_spots_output_dir}/${warped_spots_filename}", ] @@ -119,5 +138,5 @@ workflow WARP_SPOTS { } emit: - done = spots_warp_results + done = fixed_spots.mix(spots_warp_results) } From 4877fa65f10873f0ad839d25a626fc0ceb0dea95 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Mon, 21 Apr 2025 18:03:41 -0400 Subject: [PATCH 03/78] added some comments --- workflows/easifish.nf | 1 + workflows/segmentation.nf | 1 + workflows/spot_extraction.nf | 9 +++++---- workflows/warp_spots.nf | 14 +++++++++++++- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/workflows/easifish.nf b/workflows/easifish.nf index 69263a58..8763da0f 100644 --- a/workflows/easifish.nf +++ b/workflows/easifish.nf @@ -167,6 +167,7 @@ workflow EASIFISH { ) // final_spot_results includes spots for fixed and warped spots from the moving rounds final_spot_results.subscribe { log.debug "Final spot results: $it " } + } workflow.onComplete { diff --git a/workflows/segmentation.nf b/workflows/segmentation.nf index 12436ff7..62ce269f 100644 --- a/workflows/segmentation.nf +++ b/workflows/segmentation.nf @@ -23,6 +23,7 @@ workflow SEGMENTATION { def segmentation_ids = as_list(params.segmentation_ids) // get volumes to segment + // typically this is done for the DAPI channel of the fixed round def seg_volume = ch_meta | filter { meta -> params.segmentation_input || meta.id in segmentation_ids diff --git a/workflows/spot_extraction.nf b/workflows/spot_extraction.nf index 59c1fc4e..af992d31 100644 --- a/workflows/spot_extraction.nf +++ b/workflows/spot_extraction.nf @@ -24,6 +24,7 @@ workflow SPOT_EXTRACTION { spot_volume_ids.empty || meta.id in spot_volume_ids } | map { meta -> + // Spot extraction is typically done for all cell (no DAPI) channels from all rounds def input_img_dir = get_spot_extraction_input_volume(meta) def spots_output_dir = file("${outputdir}/${params.spot_extraction_subdir}/${meta.id}") [ @@ -35,7 +36,7 @@ workflow SPOT_EXTRACTION { def final_rsfish_results if (params.skip_spot_extraction) { log.info "Skipping spot extraction" - // even if we skip spot extraction + // even if we skip spot extraction // we assume we have the csv file and we apply the post processing spots_spark_input | flatMap { @@ -146,7 +147,7 @@ def get_spot_subpaths(meta) { def input_img_dir = get_spot_extraction_input_volume(meta) if (!params.spot_subpaths && !params.spot_channels && !params.spot_scales) { - return [ + return [ ['', ''], // empty subpath, empty resultnane - the input image container contains the array dataset ] } else if (params.spot_subpaths) { @@ -162,7 +163,7 @@ def get_spot_subpaths(meta) { spot_channels = as_list(params.spot_channels) log.debug "Use specified spot channels: $spot_channels" } else { - // all but the last channel whcih typically is DAPI + // all but the last channel which typically is DAPI def all_channels = as_list(params.channels) // this may throw an exception if the channel list is empty or a singleton spot_channels = all_channels[0..-2] @@ -180,4 +181,4 @@ def get_spot_subpaths(meta) { ] } } -} \ No newline at end of file +} diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index 5fffe4ab..05b66c6a 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -136,7 +136,19 @@ workflow WARP_SPOTS { log.debug "Skipping warp spots and return: $r" } } + def final_spot_results = fixed_spots.mix(spots_warp_results) + | join(spot_extraction_results, by: 0) + | map { + def (meta, spots_file, warped_spots_file, image_container, image_dataset) = it + def r = [ + meta, + image_container, image_dataset, // include the image used for spot extraction in the results + spots_file, warped_spots_file, + ] + log.debug "Final spot results: $it -> $r" + r + } emit: - done = fixed_spots.mix(spots_warp_results) + done = final_spot_results } From 278f9be733c5d084309243bebd16256edc403420 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 09:42:45 -0400 Subject: [PATCH 04/78] fixed typo --- workflows/warp_spots.nf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index 05b66c6a..81067f74 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -54,7 +54,7 @@ workflow WARP_SPOTS { } def fixed_spots = spots - | filter { !(it]0] in all_moving_rounds) } + | filter { !(it[0] in all_moving_rounds) } | map { def (id, meta, spots_file) = it def r = [ @@ -136,6 +136,7 @@ workflow WARP_SPOTS { log.debug "Skipping warp spots and return: $r" } } + def final_spot_results = fixed_spots.mix(spots_warp_results) | join(spot_extraction_results, by: 0) | map { From 279eabf6a17899e3b7e5d91afc63874d033b7cc8 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 10:09:08 -0400 Subject: [PATCH 05/78] changed result returned when skipping segmentation --- modules.json | 2 +- modules/janelia/cellpose/main.nf | 6 +++--- subworkflows/local/cellpose_segmentation.nf | 21 +++++++++++++++------ workflows/warp_spots.nf | 2 +- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/modules.json b/modules.json index 8392d1f0..19835ca7 100644 --- a/modules.json +++ b/modules.json @@ -32,7 +32,7 @@ }, "cellpose": { "branch": "main", - "git_sha": "5619a98c3ea6062e65490fc972545eb182810da6", + "git_sha": "8870958d52ff73b924944ece7e2b018c2ec9f91f", "installed_by": ["modules"] }, "cellpose/tests": { diff --git a/modules/janelia/cellpose/main.nf b/modules/janelia/cellpose/main.nf index ab9f98a2..1ff7e804 100644 --- a/modules/janelia/cellpose/main.nf +++ b/modules/janelia/cellpose/main.nf @@ -18,9 +18,9 @@ process CELLPOSE { val(cellpose_mem_in_gb) output: - tuple val(meta), path(image), path("${output_dir}/${output_name_noext}*${output_name_ext}"), emit: results - tuple val(meta), val(output_name_noext), val(output_name_ext) , emit: result_names - path('versions.yml') , emit: versions + tuple val(meta), env(input_image_fullpath), path("${output_dir}/${output_name_noext}*${output_name_ext}"), emit: results + tuple val(meta), val(output_name_noext), val(output_name_ext) , emit: result_names + path('versions.yml') , emit: versions when: task.ext.when == null || task.ext.when diff --git a/subworkflows/local/cellpose_segmentation.nf b/subworkflows/local/cellpose_segmentation.nf index 977d6283..6971d851 100644 --- a/subworkflows/local/cellpose_segmentation.nf +++ b/subworkflows/local/cellpose_segmentation.nf @@ -5,9 +5,9 @@ include { DASK_STOP } from '../janelia/dask_stop/main.nf' workflow CELLPOSE_SEGMENTATION { take: - ch_meta // channel: [ meta, + ch_meta // channel: [ meta, // img_container_dir, img_dataset, - // output_dir, + // output_dir, // segmentation_container ] skip // boolean: if true skip segmentation completely and just return the meta as if it ran models_dir // string|file: directory @@ -29,7 +29,7 @@ workflow CELLPOSE_SEGMENTATION { | multiMap { def (meta, img_container_dir, img_dataset, output_dir, segmentation_container) = it log.debug "Start to prepare inputs for cellpose segmentation: $it" - def segmentation_work_dir = work_dir + def segmentation_work_dir = work_dir ? file("${work_dir}/${meta.id}/${workflow.sessionId}/${img_dataset}") : file("${output_dir}/${meta.id}/${workflow.sessionId}/${img_dataset}") @@ -108,12 +108,12 @@ workflow CELLPOSE_SEGMENTATION { log_config ? file(log_config) : [], segmentation_cpus, segmentation_mem_gb, - ) - segmentation_results.results.subscribe { + ).results + segmentation_results.subscribe { log.debug "Segmentation results: $it" } - dask_cluster.join(segmentation_results.results, by:0) + dask_cluster.join(segmentation_results, by:0) | map { def (meta, cluster_context) = it [ meta, cluster_context ] @@ -125,6 +125,15 @@ workflow CELLPOSE_SEGMENTATION { } else { // FIXME: final_segmentation_results = ch_meta + | map { + def (meta, img_container_dir, img_dataset, output_dir, segmentation_container) = it + log.debug "Skip segmentation: $it" + [ + meta, + img_container_dir, + "${output_dir}/${segmentation_container}", + ] + } } emit: diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index 81067f74..67317ea0 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -151,5 +151,5 @@ workflow WARP_SPOTS { } emit: - done = final_spot_results + done = final_spot_results // [ meta, image_container, image_dataset, spots_file, warped_spots_file ] } From 14dadebb053ec64d67898f08c415a338b4584b2b Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 10:09:30 -0400 Subject: [PATCH 06/78] removed fixme comment --- subworkflows/local/cellpose_segmentation.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/subworkflows/local/cellpose_segmentation.nf b/subworkflows/local/cellpose_segmentation.nf index 6971d851..6cabdec9 100644 --- a/subworkflows/local/cellpose_segmentation.nf +++ b/subworkflows/local/cellpose_segmentation.nf @@ -123,7 +123,6 @@ workflow CELLPOSE_SEGMENTATION { final_segmentation_results = segmentation_results.results } else { - // FIXME: final_segmentation_results = ch_meta | map { def (meta, img_container_dir, img_dataset, output_dir, segmentation_container) = it From 99069dd96962d715362f02fe805bb9c5d140aa3c Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 10:13:49 -0400 Subject: [PATCH 07/78] fixed post-rs-fish container --- conf/spot-extraction-modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/spot-extraction-modules.config b/conf/spot-extraction-modules.config index ae160ed1..36663b7e 100644 --- a/conf/spot-extraction-modules.config +++ b/conf/spot-extraction-modules.config @@ -48,7 +48,7 @@ process { } withName: "(.*)?SPOT_EXTRACTION:POST_RS_FISH" { - container = 'ghcr.io/janeliascicomp/post-rs-fish:v1' + container = 'ghcr.io/janeliascicomp/easifish-spots-utils:v1' } } From 06dead7fbba1f3fc1e86a120d5c4ee8adf8e941d Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 10:21:13 -0400 Subject: [PATCH 08/78] renamed var --- subworkflows/local/cellpose_segmentation.nf | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/subworkflows/local/cellpose_segmentation.nf b/subworkflows/local/cellpose_segmentation.nf index 6cabdec9..917ba427 100644 --- a/subworkflows/local/cellpose_segmentation.nf +++ b/subworkflows/local/cellpose_segmentation.nf @@ -102,26 +102,27 @@ workflow CELLPOSE_SEGMENTATION { cluster_info: cluster_info } - def segmentation_results = CELLPOSE( + def cellpose_outputs = CELLPOSE( segmentation_inputs.cellpose_data, segmentation_inputs.cluster_info, log_config ? file(log_config) : [], segmentation_cpus, segmentation_mem_gb, - ).results - segmentation_results.subscribe { + ) + + final_segmentation_results = cellpose_outputs.results + + final_segmentation_results.subscribe { log.debug "Segmentation results: $it" } - dask_cluster.join(segmentation_results, by:0) + dask_cluster.join(final_segmentation_results, by:0) | map { def (meta, cluster_context) = it [ meta, cluster_context ] } | groupTuple | DASK_STOP - - final_segmentation_results = segmentation_results.results } else { final_segmentation_results = ch_meta | map { From 4f607946f1fcd238e8b5f2d71c0e45946bce02f2 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 10:45:20 -0400 Subject: [PATCH 09/78] changed log messages --- subworkflows/local/cellpose_segmentation.nf | 6 +++--- workflows/warp_spots.nf | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/subworkflows/local/cellpose_segmentation.nf b/subworkflows/local/cellpose_segmentation.nf index 917ba427..b2fe0f26 100644 --- a/subworkflows/local/cellpose_segmentation.nf +++ b/subworkflows/local/cellpose_segmentation.nf @@ -113,7 +113,7 @@ workflow CELLPOSE_SEGMENTATION { final_segmentation_results = cellpose_outputs.results final_segmentation_results.subscribe { - log.debug "Segmentation results: $it" + log.debug "Cellpose results: $it" } dask_cluster.join(final_segmentation_results, by:0) @@ -127,10 +127,10 @@ workflow CELLPOSE_SEGMENTATION { final_segmentation_results = ch_meta | map { def (meta, img_container_dir, img_dataset, output_dir, segmentation_container) = it - log.debug "Skip segmentation: $it" + log.debug "Skip cellpose segmentation: $it" [ meta, - img_container_dir, + img_container_dir, img_dataset, "${output_dir}/${segmentation_container}", ] } diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index 67317ea0..ce222ca0 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -121,7 +121,7 @@ workflow WARP_SPOTS { spots_warp_results = BIGSTREAM_TRANSFORMCOORDS.out.results - spots_warp_results.subscribe { log.debug "Warp spots results: $it " } + spots_warp_results.subscribe { log.debug "Bigstream transform coords results: $it " } } else { // skip warp spots spots_warp_results = spots_warp_input.map { @@ -146,7 +146,7 @@ workflow WARP_SPOTS { image_container, image_dataset, // include the image used for spot extraction in the results spots_file, warped_spots_file, ] - log.debug "Final spot results: $it -> $r" + log.debug "All (warped and unwarped) spot results: $it -> $r" r } From 2730bf91621d235982da0f551c6f0e1f5c2d3b2a Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 10:46:03 -0400 Subject: [PATCH 10/78] update cellpose to include dataset in the results --- modules.json | 2 +- modules/janelia/cellpose/main.nf | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules.json b/modules.json index 19835ca7..b8718365 100644 --- a/modules.json +++ b/modules.json @@ -32,7 +32,7 @@ }, "cellpose": { "branch": "main", - "git_sha": "8870958d52ff73b924944ece7e2b018c2ec9f91f", + "git_sha": "9551bf8f81326aca1ee01fb8e664533177a7a779", "installed_by": ["modules"] }, "cellpose/tests": { diff --git a/modules/janelia/cellpose/main.nf b/modules/janelia/cellpose/main.nf index 1ff7e804..ece0d7ad 100644 --- a/modules/janelia/cellpose/main.nf +++ b/modules/janelia/cellpose/main.nf @@ -18,9 +18,9 @@ process CELLPOSE { val(cellpose_mem_in_gb) output: - tuple val(meta), env(input_image_fullpath), path("${output_dir}/${output_name_noext}*${output_name_ext}"), emit: results - tuple val(meta), val(output_name_noext), val(output_name_ext) , emit: result_names - path('versions.yml') , emit: versions + tuple val(meta), env(input_image_fullpath), val(image_subpath), path("${output_dir}/${output_name_noext}*${output_name_ext}"), emit: results + tuple val(meta), val(output_name_noext), val(output_name_ext) , emit: result_names + path('versions.yml') , emit: versions when: task.ext.when == null || task.ext.when From b25b48e8fd376e5a73df423ee15eeb83d328912c Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 13:25:21 -0400 Subject: [PATCH 11/78] added step to extract spots features --- conf/modules.config | 2 + conf/spots-features-modules.config | 4 ++ modules.json | 4 +- modules/janelia/cellpose/main.nf | 11 +++-- .../janelia/cellpose/tests/nextflow.config | 2 +- .../regionprops}/main.nf | 4 +- .../{assign_spots => spots/sizes}/main.nf | 4 +- workflows/easifish.nf | 7 +++ workflows/segmentation.nf | 2 +- workflows/spots_features.nf | 43 +++++++++++++++++++ 10 files changed, 72 insertions(+), 11 deletions(-) create mode 100644 conf/spots-features-modules.config rename modules/local/{measure_spots => spots/regionprops}/main.nf (94%) rename modules/local/{assign_spots => spots/sizes}/main.nf (94%) create mode 100644 workflows/spots_features.nf diff --git a/conf/modules.config b/conf/modules.config index 069cc844..bf261c63 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -16,3 +16,5 @@ includeConfig './multiscale-modules.config' includeConfig './segmentation-modules.config' includeConfig './spot-extraction-modules.config' includeConfig './warp-spots-modules.config' +includeConfig './spots-features-modules.config' + diff --git a/conf/spots-features-modules.config b/conf/spots-features-modules.config new file mode 100644 index 00000000..f9bb6b23 --- /dev/null +++ b/conf/spots-features-modules.config @@ -0,0 +1,4 @@ +params { + spots_sizes_cores = 1 + spots_sizes_mem_gb = 1 +} diff --git a/modules.json b/modules.json index b8718365..007fcb5f 100644 --- a/modules.json +++ b/modules.json @@ -32,12 +32,12 @@ }, "cellpose": { "branch": "main", - "git_sha": "9551bf8f81326aca1ee01fb8e664533177a7a779", + "git_sha": "9b06bebf0b46ad077a8d73d488921f0dbf3262a9", "installed_by": ["modules"] }, "cellpose/tests": { "branch": "main", - "git_sha": "8232ff68e737a6788759d1f0f0cf50fd82580ea5", + "git_sha": "9b06bebf0b46ad077a8d73d488921f0dbf3262a9", "installed_by": ["modules"] }, "rs_fish": { diff --git a/modules/janelia/cellpose/main.nf b/modules/janelia/cellpose/main.nf index ece0d7ad..3ecbddb6 100644 --- a/modules/janelia/cellpose/main.nf +++ b/modules/janelia/cellpose/main.nf @@ -18,9 +18,9 @@ process CELLPOSE { val(cellpose_mem_in_gb) output: - tuple val(meta), env(input_image_fullpath), val(image_subpath), path("${output_dir}/${output_name_noext}*${output_name_ext}"), emit: results - tuple val(meta), val(output_name_noext), val(output_name_ext) , emit: result_names - path('versions.yml') , emit: versions + tuple val(meta), env(input_image_fullpath), val(image_subpath), env(output_segmentation_results), emit: results + tuple val(meta), val(output_name_noext), val(output_name_ext) , emit: result_names + path('versions.yml') , emit: versions when: task.ext.when == null || task.ext.when @@ -81,6 +81,11 @@ process CELLPOSE { ${logging_config_arg} \ ${args} + output_segmentation_results=() + for sr in \$(ls \${output_fullpath} | grep "${output_name_noext}.*${output_name_ext}") ; do + output_segmentation_results+=("\${output_fullpath}/\${sr}") + done + cellpose_version=\$(python /opt/scripts/cellpose/main_distributed_cellpose.py \ --version | \ grep "cellpose version" | \ diff --git a/modules/janelia/cellpose/tests/nextflow.config b/modules/janelia/cellpose/tests/nextflow.config index 3d52f0a2..715a5aac 100644 --- a/modules/janelia/cellpose/tests/nextflow.config +++ b/modules/janelia/cellpose/tests/nextflow.config @@ -34,7 +34,7 @@ params { process { publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - ext.container = 'ghcr.io/janeliascicomp/cellpose:3.1.0-dask2025.1.0-py12' + container = 'ghcr.io/janeliascicomp/cellpose:3.1.0-dask2025.1.0-py12' withName:CELLPOSE { ext { diff --git a/modules/local/measure_spots/main.nf b/modules/local/spots/regionprops/main.nf similarity index 94% rename from modules/local/measure_spots/main.nf rename to modules/local/spots/regionprops/main.nf index a72303a5..45096a52 100644 --- a/modules/local/measure_spots/main.nf +++ b/modules/local/spots/regionprops/main.nf @@ -1,4 +1,4 @@ -process MEASURE_SPOTS { +process SPOTS_REGIONPROPS { tag { meta.id } container { task.ext.container ?: 'ghcr.io/janeliascicomp/easifish-spots-utils:v1' } cpus { ncpus } @@ -39,7 +39,7 @@ process MEASURE_SPOTS { output_csv_file="\${full_output_dir}/${output_name}" - python /opt/scripts/spots-utils/measure-spots.py \ + python /opt/scripts/spots-utils/labeled-spots-regionprops.py \ --image-container \${full_input_container_path} \ --image-subpath ${input_dataset} \ --labels-container \${full_labels_path} \ diff --git a/modules/local/assign_spots/main.nf b/modules/local/spots/sizes/main.nf similarity index 94% rename from modules/local/assign_spots/main.nf rename to modules/local/spots/sizes/main.nf index d266cbd7..6c12f363 100644 --- a/modules/local/assign_spots/main.nf +++ b/modules/local/spots/sizes/main.nf @@ -1,4 +1,4 @@ -process ASSIGN_SPOTS { +process SPOTS_SIZES { tag { meta.id } container { task.ext.container ?: 'ghcr.io/janeliascicomp/easifish-spots-utils:v1' } cpus { ncpus } @@ -38,7 +38,7 @@ process ASSIGN_SPOTS { output_csv_file="\${full_output_dir}/count.csv" - python /opt/scripts/spots-utils/assign-spots.py \ + python /opt/scripts/spots-utils/labeled-spots-sizes.py \ --image-container \${full_input_image_path} \ --image-subpath ${input_dataset} \ --labels-container \${full_labels_path} \ diff --git a/workflows/easifish.nf b/workflows/easifish.nf index 8763da0f..dff928e2 100644 --- a/workflows/easifish.nf +++ b/workflows/easifish.nf @@ -25,6 +25,7 @@ include { REGISTRATION } from './registration' include { SEGMENTATION } from './segmentation' include { SPOT_EXTRACTION } from './spot_extraction' include { WARP_SPOTS } from './warp_spots' +include { SPOTS_FEATURES } from './spots_features' def validate_params() { @@ -168,6 +169,12 @@ workflow EASIFISH { final_spot_results.subscribe { log.debug "Final spot results: $it " } + def spots_features_inputs = final_spot_results + | join(segmentation_results, by: 0) + + def spots_features_results = SPOTS_FEATURES(spots_features_inputs) + + spots_features_results.subscribe { log.debug "Spots features result: $it " } } workflow.onComplete { diff --git a/workflows/segmentation.nf b/workflows/segmentation.nf index 62ce269f..7c4673f4 100644 --- a/workflows/segmentation.nf +++ b/workflows/segmentation.nf @@ -83,5 +83,5 @@ workflow SEGMENTATION { ) emit: - done = cellpose_results + done = cellpose_results // [ meta, input_image, input_dataset, output_segmentation_file ] } diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf new file mode 100644 index 00000000..f74620bb --- /dev/null +++ b/workflows/spots_features.nf @@ -0,0 +1,43 @@ +include { SPOTS_REGIONPROPS } from '../modules/local/spots/regionprops/main' +include { SPOTS_SIZES } from '../modules/local/spots/sizes/main' + +workflow SPOTS_FEATURES { + take: + ch_spots_inputs // channel: [ meta, input_image, input_dataset, spots, warped_spots, seg_input_image, seg_input_dataset, seg_labels ] + + main: + + def spots_sizes_input = ch_spots_inputs + | map { + def (meta, + image, + image_dataset, + spots_file, + warped_spots_file, + seg_input_image, + seg_input_dataset, + seg_labels) = it + def spots_input_dir = file(warped_spots_file).parent + def spots_sizes_output_dir =spots_input_dir + def r = [ + meta, + image, + image_dataset, + seg_labels, + seg_input_dataset, + spots_input_dir, + 'spots-*-coord.csv', + spots_sizes_output_dir, + ] + log "Prepare spots sizes input: $it -> $r" + } + + def spots_sizes_outputs = SPOTS_SIZES( + spots_sizes_input, + params.spots_sizes_cores, + params.spots_sizes_mem_gb, + ) + + emit: + done = spots_sizes_results.results +} From 1904007f3bb7d60c02f479fe831f2393bffc50c4 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 13:39:04 -0400 Subject: [PATCH 12/78] fixed var name --- workflows/spots_features.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index f74620bb..b61336c7 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -39,5 +39,5 @@ workflow SPOTS_FEATURES { ) emit: - done = spots_sizes_results.results + done = spots_sizes_outputs.results } From bfaf94942b2c9cbd533f3889237f9313b6a5ff70 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 13:57:04 -0400 Subject: [PATCH 13/78] use the same meta for segmentation instead of creating a different one which prevents further joins downstream --- subworkflows/local/cellpose_segmentation.nf | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/subworkflows/local/cellpose_segmentation.nf b/subworkflows/local/cellpose_segmentation.nf index b2fe0f26..46993793 100644 --- a/subworkflows/local/cellpose_segmentation.nf +++ b/subworkflows/local/cellpose_segmentation.nf @@ -35,13 +35,8 @@ workflow CELLPOSE_SEGMENTATION { def cellpose_models_dir = models_dir ? file(models_dir) : "${output_dir}/cellpose-models" - def segmentation_meta = meta + [ - segmentation_work_dir: segmentation_work_dir, - cellpose_models_dir: cellpose_models_dir, - ] - def cellpose_data = [ - segmentation_meta, + meta, img_container_dir, img_dataset, cellpose_models_dir, @@ -56,7 +51,7 @@ workflow CELLPOSE_SEGMENTATION { ] + (cellpose_models_dir ? [ cellpose_models_dir ] : []) def cluster_data = [ - segmentation_meta, + meta, cluster_dirs, ] From d74ddea0afc1ecfbc36a1aac2c13cf24824f7bdc Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 14:05:53 -0400 Subject: [PATCH 14/78] missed return --- workflows/spots_features.nf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index b61336c7..345284c4 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -17,8 +17,10 @@ workflow SPOTS_FEATURES { seg_input_image, seg_input_dataset, seg_labels) = it + log "Prepare spots sizes input from $it" + def spots_input_dir = file(warped_spots_file).parent - def spots_sizes_output_dir =spots_input_dir + def spots_sizes_output_dir = spots_input_dir def r = [ meta, image, @@ -29,7 +31,8 @@ workflow SPOTS_FEATURES { 'spots-*-coord.csv', spots_sizes_output_dir, ] - log "Prepare spots sizes input: $it -> $r" + log "Spots sizes input: $r" + r } def spots_sizes_outputs = SPOTS_SIZES( From c26d909b22b3df462fd6f186e3a49c780e9b9583 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 14:24:21 -0400 Subject: [PATCH 15/78] logging --- workflows/warp_spots.nf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index ce222ca0..88edb006 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -45,12 +45,14 @@ workflow WARP_SPOTS { ) = it def id = meta.id - [ + def r = [ id, meta, spots_file, image_container, image_dataset, ] + log.debug "Spots to warp: $id - $it -> $r" + r } def fixed_spots = spots @@ -62,7 +64,7 @@ workflow WARP_SPOTS { spots_file, spots, ] - log.debug "Fixed spots: $id -> $r" + log.debug "Fixed spots: $id - $it -> $r" r } From f14c5e313c28d6762d04bf6942e564c4eee0beb5 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 14:31:04 -0400 Subject: [PATCH 16/78] fixed the channel passed in the tuple --- workflows/warp_spots.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index 88edb006..f9c56cba 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -62,7 +62,7 @@ workflow WARP_SPOTS { def r = [ meta, spots_file, - spots, + spots_file, // no warping for fixed spots ] log.debug "Fixed spots: $id - $it -> $r" r From 698d6ea5768c8d540a2ed31009ee6d24b1a89b2d Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 14:38:42 -0400 Subject: [PATCH 17/78] rs-fish return real input image path --- modules.json | 2 +- modules/janelia/rs_fish/main.nf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules.json b/modules.json index 007fcb5f..73393cb9 100644 --- a/modules.json +++ b/modules.json @@ -42,7 +42,7 @@ }, "rs_fish": { "branch": "main", - "git_sha": "5619a98c3ea6062e65490fc972545eb182810da6", + "git_sha": "bf3bbef8f048aeb79dcbf5adada70d6e587eae5c", "installed_by": ["modules"] } } diff --git a/modules/janelia/rs_fish/main.nf b/modules/janelia/rs_fish/main.nf index c5a96297..1a1e1fdf 100755 --- a/modules/janelia/rs_fish/main.nf +++ b/modules/janelia/rs_fish/main.nf @@ -1,6 +1,6 @@ process RS_FISH { tag "${meta.id}" - container { task && task.ext.container ?: 'ghcr.io/janeliascicomp/rs-fish-spark:4d1b901' } + container { task && task.ext.container ?: 'ghcr.io/janeliascicomp/rs-fish-spark:8f8954f' } cpus { spark.driver_cores } memory { spark.driver_memory } @@ -14,7 +14,7 @@ process RS_FISH { output: tuple val(meta), - path(input_image), + env(INPUT_N5), val(input_dataset), path(spots_output_dir), val(spots_result_name), From 7388bdf2013c110eba2dce708ed7357f184350fb Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 14:52:26 -0400 Subject: [PATCH 18/78] fixed log call --- workflows/spots_features.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 345284c4..f517822f 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -17,7 +17,7 @@ workflow SPOTS_FEATURES { seg_input_image, seg_input_dataset, seg_labels) = it - log "Prepare spots sizes input from $it" + log.debug "Prepare spots sizes input from $it" def spots_input_dir = file(warped_spots_file).parent def spots_sizes_output_dir = spots_input_dir @@ -31,7 +31,7 @@ workflow SPOTS_FEATURES { 'spots-*-coord.csv', spots_sizes_output_dir, ] - log "Spots sizes input: $r" + log.debug "Spots sizes input: $r" r } From 8d9cd6cc047c79ea99b5831431314f9f4ed89998 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 15:04:58 -0400 Subject: [PATCH 19/78] stage paths to avoid conflict --- modules/local/spots/regionprops/main.nf | 6 +++--- modules/local/spots/sizes/main.nf | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/local/spots/regionprops/main.nf b/modules/local/spots/regionprops/main.nf index 45096a52..2be49ede 100644 --- a/modules/local/spots/regionprops/main.nf +++ b/modules/local/spots/regionprops/main.nf @@ -6,13 +6,13 @@ process SPOTS_REGIONPROPS { input: tuple val(meta), - path(input_image_path), + path(input_image_path, stageAs: 'image/*'), val(input_dataset), - path(labels_path), + path(labels_path, stageAs: 'labels/*'), val(labels_dataset), val(dapi_dataset), val(bleed_dataset), - path(output_dir), + path(output_dir, stageAs: 'output/*'), val(output_name) val(ncpus) val(mem_in_gb) diff --git a/modules/local/spots/sizes/main.nf b/modules/local/spots/sizes/main.nf index 6c12f363..97802d64 100644 --- a/modules/local/spots/sizes/main.nf +++ b/modules/local/spots/sizes/main.nf @@ -6,13 +6,13 @@ process SPOTS_SIZES { input: tuple val(meta), - path(input_image_path), + path(input_image_path, stageAs: 'image/*'), val(input_dataset), - path(labels_path), + path(labels_path, stageAs: 'labels/*'), val(labels_dataset), - path(spots_input_dir), + path(spots_input_dir, stageAs: 'spots/*'), val(input_pattern), - path(output_dir) + path(output_dir, stageAs: 'output/*'), val(ncpus) val(mem_in_gb) From e70def43e7086e1d3d1023fe4e65cff1b2833b2a Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 15:08:27 -0400 Subject: [PATCH 20/78] fixed syntax error - extra comma --- modules/local/spots/sizes/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/spots/sizes/main.nf b/modules/local/spots/sizes/main.nf index 97802d64..6da954dc 100644 --- a/modules/local/spots/sizes/main.nf +++ b/modules/local/spots/sizes/main.nf @@ -12,7 +12,7 @@ process SPOTS_SIZES { val(labels_dataset), path(spots_input_dir, stageAs: 'spots/*'), val(input_pattern), - path(output_dir, stageAs: 'output/*'), + path(output_dir, stageAs: 'output/*') val(ncpus) val(mem_in_gb) From 63df28d27e517344889522623bfdffdba88e8b68 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 15:24:37 -0400 Subject: [PATCH 21/78] concat fixed and warped spots instead of mix --- modules/local/post_rs_fish/main.nf | 2 +- modules/local/spots/sizes/main.nf | 2 +- workflows/warp_spots.nf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/local/post_rs_fish/main.nf b/modules/local/post_rs_fish/main.nf index 07f91930..1ba3bc15 100644 --- a/modules/local/post_rs_fish/main.nf +++ b/modules/local/post_rs_fish/main.nf @@ -10,7 +10,7 @@ process POST_RS_FISH { output: tuple val(meta), - path(input_path), + env(full_input_path), val(input_dataset), env(coord_spots_csv_file), emit: results path "versions.yml" , emit: versions diff --git a/modules/local/spots/sizes/main.nf b/modules/local/spots/sizes/main.nf index 6da954dc..62e78054 100644 --- a/modules/local/spots/sizes/main.nf +++ b/modules/local/spots/sizes/main.nf @@ -43,7 +43,7 @@ process SPOTS_SIZES { --image-subpath ${input_dataset} \ --labels-container \${full_labels_path} \ --labels-subpath ${labels_dataset} \ - --spots-pattern "\"\${full_spots_input_dir}/${input_pattern}\"" \ + --spots-pattern \"\${full_spots_input_dir}/${input_pattern}\" \ --output \${output_csv_file} cat <<-END_VERSIONS > versions.yml diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index f9c56cba..deee04d1 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -139,7 +139,7 @@ workflow WARP_SPOTS { } } - def final_spot_results = fixed_spots.mix(spots_warp_results) + def final_spot_results = fixed_spots.concat(spots_warp_results) | join(spot_extraction_results, by: 0) | map { def (meta, spots_file, warped_spots_file, image_container, image_dataset) = it From de64e9f9873421534edbd215b706234ea5d3c829 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 15:47:23 -0400 Subject: [PATCH 22/78] logging - the output from warp spots is not as I expected --- workflows/warp_spots.nf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index deee04d1..e2a98fd3 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -36,7 +36,7 @@ workflow WARP_SPOTS { def spots = spot_extraction_results | map { - log.debug "Prepare warp spot inputs: $it" + log.debug "Extracted spot: $it" def ( meta, image_container, @@ -51,12 +51,15 @@ workflow WARP_SPOTS { spots_file, image_container, image_dataset, ] - log.debug "Spots to warp: $id - $it -> $r" + log.debug "Extracted spot candidate: $id: $r" r } def fixed_spots = spots - | filter { !(it[0] in all_moving_rounds) } + | filter { + log.debug "Check if ${it[0]} is a fixed spot - should not be in ${all_moving_rounds}" + !(it[0] in all_moving_rounds) + } | map { def (id, meta, spots_file) = it def r = [ From f55a813a21e649767e0e699af271216326957db9 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 15:59:59 -0400 Subject: [PATCH 23/78] more logging; changed some cellpose params --- conf/segmentation-modules.config | 4 ++-- examples/tiny_sample_params.json | 9 +++++---- workflows/warp_spots.nf | 4 +++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/conf/segmentation-modules.config b/conf/segmentation-modules.config index 1b2ea4ca..4d3954c9 100644 --- a/conf/segmentation-modules.config +++ b/conf/segmentation-modules.config @@ -41,7 +41,7 @@ params { cellpose_preprocessing_steps = '' cellpose_preprocessing_params_file = '' - + cellpose_segmentation_cpus = 1 cellpose_segmentation_mem_gb = 0 @@ -77,7 +77,7 @@ process { containerOptions = "${params.runtime_opts} ${params.cellpose_worker_runtime_opts}" clusterOptions = "${params.lsf_opts} ${params.cellpose_worker_lsf_opts}" ext.args = [ - "--nthreads 1", + "--nthreads ${params.cellpose_dask_worker_cpus}", ].join(' ') } diff --git a/examples/tiny_sample_params.json b/examples/tiny_sample_params.json index 29f01953..2c263484 100644 --- a/examples/tiny_sample_params.json +++ b/examples/tiny_sample_params.json @@ -7,13 +7,13 @@ "registration_subdir": "registration", "segmentation_subdir": "segmentation", - "skip_stitching": false, - "skip_registration": false, + "skip_stitching": true, + "skip_registration": true, "skip_global_align": false, "skip_local_align": false, "skip_deformations": false, "skip_inverse": false, - "skip_multiscale": false, + "skip_multiscale": true, "skip_segmentation": false, "skip_spot_extraction": false, "skip_warp_spots": false, @@ -33,7 +33,8 @@ "cellpose_test": false, "cellpose_verbose": true, - "cellpose_dask_workers": 3, + "cellpose_dask_workers": 1, + "cellpose_dask_worker_cpus": 4, "cellpose_merge_with_iou_only": false, "cellpose_worker_runtime_opts": "--nv", diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index e2a98fd3..295c56d1 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -33,6 +33,7 @@ workflow WARP_SPOTS { def all_moving_rounds = transform | map { it[0] /* id */ } | toList() + | first() def spots = spot_extraction_results | map { @@ -62,12 +63,13 @@ workflow WARP_SPOTS { } | map { def (id, meta, spots_file) = it + log.debug "Extract only fixed spots from $it" def r = [ meta, spots_file, spots_file, // no warping for fixed spots ] - log.debug "Fixed spots: $id - $it -> $r" + log.debug "Fixed spots: $id: $r" r } From 8a00651276cde3fe7b076ca5c639071edca32437 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 16:16:35 -0400 Subject: [PATCH 24/78] changed how I get the fixed spots --- workflows/warp_spots.nf | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index 295c56d1..c75b5787 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -30,10 +30,12 @@ workflow WARP_SPOTS { [ id, reg_meta, inv_transform_output, inv_transform_name, inv_transform_subpath ] } - def all_moving_rounds = transform - | map { it[0] /* id */ } - | toList() - | first() + def registration_fix = registration_results + | map { + def (reg_meta) = it + def id = reg_meta.fix_id + [ id ] + } def spots = spot_extraction_results | map { @@ -55,12 +57,10 @@ workflow WARP_SPOTS { log.debug "Extracted spot candidate: $id: $r" r } + | unique(by: 0) // unique by id def fixed_spots = spots - | filter { - log.debug "Check if ${it[0]} is a fixed spot - should not be in ${all_moving_rounds}" - !(it[0] in all_moving_rounds) - } + | join(registration_fix, by: 0) | map { def (id, meta, spots_file) = it log.debug "Extract only fixed spots from $it" @@ -148,12 +148,13 @@ workflow WARP_SPOTS { | join(spot_extraction_results, by: 0) | map { def (meta, spots_file, warped_spots_file, image_container, image_dataset) = it + log.debug "add image info to spot results: $it" def r = [ meta, image_container, image_dataset, // include the image used for spot extraction in the results spots_file, warped_spots_file, ] - log.debug "All (warped and unwarped) spot results: $it -> $r" + log.debug "All (fixed and warped) spot results: $r" r } From 6a8e0bac4f73bc85f05fadcddb9293da71f94bcc Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 16:18:29 -0400 Subject: [PATCH 25/78] fixed unique call --- workflows/warp_spots.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index c75b5787..99bcfda0 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -57,7 +57,7 @@ workflow WARP_SPOTS { log.debug "Extracted spot candidate: $id: $r" r } - | unique(by: 0) // unique by id + | unique { it[0] } // unique by id def fixed_spots = spots | join(registration_fix, by: 0) From 17dab80a35d278c358c32bf7e7702aaea7e6e7ae Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 16:25:01 -0400 Subject: [PATCH 26/78] combine warped spots with segmentaton results --- workflows/easifish.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/easifish.nf b/workflows/easifish.nf index dff928e2..7efeca3d 100644 --- a/workflows/easifish.nf +++ b/workflows/easifish.nf @@ -170,7 +170,7 @@ workflow EASIFISH { final_spot_results.subscribe { log.debug "Final spot results: $it " } def spots_features_inputs = final_spot_results - | join(segmentation_results, by: 0) + | combine(segmentation_results, by: 0) def spots_features_results = SPOTS_FEATURES(spots_features_inputs) From 4d929fabd62effa29b2da5a344be7060b6ece81c Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 16:34:46 -0400 Subject: [PATCH 27/78] changed how results are combined --- workflows/easifish.nf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workflows/easifish.nf b/workflows/easifish.nf index 7efeca3d..d09cd822 100644 --- a/workflows/easifish.nf +++ b/workflows/easifish.nf @@ -170,7 +170,9 @@ workflow EASIFISH { final_spot_results.subscribe { log.debug "Final spot results: $it " } def spots_features_inputs = final_spot_results - | combine(segmentation_results, by: 0) + | combine(segmentation_results) + + spots_features_inputs.subscribe { log.debug "Spots for features input: $it " } def spots_features_results = SPOTS_FEATURES(spots_features_inputs) From ceaa214f20e90b33e57fd4f8e105b41fce3e9158 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 16:49:31 -0400 Subject: [PATCH 28/78] eliminate seg meta from combined results --- modules.json | 2 +- .../janelia/bigstream/transformcoords/main.nf | 5 +++-- workflows/easifish.nf | 17 +++++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/modules.json b/modules.json index 73393cb9..f4b8ddc0 100644 --- a/modules.json +++ b/modules.json @@ -27,7 +27,7 @@ }, "bigstream/transformcoords": { "branch": "main", - "git_sha": "8045802c52bf97c2f04ea1744348d06ba902cfb1", + "git_sha": "37dcd298f0e63b697fa18cdc15ffac36958e1150", "installed_by": ["modules"] }, "cellpose": { diff --git a/modules/janelia/bigstream/transformcoords/main.nf b/modules/janelia/bigstream/transformcoords/main.nf index aec1d111..acca9c8b 100644 --- a/modules/janelia/bigstream/transformcoords/main.nf +++ b/modules/janelia/bigstream/transformcoords/main.nf @@ -20,7 +20,7 @@ process BIGSTREAM_TRANSFORMCOORDS { val(bigstream_mem_in_gb) output: - tuple val(meta), path(input_coords), env(warped_coords), emit: results + tuple val(meta), env(full_input_coords), env(warped_coords), emit: results tuple val(meta), env(source_fullpath), val(source_image_subpath), emit: source when: @@ -54,11 +54,12 @@ process BIGSTREAM_TRANSFORMCOORDS { else source_fullpath= fi + full_input_coords=\$(readlink -e ${input_coords}) output_fullpath=\$(readlink ${output_dir}) mkdir -p \${output_fullpath} warped_coords="\${output_fullpath}/${warped_coords_name}" python /app/bigstream/scripts/main_apply_transform_coords.py \ - --input-coords ${input_coords} \ + --input-coords \${full_input_coords} \ --output-coords \${warped_coords} \ ${pixel_resolution_arg} \ ${downsampling_arg} \ diff --git a/workflows/easifish.nf b/workflows/easifish.nf index d09cd822..dce762ce 100644 --- a/workflows/easifish.nf +++ b/workflows/easifish.nf @@ -171,6 +171,23 @@ workflow EASIFISH { def spots_features_inputs = final_spot_results | combine(segmentation_results) + | map { + def (meta_spots, + spots_image_container, spots_dataset, + source_spots, warped_spots, + meta_seg, seg_input_image, seg_input_dataset, seg_labels) = it + log.debug "Combined spots and segmentation results: $it" + [ + meta_spots, + spots_image_container, + spots_dataset, + source_spots, + warped_spots, + seg_input_image, + seg_input_dataset, + seg_labels, + ] + } spots_features_inputs.subscribe { log.debug "Spots for features input: $it " } From e1bc1a9c6b7c43f537e86c15150755173c7f4bee Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 17:04:25 -0400 Subject: [PATCH 29/78] fixed the pattern; display command --- modules/local/spots/sizes/main.nf | 17 +++++++++++------ workflows/spots_features.nf | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/modules/local/spots/sizes/main.nf b/modules/local/spots/sizes/main.nf index 62e78054..a8771e02 100644 --- a/modules/local/spots/sizes/main.nf +++ b/modules/local/spots/sizes/main.nf @@ -38,13 +38,18 @@ process SPOTS_SIZES { output_csv_file="\${full_output_dir}/count.csv" - python /opt/scripts/spots-utils/labeled-spots-sizes.py \ - --image-container \${full_input_image_path} \ - --image-subpath ${input_dataset} \ - --labels-container \${full_labels_path} \ - --labels-subpath ${labels_dataset} \ - --spots-pattern \"\${full_spots_input_dir}/${input_pattern}\" \ + CMD=( + python + /opt/scripts/spots-utils/labeled-spots-sizes.py + --image-container \${full_input_image_path} + --image-subpath ${input_dataset} + --labels-container \${full_labels_path} + --labels-subpath ${labels_dataset} + --spots-pattern \"\${full_spots_input_dir}/${input_pattern}\" --output \${output_csv_file} + ) + echo "CMD: \${CMD[@]}" + (exec "\${CMD[@]}") cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index f517822f..5558a95e 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -28,7 +28,7 @@ workflow SPOTS_FEATURES { seg_labels, seg_input_dataset, spots_input_dir, - 'spots-*-coord.csv', + '*coord.csv', spots_sizes_output_dir, ] log.debug "Spots sizes input: $r" From 0f73426dbbb9d5983b198a7f402e03289c0b7703 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 17:45:26 -0400 Subject: [PATCH 30/78] pass different output dir for spot features --- conf/spots-features-modules.config | 5 +++-- workflows/easifish.nf | 5 ++++- workflows/spots_features.nf | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/conf/spots-features-modules.config b/conf/spots-features-modules.config index f9bb6b23..fa66fa39 100644 --- a/conf/spots-features-modules.config +++ b/conf/spots-features-modules.config @@ -1,4 +1,5 @@ params { - spots_sizes_cores = 1 - spots_sizes_mem_gb = 1 + spots_sizes_cores = 1 + spots_sizes_mem_gb = 1 + spots_features_subdir = 'spot-features' } diff --git a/workflows/easifish.nf b/workflows/easifish.nf index dce762ce..fc99c082 100644 --- a/workflows/easifish.nf +++ b/workflows/easifish.nf @@ -191,7 +191,10 @@ workflow EASIFISH { spots_features_inputs.subscribe { log.debug "Spots for features input: $it " } - def spots_features_results = SPOTS_FEATURES(spots_features_inputs) + def spots_features_results = SPOTS_FEATURES( + spots_features_inputs, + outdir, + ) spots_features_results.subscribe { log.debug "Spots features result: $it " } } diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 5558a95e..0e0127f7 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -4,6 +4,7 @@ include { SPOTS_SIZES } from '../modules/local/spots/sizes/main' workflow SPOTS_FEATURES { take: ch_spots_inputs // channel: [ meta, input_image, input_dataset, spots, warped_spots, seg_input_image, seg_input_dataset, seg_labels ] + outdir main: @@ -20,7 +21,8 @@ workflow SPOTS_FEATURES { log.debug "Prepare spots sizes input from $it" def spots_input_dir = file(warped_spots_file).parent - def spots_sizes_output_dir = spots_input_dir + def spots_sizes_output_dir = file("${outputdir}/${params.spots_features_subdir}/${meta.id}") + def r = [ meta, image, From cce075f3398a04942077ebc1b387ed1be283bbd9 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 17:49:50 -0400 Subject: [PATCH 31/78] fixed var --- workflows/spots_features.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 0e0127f7..526ba6ef 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -4,7 +4,7 @@ include { SPOTS_SIZES } from '../modules/local/spots/sizes/main' workflow SPOTS_FEATURES { take: ch_spots_inputs // channel: [ meta, input_image, input_dataset, spots, warped_spots, seg_input_image, seg_input_dataset, seg_labels ] - outdir + outputdir main: From d608f87d0a10c4a401e57e54fbe00665ff54b12d Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 18:11:58 -0400 Subject: [PATCH 32/78] adjust input image scale --- workflows/spots_features.nf | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 526ba6ef..1bd304a1 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -23,10 +23,18 @@ workflow SPOTS_FEATURES { def spots_input_dir = file(warped_spots_file).parent def spots_sizes_output_dir = file("${outputdir}/${params.spots_features_subdir}/${meta.id}") + def image_dataset_comps = image_dataset.split('/') + def labels_dataset_comps = seg_input_dataset.split('/') + def = + if (labels_dataset_comps && labels_dataset_comps[-1] != image_dataset_comps[-1]) { + log.debug "Use labels scale for input image: ${labels_dataset_comps[-1]}" + image_dataset_comps[-1] = labels_dataset_comps[-1] + } + def r = [ meta, image, - image_dataset, + image_dataset_comps.join('/'), seg_labels, seg_input_dataset, spots_input_dir, From 4078873c5fcff90d8cbfa1047cd16fad44909b49 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 22 Apr 2025 18:13:41 -0400 Subject: [PATCH 33/78] removed bad def --- workflows/spots_features.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 1bd304a1..f8774897 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -25,7 +25,6 @@ workflow SPOTS_FEATURES { def image_dataset_comps = image_dataset.split('/') def labels_dataset_comps = seg_input_dataset.split('/') - def = if (labels_dataset_comps && labels_dataset_comps[-1] != image_dataset_comps[-1]) { log.debug "Use labels scale for input image: ${labels_dataset_comps[-1]}" image_dataset_comps[-1] = labels_dataset_comps[-1] From 9d25022395f7658236a8b56af38663ca49956483 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 23 Apr 2025 16:45:30 -0400 Subject: [PATCH 34/78] changed meta for warp spots; changed how input is created for spots sizes; region props inputs --- conf/spots-features-modules.config | 9 +- workflows/easifish.nf | 55 ++++------ workflows/spots_features.nf | 164 ++++++++++++++++++++++++----- workflows/warp_spots.nf | 48 +++++---- 4 files changed, 189 insertions(+), 87 deletions(-) diff --git a/conf/spots-features-modules.config b/conf/spots-features-modules.config index fa66fa39..8044cd71 100644 --- a/conf/spots-features-modules.config +++ b/conf/spots-features-modules.config @@ -1,5 +1,8 @@ params { - spots_sizes_cores = 1 - spots_sizes_mem_gb = 1 - spots_features_subdir = 'spot-features' + spots_sizes_cores = 1 + spots_sizes_mem_gb = 1 + spots_sizes_subdir = 'cell-sizes' + cells_regionprops_subdir = 'cell-props' + dapi_channel = '' + bleeding_channel = '' } diff --git a/workflows/easifish.nf b/workflows/easifish.nf index fc99c082..b78a2509 100644 --- a/workflows/easifish.nf +++ b/workflows/easifish.nf @@ -19,13 +19,14 @@ include { */ -include { INPUT_CHECK } from '../subworkflows/local/input_check' -include { STITCHING } from '../subworkflows/local/stitching' -include { REGISTRATION } from './registration' -include { SEGMENTATION } from './segmentation' -include { SPOT_EXTRACTION } from './spot_extraction' -include { WARP_SPOTS } from './warp_spots' -include { SPOTS_FEATURES } from './spots_features' +include { INPUT_CHECK } from '../subworkflows/local/input_check' +include { STITCHING } from '../subworkflows/local/stitching' +include { REGISTRATION } from './registration' +include { SEGMENTATION } from './segmentation' +include { SPOT_EXTRACTION } from './spot_extraction' +include { WARP_SPOTS } from './warp_spots' +include { MEASURE_SPOTS } from './spots_features' +include { EXTRACT_CELL_REGIONPROPS } from './spots_features' def validate_params() { @@ -161,44 +162,32 @@ workflow EASIFISH { spot_extraction_results.subscribe { log.debug "Spot extraction result: $it " } - def final_spot_results = WARP_SPOTS( + def warped_spots_results = WARP_SPOTS( registration_results, spot_extraction_results, outdir, ) // final_spot_results includes spots for fixed and warped spots from the moving rounds - final_spot_results.subscribe { log.debug "Final spot results: $it " } - - def spots_features_inputs = final_spot_results - | combine(segmentation_results) - | map { - def (meta_spots, - spots_image_container, spots_dataset, - source_spots, warped_spots, - meta_seg, seg_input_image, seg_input_dataset, seg_labels) = it - log.debug "Combined spots and segmentation results: $it" - [ - meta_spots, - spots_image_container, - spots_dataset, - source_spots, - warped_spots, - seg_input_image, - seg_input_dataset, - seg_labels, - ] - } + warped_spots_results.subscribe { log.debug "Warped spots results: $it " } + + def sized_spots_results = MEASURE_SPOTS( + warped_spots_results, + segmentation_results, + outdir, + ) - spots_features_inputs.subscribe { log.debug "Spots for features input: $it " } + sized_spots_results.subscribe { log.debug "Spots sizes: $it " } - def spots_features_results = SPOTS_FEATURES( - spots_features_inputs, + def cell_regionprops = EXTRACT_CELL_REGIONPROPS( + registration_results, + segmentation_results, outdir, ) - spots_features_results.subscribe { log.debug "Spots features result: $it " } + cell_regionprops.subscribe { log.debug "Cell regionprops: $it " } } + workflow.onComplete { if (params.email || params.email_on_fail) { NfcoreTemplate.email(workflow, params, summary_params, projectDir, log) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index f8774897..d19df409 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -1,42 +1,45 @@ include { SPOTS_REGIONPROPS } from '../modules/local/spots/regionprops/main' include { SPOTS_SIZES } from '../modules/local/spots/sizes/main' -workflow SPOTS_FEATURES { +workflow MEASURE_SPOTS { take: - ch_spots_inputs // channel: [ meta, input_image, input_dataset, spots, warped_spots, seg_input_image, seg_input_dataset, seg_labels ] + ch_spots // channel: [ meta_spots, meta_reg, spots_input_image, spots_input_dataset, spots, warped_spots ] + ch_segmentation // channel: [ meta, seg_input_image, seg_input_dataset, seg_labels ] outputdir main: + def ch_segmentation_with_id = ch_segmentation + | map { + def (meta, seg_input_image, seg_input_dataset, seg_labels) = it + def id = meta.id + [ id, meta, seg_input_image, seg_input_dataset, seg_labels ] + } - def spots_sizes_input = ch_spots_inputs + def spots_sizes_input = ch_spots + | map { + def (meta_spots, meta_reg, spots_input_image, spots_input_dataset, source_spots, final_spots) = it + def id = meta_reg.fix_id + [ id, meta_spots, spots_input_image, spots_input_dataset, source_spots, final_spots ] + } + | combine(ch_segmentation, by: 0) | map { - def (meta, - image, - image_dataset, - spots_file, - warped_spots_file, - seg_input_image, - seg_input_dataset, - seg_labels) = it - log.debug "Prepare spots sizes input from $it" - - def spots_input_dir = file(warped_spots_file).parent - def spots_sizes_output_dir = file("${outputdir}/${params.spots_features_subdir}/${meta.id}") - - def image_dataset_comps = image_dataset.split('/') - def labels_dataset_comps = seg_input_dataset.split('/') - if (labels_dataset_comps && labels_dataset_comps[-1] != image_dataset_comps[-1]) { - log.debug "Use labels scale for input image: ${labels_dataset_comps[-1]}" - image_dataset_comps[-1] = labels_dataset_comps[-1] - } + def (id, + meta_spots, + spots_image_container, spots_dataset, + source_spots, final_spots, + meta_seg, seg_input_image, seg_input_dataset, seg_labels) = it + log.debug "Combined spots and segmentation results: $it" + + def spots_dir = file(warped_spots).parent + def spots_sizes_output_dir = file("${outputdir}/${params.spots_sizes_subdir}/${meta_spots.id}") + + def spots_dataset = sync_image_scale_with_labels_scale(spots_dataset, seg_input_dataset) def r = [ - meta, - image, - image_dataset_comps.join('/'), - seg_labels, - seg_input_dataset, - spots_input_dir, + meta_spots, + spots_image_container, spots_dataset_comps, + seg_labels, seg_input_dataset, + spots_dir, '*coord.csv', spots_sizes_output_dir, ] @@ -53,3 +56,108 @@ workflow SPOTS_FEATURES { emit: done = spots_sizes_outputs.results } + +workflow EXTRACT_CELL_REGIONPROPS { + take: + ch_registration + ch_segmentation + outdir, + + main: + def registered_images = registration_results + | map { + def (reg_meta, + fix, fix_subpath, + mov, mov_subpath, + warped, warped_subpath, + transform_output, + transform_name, transform_subpath, + inv_transform_output, + inv_transform_name, inv_transform_subpath) = it + [ reg_meta.fix_id, warped, warped_subpath ] + } + + def fixed_images = registration_results + | map { + def (reg_meta, + fix, fix_subpath, + mov, mov_subpath, + warped, warped_subpath, + transform_output, + transform_name, transform_subpath, + inv_transform_output, + inv_transform_name, inv_transform_subpath) = it + [ reg_meta.fix_id, fix, fix_subpath ] + } + | unique { it[0] } // unique by id + + def ch_segmentation_with_id = ch_segmentation + | map { + def (meta, seg_input_image, seg_input_dataset, seg_labels) = it + def id = meta.id + [ id, meta, seg_input_image, seg_input_dataset, seg_labels ] + } + + def regionprops_inputs = fixed_images + | concat(registered_images) + | combine(ch_segmentation_with_id, by: 0) + | map { + def (id, + image_container, image_dataset, + meta, + seg_input_image, seg_input_dataset, seg_labels) = it + log.debug "Combined cell images with segmentation: $it" + + + def regionprops_output_dir = file("${outdir}/${params.cells_regionprops_subdir}/${meta.id}") + def image_dataset = sync_image_scale_with_labels_scale(image_dataset, seg_input_dataset) + + def dapi_dataset = params.dapi_channel + ? change_dataset_channel(image_dataset, params.dapi_channel) + : '' + def bleeding_dataset = params.bleeding_channel + ? change_dataset_channel(image_dataset, params.bleeding_channel) + : '' + + def dataset_ch = get_dataset_channel(image_dataset) + def r = [ + meta, + image_container, image_dataset, + seg_labels, seg_input_dataset, + dapi_dataset, + bleeding_dataset, + regionprops_output_dir, + "${meta.id}-${dataset_ch}-regionprops.csv", + ] + log.debug "Cell regionprops input: $r" + r + } + + emit: + done = regionprops_inputs + +} + + +def sync_image_scale_with_labels_scale(image_dataset, labels_dataset) { + def image_dataset_comps = image_dataset.split('/') + def labels_dataset_comps = labels_dataset.split('/') + if (labels_dataset_comps && labels_dataset_comps[-1] != image_dataset[-1]) { + log.debug "Use labels scale for input image: ${labels_dataset_comps[-1]}" + image_dataset_comps[-1] = labels_dataset_comps[-1] + } + return image_dataset_comps.join('/') +} + +def change_dataset_channel(image_dataset, channel) { + def image_dataset_comps = image_dataset.split('/') + if (image_dataset_comps) { + image_dataset_comps[-2] = channel + } + return image_dataset_comps.join('/') +} + +def get_dataset_channel(image_dataset) { + def image_dataset_comps = image_dataset.split('/') + return image_dataset_comps ? image_dataset_comps[-2] : '' +} diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index 99bcfda0..3897b365 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -16,34 +16,35 @@ workflow WARP_SPOTS { def transform = registration_results | map { def ( - reg_meta, + meta_reg, fix, fix_subpath, mov, mov_subpath, warped, warped_subpath, transform_output, transform_name, transform_subpath, - inv_transform_output, + inv_transform_path, inv_transform_name, inv_transform_subpath ) = it - def id = reg_meta.mov_id - [ id, reg_meta, inv_transform_output, inv_transform_name, inv_transform_subpath ] + def id = meta_reg.mov_id + [ id, meta_reg, inv_transform_path, inv_transform_name, inv_transform_subpath ] } def registration_fix = registration_results | map { - def (reg_meta) = it - def id = reg_meta.fix_id - [ id ] + def (meta_reg) = it + def id = meta_reg.fix_id + [ id, meta_reg ] } + | unique { it[0] } // unique by id def spots = spot_extraction_results | map { log.debug "Extracted spot: $it" def ( meta, - image_container, - image_dataset, + spots_image_container, + spots_dataset, spots_file ) = it @@ -52,20 +53,19 @@ workflow WARP_SPOTS { id, meta, spots_file, - image_container, image_dataset, + spots_image_container, spots_dataset, ] log.debug "Extracted spot candidate: $id: $r" r } - | unique { it[0] } // unique by id - def fixed_spots = spots - | join(registration_fix, by: 0) + def fixed_spots = registration_fix + | join(spots, by: 0) | map { - def (id, meta, spots_file) = it + def (id, meta_reg, meta_spots, spots_file) = it log.debug "Extract only fixed spots from $it" def r = [ - meta, + [ meta_spots:meta_spots, meta_reg:meta_reg ], spots_file, spots_file, // no warping for fixed spots ] @@ -78,23 +78,24 @@ workflow WARP_SPOTS { | map { def ( id, - meta, + meta_spots, spots_file, - image_container, image_dataset, - reg_meta, - inv_transform_output, + spots_image_container, spots_dataset, + meta_reg, + inv_transform_path, inv_transform_name, inv_transform_subpath ) = it def warped_spots_output_dir = file("${outdir}/${params.warped_spots_subdir}/${id}") + def meta = [ meta_spots:meta_spots, meta_reg:meta_reg ] def spots_filename = file(spots_file).name [ [ meta, spots_file, warped_spots_output_dir, "warped-${spots_filename}", ], [ - image_container, image_dataset, + spots_image_container, spots_dataset, ], [ '' /* resolution */, '' /* downsampling factors */ @@ -150,8 +151,9 @@ workflow WARP_SPOTS { def (meta, spots_file, warped_spots_file, image_container, image_dataset) = it log.debug "add image info to spot results: $it" def r = [ - meta, - image_container, image_dataset, // include the image used for spot extraction in the results + meta.meta_spots, + meta.meta_reg, + spots_image_container, spots_dataset, // include the image used for spot extraction in the results spots_file, warped_spots_file, ] log.debug "All (fixed and warped) spot results: $r" @@ -159,5 +161,5 @@ workflow WARP_SPOTS { } emit: - done = final_spot_results // [ meta, image_container, image_dataset, spots_file, warped_spots_file ] + done = final_spot_results // [ meta_spots, meta_reg, image_container, image_dataset, spots_file, warped_spots_file ] } From 840f7d049755accbb6bdefab1590146253b3454f Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 23 Apr 2025 16:48:05 -0400 Subject: [PATCH 35/78] fixed syntax error --- workflows/spots_features.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index d19df409..8677454f 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -61,7 +61,7 @@ workflow EXTRACT_CELL_REGIONPROPS { take: ch_registration ch_segmentation - outdir, + outdir main: def registered_images = registration_results From 92070934a9b250200cfb45246e4c35fc6cb7d76c Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 23 Apr 2025 16:50:59 -0400 Subject: [PATCH 36/78] fixed vars --- workflows/spots_features.nf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 8677454f..b38f92b1 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -33,11 +33,11 @@ workflow MEASURE_SPOTS { def spots_dir = file(warped_spots).parent def spots_sizes_output_dir = file("${outputdir}/${params.spots_sizes_subdir}/${meta_spots.id}") - def spots_dataset = sync_image_scale_with_labels_scale(spots_dataset, seg_input_dataset) + adjusted_spots_dataset = sync_image_scale_with_labels_scale(spots_dataset, seg_input_dataset) def r = [ meta_spots, - spots_image_container, spots_dataset_comps, + spots_image_container, adjusted_spots_dataset, seg_labels, seg_input_dataset, spots_dir, '*coord.csv', @@ -108,21 +108,21 @@ workflow EXTRACT_CELL_REGIONPROPS { seg_input_image, seg_input_dataset, seg_labels) = it log.debug "Combined cell images with segmentation: $it" - def regionprops_output_dir = file("${outdir}/${params.cells_regionprops_subdir}/${meta.id}") - def image_dataset = sync_image_scale_with_labels_scale(image_dataset, seg_input_dataset) + def adjusted_image_dataset = sync_image_scale_with_labels_scale(image_dataset, seg_input_dataset) def dapi_dataset = params.dapi_channel - ? change_dataset_channel(image_dataset, params.dapi_channel) + ? change_dataset_channel(adjusted_image_dataset, params.dapi_channel) : '' def bleeding_dataset = params.bleeding_channel - ? change_dataset_channel(image_dataset, params.bleeding_channel) + ? change_dataset_channel(adjusted_image_dataset, params.bleeding_channel) : '' - def dataset_ch = get_dataset_channel(image_dataset) + def dataset_ch = get_dataset_channel(adjusted_image_dataset) + def r = [ meta, - image_container, image_dataset, + image_container, adjusted_image_dataset, seg_labels, seg_input_dataset, dapi_dataset, bleeding_dataset, From 57a8226b0d60164c5e08a0e55f60e9a6c2870d43 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 23 Apr 2025 16:52:01 -0400 Subject: [PATCH 37/78] more vars fixed --- workflows/spots_features.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index b38f92b1..6947466a 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -64,7 +64,7 @@ workflow EXTRACT_CELL_REGIONPROPS { outdir main: - def registered_images = registration_results + def registered_images = ch_registration | map { def (reg_meta, fix, fix_subpath, @@ -77,7 +77,7 @@ workflow EXTRACT_CELL_REGIONPROPS { [ reg_meta.fix_id, warped, warped_subpath ] } - def fixed_images = registration_results + def fixed_images = ch_registration | map { def (reg_meta, fix, fix_subpath, From fd6d46c032296f5072941b012eda844f4f89a21b Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 23 Apr 2025 17:06:17 -0400 Subject: [PATCH 38/78] fixed inv transform path --- workflows/warp_spots.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index 3897b365..5e859bdf 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -103,7 +103,7 @@ workflow WARP_SPOTS { [], // affine transform [ - "${inv_transform_output}/${inv_transform_name}", + "${inv_transform_path}/${inv_transform_name}", inv_transform_subpath, ], [ From 9c2a752a621450d42a4289e52ccadd215f805e58 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 23 Apr 2025 17:23:05 -0400 Subject: [PATCH 39/78] fixed join for final spots --- workflows/warp_spots.nf | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index 5e859bdf..29600f2c 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -146,15 +146,26 @@ workflow WARP_SPOTS { } def final_spot_results = fixed_spots.concat(spots_warp_results) - | join(spot_extraction_results, by: 0) | map { - def (meta, spots_file, warped_spots_file, image_container, image_dataset) = it - log.debug "add image info to spot results: $it" + def (meta, source_spots, final_spots) = it def r = [ meta.meta_spots, meta.meta_reg, + source_spots, final_spots, + ] + log.debug "All (fixed and warped) spot results: $r" + r + } + | join(spot_extraction_results, by: 0) + | map { + def (meta_spots, meta_reg, source_spots, final_spots, + image_container, image_dataset) = it + log.debug "Add source spots image to spot results: $it" + def r = [ + meta_spots, + meta_reg, spots_image_container, spots_dataset, // include the image used for spot extraction in the results - spots_file, warped_spots_file, + source_spots, final_spots, ] log.debug "All (fixed and warped) spot results: $r" r From 6c0b2939a3c32d8c22b4af34287415c75209c3ad Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 23 Apr 2025 17:28:20 -0400 Subject: [PATCH 40/78] fixed vars --- workflows/warp_spots.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index 29600f2c..12addde1 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -159,7 +159,7 @@ workflow WARP_SPOTS { | join(spot_extraction_results, by: 0) | map { def (meta_spots, meta_reg, source_spots, final_spots, - image_container, image_dataset) = it + spots_image_container, spots_dataset) = it log.debug "Add source spots image to spot results: $it" def r = [ meta_spots, From 48d70d846d1641a289fdad01afb789a258807044 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 23 Apr 2025 17:34:39 -0400 Subject: [PATCH 41/78] fixed combine for measure spots --- workflows/spots_features.nf | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 6947466a..a23ebf3a 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -12,16 +12,20 @@ workflow MEASURE_SPOTS { | map { def (meta, seg_input_image, seg_input_dataset, seg_labels) = it def id = meta.id - [ id, meta, seg_input_image, seg_input_dataset, seg_labels ] + def r = [ id, meta, seg_input_image, seg_input_dataset, seg_labels ] + log.debug "Segmentation data for spots sizes: $r" + r } def spots_sizes_input = ch_spots | map { def (meta_spots, meta_reg, spots_input_image, spots_input_dataset, source_spots, final_spots) = it def id = meta_reg.fix_id - [ id, meta_spots, spots_input_image, spots_input_dataset, source_spots, final_spots ] + def r = [ id, meta_spots, spots_input_image, spots_input_dataset, source_spots, final_spots ] + log.debug "Spots data for spots sizes: $r" + r } - | combine(ch_segmentation, by: 0) + | combine(ch_segmentation_with_id, by: 0) | map { def (id, meta_spots, From 1793bf43d1094c5ae1c2917cf02cb28ead57150b Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 23 Apr 2025 17:40:29 -0400 Subject: [PATCH 42/78] call spots regionprops --- conf/spots-features-modules.config | 4 +++- examples/tiny_sample_params.json | 3 +++ workflows/spots_features.nf | 9 +++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/conf/spots-features-modules.config b/conf/spots-features-modules.config index 8044cd71..4ce67539 100644 --- a/conf/spots-features-modules.config +++ b/conf/spots-features-modules.config @@ -1,6 +1,8 @@ params { spots_sizes_cores = 1 - spots_sizes_mem_gb = 1 + spots_sizes_mem_gb = 2 + cells_regionprops_cores = 1 + cells_regionprops_mem_gb = 2 spots_sizes_subdir = 'cell-sizes' cells_regionprops_subdir = 'cell-props' dapi_channel = '' diff --git a/examples/tiny_sample_params.json b/examples/tiny_sample_params.json index 2c263484..fc089c15 100644 --- a/examples/tiny_sample_params.json +++ b/examples/tiny_sample_params.json @@ -51,6 +51,9 @@ "multiscale_spark_worker_cores": 20, "multiscale_spark_gb_per_core": 5, + "dapi_channel": "c1", + "bleeding_channel": "c0", + "lsf_opts": "-P scicompsoft", "cellpose_worker_lsf_opts": "-q gpu_l4 -gpu 'num=1'" } diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index a23ebf3a..1f324940 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -137,9 +137,14 @@ workflow EXTRACT_CELL_REGIONPROPS { r } - emit: - done = regionprops_inputs + SPOTS_REGIONPROPS( + regionprops_inputs, + params.cells_regionprops_cores, + params.cells_regionprops_mem_gb, + ) + emit: + done = SPOTS_REGIONPROPS.out.results } From 5a6a40104812c4159bc589172077bf0cc9f5261e Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 23 Apr 2025 17:44:26 -0400 Subject: [PATCH 43/78] var fix --- workflows/spots_features.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 1f324940..dd6ae07e 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -34,7 +34,7 @@ workflow MEASURE_SPOTS { meta_seg, seg_input_image, seg_input_dataset, seg_labels) = it log.debug "Combined spots and segmentation results: $it" - def spots_dir = file(warped_spots).parent + def spots_dir = file(final_spots).parent def spots_sizes_output_dir = file("${outputdir}/${params.spots_sizes_subdir}/${meta_spots.id}") adjusted_spots_dataset = sync_image_scale_with_labels_scale(spots_dataset, seg_input_dataset) From 49e39eddc5448433270a154b017e40da8cdc0c90 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 23 Apr 2025 18:12:17 -0400 Subject: [PATCH 44/78] fixed unbound var --- modules/local/spots/regionprops/main.nf | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/modules/local/spots/regionprops/main.nf b/modules/local/spots/regionprops/main.nf index 2be49ede..9e49b4a0 100644 --- a/modules/local/spots/regionprops/main.nf +++ b/modules/local/spots/regionprops/main.nf @@ -39,14 +39,19 @@ process SPOTS_REGIONPROPS { output_csv_file="\${full_output_dir}/${output_name}" - python /opt/scripts/spots-utils/labeled-spots-regionprops.py \ - --image-container \${full_input_container_path} \ - --image-subpath ${input_dataset} \ - --labels-container \${full_labels_path} \ - --labels-subpath ${labels_dataset} \ - ${dapi_dataset_arg} \ - ${bleed_dataset_arg} \ + CMD=( + python + /opt/scripts/spots-utils/labeled-cells-regionprops.py + --image-container \${full_input_image_path} + --image-subpath ${input_dataset} + --labels-container \${full_labels_path} + --labels-subpath ${labels_dataset} + ${dapi_dataset_arg} + ${bleed_dataset_arg} --output \${output_csv_file} + ) + echo "CMD: \${CMD[@]}" + (exec "\${CMD[@]}") cat <<-END_VERSIONS > versions.yml "${task.process}": From f5715965adfcb2e112f62eafa0e2df055e78c440 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Mon, 28 Apr 2025 15:08:55 -0400 Subject: [PATCH 45/78] re-enable all steps --- examples/medium_sample_params.json | 35 +++++++++++++++++++++++++----- examples/tiny_sample_params.json | 7 +++--- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/examples/medium_sample_params.json b/examples/medium_sample_params.json index 9bdb683f..d59a7a8a 100644 --- a/examples/medium_sample_params.json +++ b/examples/medium_sample_params.json @@ -3,7 +3,9 @@ "input": "examples/local_samplesheets/medium_samplesheet.csv", "indir": "/nrs/multifish/Pipeline/Examples/subset", "outdir": "/nrs/scicompsoft/goinac/multifish/medium/results", + "registration_subdir": "registration", + "segmentation_subdir": "segmentation", "skip_stitching": false, "skip_registration": false, @@ -12,22 +14,45 @@ "skip_deformations": false, "skip_inverse": false, "skip_multiscale": false, + "skip_segmentation": false, + "skip_spot_extraction": false, + "skip_warp_spots": false, "flatfield_correction": false, "darkfieldfile": "", "flatfieldfile": "", + "registration_fix_id": "t1", - "stitching_channel": "c2", - "reg_ch": "c2", + "stitching_channel": "c3", + "reg_ch": "c3", "global_steps": "ransac,affine", "local_steps": "ransac,affine,deform", "global_scale": "s3", "local_scale": "s2", "warped_scales": "s0", + + "cellpose_test": false, + "cellpose_verbose": true, + "cellpose_dask_workers": 1, + "cellpose_dask_worker_cpus": 4, + "cellpose_worker_runtime_opts": "--nv", + + "spark_workers": 5, "min_spark_workers": 2, "spark_gb_per_core": 10, - "local_align_workers": 35, + + "local_align_workers": 10, "local_align_worker_mem_gb": 10, - "bigstream_scheduler_port": 8786, - "bigstream_dashboard_port": 8787 + "bigstream_scheduler_port": 0, + "bigstream_dashboard_port": 0, + + "multiscale_spark_workers": 1, + "multiscale_spark_worker_cores": 20, + "multiscale_spark_gb_per_core": 5, + + "dapi_channel": "c1", + "bleeding_channel": "c0", + + "lsf_opts": "-P scicompsoft", + "cellpose_worker_lsf_opts": "-q gpu_l4 -gpu 'num=1'" } diff --git a/examples/tiny_sample_params.json b/examples/tiny_sample_params.json index fc089c15..243bdfe3 100644 --- a/examples/tiny_sample_params.json +++ b/examples/tiny_sample_params.json @@ -7,13 +7,13 @@ "registration_subdir": "registration", "segmentation_subdir": "segmentation", - "skip_stitching": true, - "skip_registration": true, + "skip_stitching": false, + "skip_registration": false, "skip_global_align": false, "skip_local_align": false, "skip_deformations": false, "skip_inverse": false, - "skip_multiscale": true, + "skip_multiscale": false, "skip_segmentation": false, "skip_spot_extraction": false, "skip_warp_spots": false, @@ -35,7 +35,6 @@ "cellpose_verbose": true, "cellpose_dask_workers": 1, "cellpose_dask_worker_cpus": 4, - "cellpose_merge_with_iou_only": false, "cellpose_worker_runtime_opts": "--nv", "spark_workers": 5, From 7774d40a1bdeebf0243230830dba441e99239997 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 29 Apr 2025 10:19:44 -0400 Subject: [PATCH 46/78] try cellpose merge using iou only --- examples/medium_sample_params.json | 3 ++- examples/tiny_sample_params.json | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/medium_sample_params.json b/examples/medium_sample_params.json index d59a7a8a..18a33c6d 100644 --- a/examples/medium_sample_params.json +++ b/examples/medium_sample_params.json @@ -7,7 +7,7 @@ "registration_subdir": "registration", "segmentation_subdir": "segmentation", - "skip_stitching": false, + "skip_stitching": true, "skip_registration": false, "skip_global_align": false, "skip_local_align": false, @@ -35,6 +35,7 @@ "cellpose_verbose": true, "cellpose_dask_workers": 1, "cellpose_dask_worker_cpus": 4, + "cellpose_merge_with_iou_only": true, "cellpose_worker_runtime_opts": "--nv", "spark_workers": 5, diff --git a/examples/tiny_sample_params.json b/examples/tiny_sample_params.json index 243bdfe3..a2463f52 100644 --- a/examples/tiny_sample_params.json +++ b/examples/tiny_sample_params.json @@ -35,6 +35,7 @@ "cellpose_verbose": true, "cellpose_dask_workers": 1, "cellpose_dask_worker_cpus": 4, + "cellpose_merge_with_iou_only": false, "cellpose_worker_runtime_opts": "--nv", "spark_workers": 5, From ddd5e4c4ab1a10633a85a7e1dfe01fd877eabfd5 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 29 Apr 2025 10:26:43 -0400 Subject: [PATCH 47/78] changed cellpose dask configuration --- examples/medium_sample_params.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/medium_sample_params.json b/examples/medium_sample_params.json index 18a33c6d..9289f94a 100644 --- a/examples/medium_sample_params.json +++ b/examples/medium_sample_params.json @@ -33,8 +33,8 @@ "cellpose_test": false, "cellpose_verbose": true, - "cellpose_dask_workers": 1, - "cellpose_dask_worker_cpus": 4, + "cellpose_dask_workers": 8, + "cellpose_dask_worker_cpus": 1, "cellpose_merge_with_iou_only": true, "cellpose_worker_runtime_opts": "--nv", From e0cb46a180da4bc68e19489980799a8a2799385e Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 29 Apr 2025 15:10:27 -0400 Subject: [PATCH 48/78] do not measure spots until all spots were extracted --- workflows/spots_features.nf | 50 ++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index dd6ae07e..33bf3451 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -25,30 +25,40 @@ workflow MEASURE_SPOTS { log.debug "Spots data for spots sizes: $r" r } - | combine(ch_segmentation_with_id, by: 0) - | map { + | groupTuple(by: 0) + | join(ch_segmentation_with_id, by: 0) + | flatMap { def (id, - meta_spots, - spots_image_container, spots_dataset, - source_spots, final_spots, - meta_seg, seg_input_image, seg_input_dataset, seg_labels) = it - log.debug "Combined spots and segmentation results: $it" + all_meta_spots, + all_spots_image_containers, all_spots_datasets, + all_source_spots, all_final_spots, + meta_seg, + seg_input_image, + seg_input_dataset, + seg_labels) = it def spots_dir = file(final_spots).parent - def spots_sizes_output_dir = file("${outputdir}/${params.spots_sizes_subdir}/${meta_spots.id}") - adjusted_spots_dataset = sync_image_scale_with_labels_scale(spots_dataset, seg_input_dataset) - - def r = [ - meta_spots, - spots_image_container, adjusted_spots_dataset, - seg_labels, seg_input_dataset, - spots_dir, - '*coord.csv', - spots_sizes_output_dir, - ] - log.debug "Spots sizes input: $r" - r + [all_meta_spots, + all_spots_image_containers, + all_spots_datasets, + all_source_spots, + all_final_spots].transpose().collect { + def (meta_spots, spots_image_container, spots_dataset, source_spots, final_spots) = it + def spots_sizes_output_dir = file("${outputdir}/${params.spots_sizes_subdir}/${meta_spots.id}") + def adjusted_spots_dataset = sync_image_scale_with_labels_scale(spots_dataset, seg_input_dataset) + + def r = [ + meta_spots, + spots_image_container, adjusted_spots_dataset, + seg_labels, seg_input_dataset, + spots_dir, + '*coord.csv', + spots_sizes_output_dir, + ] + log.debug "Spots sizes input: $r" + r + } } def spots_sizes_outputs = SPOTS_SIZES( From d41de73010326a995b01aaace902e011b342df64 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 30 Apr 2025 17:07:30 -0400 Subject: [PATCH 49/78] renamed post-rs-fish to verify when spot extraction is skipped --- workflows/spot_extraction.nf | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/workflows/spot_extraction.nf b/workflows/spot_extraction.nf index af992d31..f4e62c9c 100644 --- a/workflows/spot_extraction.nf +++ b/workflows/spot_extraction.nf @@ -4,11 +4,12 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { POST_RS_FISH } from '../modules/local/post_rs_fish/main' -include { RS_FISH } from '../modules/janelia/rs_fish/main' -include { SPARK_START } from '../subworkflows/janelia/spark_start/main' -include { SPARK_STOP } from '../subworkflows/janelia/spark_stop/main' -include { as_list } from './util_functions' +include { POST_RS_FISH } from '../modules/local/post_rs_fish/main' +include { RS_FISH } from '../modules/janelia/rs_fish/main' +include { RS_FISH as VERIFY_RS_FISH_SPOTS } from '../modules/janelia/rs_fish/main' +include { SPARK_START } from '../subworkflows/janelia/spark_start/main' +include { SPARK_STOP } from '../subworkflows/janelia/spark_stop/main' +include { as_list } from './util_functions' workflow SPOT_EXTRACTION { take: @@ -43,17 +44,19 @@ workflow SPOT_EXTRACTION { def (meta, spots_inout_dirs) = it def (input_img_dir, spots_output_dir) = spots_inout_dirs get_spot_subpaths(meta).collect { input_spot_subpath, spots_result_name -> - [ + def r = [ meta, input_img_dir, input_spot_subpath, "${spots_output_dir}/${spots_result_name}", ] + log.debug "Verify spots: $r" + r } } - | POST_RS_FISH + | VERIFY_RS_FISH_SPOTS - final_rsfish_results = POST_RS_FISH.out.results + final_rsfish_results = VERIFY_RS_FISH_SPOTS.out.results } else { spots_spark_input.subscribe { log.debug "Spot extraction spark input: $it" } @@ -144,8 +147,6 @@ def get_spot_extraction_input_volume(meta) { } def get_spot_subpaths(meta) { - def input_img_dir = get_spot_extraction_input_volume(meta) - if (!params.spot_subpaths && !params.spot_channels && !params.spot_scales) { return [ ['', ''], // empty subpath, empty resultnane - the input image container contains the array dataset @@ -166,7 +167,7 @@ def get_spot_subpaths(meta) { // all but the last channel which typically is DAPI def all_channels = as_list(params.channels) // this may throw an exception if the channel list is empty or a singleton - spot_channels = all_channels[0..-2] + spot_channels = all_channels[0..-2] // all but the last channel log.debug "Spot channels: $spot_channels (all from ${params.channels} except the last one)" } def spot_scales = as_list(params.spot_scales) @@ -175,10 +176,12 @@ def get_spot_subpaths(meta) { .collect { ch, scale -> // when channel and scale is used we also prepend the stitched dataset def dataset = "${ch}/${scale}" - [ + def r = [ "${meta.stitched_dataset}/${dataset}", "spots-rsfish-${ch}.csv" ] + log.debug "Spot dataset: $r" + r } } } From 00768829c215e0cf557471e60ed790764dd11407 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 30 Apr 2025 17:53:13 -0400 Subject: [PATCH 50/78] trying to debug the case when spots are skipped but they are also missing --- workflows/spot_extraction.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/spot_extraction.nf b/workflows/spot_extraction.nf index f4e62c9c..fa37c55c 100644 --- a/workflows/spot_extraction.nf +++ b/workflows/spot_extraction.nf @@ -39,7 +39,7 @@ workflow SPOT_EXTRACTION { log.info "Skipping spot extraction" // even if we skip spot extraction // we assume we have the csv file and we apply the post processing - spots_spark_input + def verify_spot_results = spots_spark_input | flatMap { def (meta, spots_inout_dirs) = it def (input_img_dir, spots_output_dir) = spots_inout_dirs @@ -56,7 +56,7 @@ workflow SPOT_EXTRACTION { } | VERIFY_RS_FISH_SPOTS - final_rsfish_results = VERIFY_RS_FISH_SPOTS.out.results + final_rsfish_results = verify_spot_results } else { spots_spark_input.subscribe { log.debug "Spot extraction spark input: $it" } From 0e2179f0481d5c60e77e47d7c641eeebebe40f6d Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 30 Apr 2025 18:05:28 -0400 Subject: [PATCH 51/78] changed how missing spots csv is handled --- modules/local/post_rs_fish/main.nf | 8 +++++++- workflows/spot_extraction.nf | 14 +++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/modules/local/post_rs_fish/main.nf b/modules/local/post_rs_fish/main.nf index 1ba3bc15..1ff7a65e 100644 --- a/modules/local/post_rs_fish/main.nf +++ b/modules/local/post_rs_fish/main.nf @@ -25,7 +25,13 @@ process POST_RS_FISH { """ # Create command line parameters full_input_path=\$(readlink -e ${input_path}) - voxel_spots_csv_file=\$(readlink -e ${voxel_spots_csv_file}) + voxel_spots_csv_file=\$(readlink ${voxel_spots_csv_file}) + + if [[ -f \${voxel_spots_csv_file} ]]; then + echo "Voxel spots CSV file not found: \${voxel_spots_csv_file}" + exit 0 + fi + voxel_spots_csv_dir=\$(dirname \${voxel_spots_csv_file}) coord_spots_csv_file=\${voxel_spots_csv_dir}/${spots_filename} diff --git a/workflows/spot_extraction.nf b/workflows/spot_extraction.nf index fa37c55c..9472c9c9 100644 --- a/workflows/spot_extraction.nf +++ b/workflows/spot_extraction.nf @@ -4,12 +4,12 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { POST_RS_FISH } from '../modules/local/post_rs_fish/main' -include { RS_FISH } from '../modules/janelia/rs_fish/main' -include { RS_FISH as VERIFY_RS_FISH_SPOTS } from '../modules/janelia/rs_fish/main' -include { SPARK_START } from '../subworkflows/janelia/spark_start/main' -include { SPARK_STOP } from '../subworkflows/janelia/spark_stop/main' -include { as_list } from './util_functions' +include { POST_RS_FISH } from '../modules/local/post_rs_fish/main' +include { POST_RS_FISH as VERIFY_RS_FISH_SPOTS } from '../modules/local/post_rs_fish/main' +include { RS_FISH } from '../modules/janelia/rs_fish/main' +include { SPARK_START } from '../subworkflows/janelia/spark_start/main' +include { SPARK_STOP } from '../subworkflows/janelia/spark_stop/main' +include { as_list } from './util_functions' workflow SPOT_EXTRACTION { take: @@ -56,7 +56,7 @@ workflow SPOT_EXTRACTION { } | VERIFY_RS_FISH_SPOTS - final_rsfish_results = verify_spot_results + final_rsfish_results = verify_spot_results.results } else { spots_spark_input.subscribe { log.debug "Spot extraction spark input: $it" } From de944fde8ed985ba05b398300fdd6cf0c2e52620 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 30 Apr 2025 18:11:11 -0400 Subject: [PATCH 52/78] set output to empty --- modules/local/post_rs_fish/main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/local/post_rs_fish/main.nf b/modules/local/post_rs_fish/main.nf index 1ff7a65e..c3876682 100644 --- a/modules/local/post_rs_fish/main.nf +++ b/modules/local/post_rs_fish/main.nf @@ -29,6 +29,7 @@ process POST_RS_FISH { if [[ -f \${voxel_spots_csv_file} ]]; then echo "Voxel spots CSV file not found: \${voxel_spots_csv_file}" + coord_spots_csv_file= exit 0 fi From 651d8cbf9fb19eb9b86a6a0a205d3cb2ca6a003f Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 30 Apr 2025 18:13:32 -0400 Subject: [PATCH 53/78] set version --- modules/local/post_rs_fish/main.nf | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/local/post_rs_fish/main.nf b/modules/local/post_rs_fish/main.nf index c3876682..d157ba95 100644 --- a/modules/local/post_rs_fish/main.nf +++ b/modules/local/post_rs_fish/main.nf @@ -30,17 +30,17 @@ process POST_RS_FISH { if [[ -f \${voxel_spots_csv_file} ]]; then echo "Voxel spots CSV file not found: \${voxel_spots_csv_file}" coord_spots_csv_file= - exit 0 + else + voxel_spots_csv_dir=\$(dirname \${voxel_spots_csv_file}) + coord_spots_csv_file=\${voxel_spots_csv_dir}/${spots_filename} + + python /opt/scripts/spots-utils/post-rs-fish.py \ + --image-container \${full_input_path} \ + --image-subpath ${input_dataset} \ + --input \${voxel_spots_csv_file} \ + --output \${coord_spots_csv_file} fi - voxel_spots_csv_dir=\$(dirname \${voxel_spots_csv_file}) - coord_spots_csv_file=\${voxel_spots_csv_dir}/${spots_filename} - - python /opt/scripts/spots-utils/post-rs-fish.py \ - --image-container \${full_input_path} \ - --image-subpath ${input_dataset} \ - --input \${voxel_spots_csv_file} \ - --output \${coord_spots_csv_file} cat <<-END_VERSIONS > versions.yml "${task.process}": From ecab029e9f6c0223b0e96cfd478bac9d7d2d7708 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 30 Apr 2025 18:18:49 -0400 Subject: [PATCH 54/78] changed vars --- modules/local/post_rs_fish/main.nf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/local/post_rs_fish/main.nf b/modules/local/post_rs_fish/main.nf index d157ba95..897205e3 100644 --- a/modules/local/post_rs_fish/main.nf +++ b/modules/local/post_rs_fish/main.nf @@ -25,19 +25,19 @@ process POST_RS_FISH { """ # Create command line parameters full_input_path=\$(readlink -e ${input_path}) - voxel_spots_csv_file=\$(readlink ${voxel_spots_csv_file}) + full_voxel_spots_csv_file=\$(readlink ${voxel_spots_csv_file}) - if [[ -f \${voxel_spots_csv_file} ]]; then - echo "Voxel spots CSV file not found: \${voxel_spots_csv_file}" + if [[ -f \${full_voxel_spots_csv_file} ]]; then + echo "Voxel spots CSV file not found: \${full_voxel_spots_csv_file}" coord_spots_csv_file= else - voxel_spots_csv_dir=\$(dirname \${voxel_spots_csv_file}) + voxel_spots_csv_dir=\$(dirname \${full_voxel_spots_csv_file}) coord_spots_csv_file=\${voxel_spots_csv_dir}/${spots_filename} python /opt/scripts/spots-utils/post-rs-fish.py \ --image-container \${full_input_path} \ --image-subpath ${input_dataset} \ - --input \${voxel_spots_csv_file} \ + --input \${full_voxel_spots_csv_file} \ --output \${coord_spots_csv_file} fi From ec087fcb866c93997ca3f539896ca77a312545c4 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 30 Apr 2025 18:41:10 -0400 Subject: [PATCH 55/78] check if spots file exist - bug fix --- modules/local/post_rs_fish/main.nf | 13 ++++++++----- workflows/warp_spots.nf | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/modules/local/post_rs_fish/main.nf b/modules/local/post_rs_fish/main.nf index 897205e3..49645e2d 100644 --- a/modules/local/post_rs_fish/main.nf +++ b/modules/local/post_rs_fish/main.nf @@ -6,7 +6,7 @@ process POST_RS_FISH { tuple val(meta), path(input_path), val(input_dataset), - val(voxel_spots_csv_file) + path(voxel_spots_csv_file) output: tuple val(meta), @@ -25,12 +25,12 @@ process POST_RS_FISH { """ # Create command line parameters full_input_path=\$(readlink -e ${input_path}) + echo "Input volume: \${full_input_path}" full_voxel_spots_csv_file=\$(readlink ${voxel_spots_csv_file}) + echo "Input spots CSV file: \${full_voxel_spots_csv_file}" - if [[ -f \${full_voxel_spots_csv_file} ]]; then - echo "Voxel spots CSV file not found: \${full_voxel_spots_csv_file}" - coord_spots_csv_file= - else + if [[ -f \${full_voxel_spots_csv_file} && ! -L \${full_voxel_spots_csv_file} ]]; then + echo "Found voxel spots CSV file: \${full_voxel_spots_csv_file}" voxel_spots_csv_dir=\$(dirname \${full_voxel_spots_csv_file}) coord_spots_csv_file=\${voxel_spots_csv_dir}/${spots_filename} @@ -39,6 +39,9 @@ process POST_RS_FISH { --image-subpath ${input_dataset} \ --input \${full_voxel_spots_csv_file} \ --output \${coord_spots_csv_file} + else + echo "Voxel spots CSV file not found: \${full_voxel_spots_csv_file}" + coord_spots_csv_file= fi diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index 12addde1..dff229bd 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -74,6 +74,7 @@ workflow WARP_SPOTS { } def spots_warp_input = spots + | filter { it[2] /* spots_file must be defined */} | join(transform, by: 0) | map { def ( From d8d1f97d514f737b64ab84730c78b73af934320e Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 30 Apr 2025 18:42:14 -0400 Subject: [PATCH 56/78] only test with -f --- modules/local/post_rs_fish/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/post_rs_fish/main.nf b/modules/local/post_rs_fish/main.nf index 49645e2d..37573831 100644 --- a/modules/local/post_rs_fish/main.nf +++ b/modules/local/post_rs_fish/main.nf @@ -29,7 +29,7 @@ process POST_RS_FISH { full_voxel_spots_csv_file=\$(readlink ${voxel_spots_csv_file}) echo "Input spots CSV file: \${full_voxel_spots_csv_file}" - if [[ -f \${full_voxel_spots_csv_file} && ! -L \${full_voxel_spots_csv_file} ]]; then + if [[ -f \${full_voxel_spots_csv_file} ]]; then echo "Found voxel spots CSV file: \${full_voxel_spots_csv_file}" voxel_spots_csv_dir=\$(dirname \${full_voxel_spots_csv_file}) coord_spots_csv_file=\${voxel_spots_csv_dir}/${spots_filename} From 29790540ba29412502eb25b7c7e41c883f21e2ac Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Thu, 1 May 2025 11:30:16 -0400 Subject: [PATCH 57/78] fixed var --- workflows/spots_features.nf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 33bf3451..3e7cee37 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -37,14 +37,13 @@ workflow MEASURE_SPOTS { seg_input_dataset, seg_labels) = it - def spots_dir = file(final_spots).parent - [all_meta_spots, all_spots_image_containers, all_spots_datasets, all_source_spots, all_final_spots].transpose().collect { def (meta_spots, spots_image_container, spots_dataset, source_spots, final_spots) = it + def spots_dir = file(final_spots).parent def spots_sizes_output_dir = file("${outputdir}/${params.spots_sizes_subdir}/${meta_spots.id}") def adjusted_spots_dataset = sync_image_scale_with_labels_scale(spots_dataset, seg_input_dataset) From 791825e70b043ff80c236518618db60ae71b8503 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Thu, 1 May 2025 15:29:03 -0400 Subject: [PATCH 58/78] filter empty spots --- workflows/spots_features.nf | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 3e7cee37..1fae352f 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -18,6 +18,14 @@ workflow MEASURE_SPOTS { } def spots_sizes_input = ch_spots + | filter { + def (meta_spots, meta_reg, spots_input_image, spots_input_dataset, source_spots, final_spots) = it + if (!final_spots) { + log.debug "Filter out spots input for: $it" + false + } + true + } | map { def (meta_spots, meta_reg, spots_input_image, spots_input_dataset, source_spots, final_spots) = it def id = meta_reg.fix_id @@ -41,7 +49,8 @@ workflow MEASURE_SPOTS { all_spots_image_containers, all_spots_datasets, all_source_spots, - all_final_spots].transpose().collect { + all_final_spots].transpose() + .collect { def (meta_spots, spots_image_container, spots_dataset, source_spots, final_spots) = it def spots_dir = file(final_spots).parent def spots_sizes_output_dir = file("${outputdir}/${params.spots_sizes_subdir}/${meta_spots.id}") From 750a7e7fc624e2c7dfe6432803203869da057b53 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Thu, 1 May 2025 16:30:22 -0400 Subject: [PATCH 59/78] bug fix - explicit return from filter --- modules/local/spots/regionprops/main.nf | 4 ++-- workflows/spots_features.nf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/local/spots/regionprops/main.nf b/modules/local/spots/regionprops/main.nf index 9e49b4a0..522928b9 100644 --- a/modules/local/spots/regionprops/main.nf +++ b/modules/local/spots/regionprops/main.nf @@ -21,8 +21,8 @@ process SPOTS_REGIONPROPS { tuple val(meta), env(full_input_image_path), val(input_dataset), - env(output_csv_file), emit: results - path "versions.yml" , emit: versions + env(output_csv_file) , emit: results + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 1fae352f..c03ba4c0 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -22,9 +22,9 @@ workflow MEASURE_SPOTS { def (meta_spots, meta_reg, spots_input_image, spots_input_dataset, source_spots, final_spots) = it if (!final_spots) { log.debug "Filter out spots input for: $it" - false + return false } - true + return true } | map { def (meta_spots, meta_reg, spots_input_image, spots_input_dataset, source_spots, final_spots) = it From 8eefa624b5f21be67c3b2c02d77ae0ffa0bc89b9 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Thu, 1 May 2025 16:53:27 -0400 Subject: [PATCH 60/78] changed log level --- workflows/spot_extraction.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/spot_extraction.nf b/workflows/spot_extraction.nf index 9472c9c9..e2dea65d 100644 --- a/workflows/spot_extraction.nf +++ b/workflows/spot_extraction.nf @@ -36,7 +36,7 @@ workflow SPOT_EXTRACTION { def final_rsfish_results if (params.skip_spot_extraction) { - log.info "Skipping spot extraction" + log.debug "Skipping spot extraction" // even if we skip spot extraction // we assume we have the csv file and we apply the post processing def verify_spot_results = spots_spark_input From db95429cf18c9030ab1bb3b1bb29872b5e587b0d Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Fri, 2 May 2025 18:33:34 -0400 Subject: [PATCH 61/78] use dapi channel for segmentation --- conf/segmentation-modules.config | 2 +- examples/medium_sample_params.json | 8 ++++---- workflows/segmentation.nf | 7 +++++-- workflows/spot_extraction.nf | 9 +++++++-- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/conf/segmentation-modules.config b/conf/segmentation-modules.config index 4d3954c9..fe6f24e9 100644 --- a/conf/segmentation-modules.config +++ b/conf/segmentation-modules.config @@ -4,7 +4,7 @@ params { segmentation_ids = 't1' segmentation_input = '' segmentation_subpaths = '' - seg_channels = 'c1' + seg_channels = '' seg_scales = 's2' segmentation_subdir = 'segmentation' segmentation_imgname = 'segmentation.n5' diff --git a/examples/medium_sample_params.json b/examples/medium_sample_params.json index 9289f94a..95e6ba6d 100644 --- a/examples/medium_sample_params.json +++ b/examples/medium_sample_params.json @@ -24,7 +24,7 @@ "registration_fix_id": "t1", "stitching_channel": "c3", - "reg_ch": "c3", + "reg_ch": "c2", "global_steps": "ransac,affine", "local_steps": "ransac,affine,deform", "global_scale": "s3", @@ -33,7 +33,7 @@ "cellpose_test": false, "cellpose_verbose": true, - "cellpose_dask_workers": 8, + "cellpose_dask_workers": 20, "cellpose_dask_worker_cpus": 1, "cellpose_merge_with_iou_only": true, "cellpose_worker_runtime_opts": "--nv", @@ -51,8 +51,8 @@ "multiscale_spark_worker_cores": 20, "multiscale_spark_gb_per_core": 5, - "dapi_channel": "c1", - "bleeding_channel": "c0", + "dapi_channel": "c2", + "bleeding_channel": "c3", "lsf_opts": "-P scicompsoft", "cellpose_worker_lsf_opts": "-q gpu_l4 -gpu 'num=1'" diff --git a/workflows/segmentation.nf b/workflows/segmentation.nf index 7c4673f4..1ff7a718 100644 --- a/workflows/segmentation.nf +++ b/workflows/segmentation.nf @@ -36,13 +36,16 @@ workflow SEGMENTATION { } else { input_img_dir = "${meta.stitching_result_dir}/${meta.stitching_container}" } - if (!params.segmentation_subpaths && !params.seg_channels && !params.seg_scales) { + if (!params.segmentation_subpaths && (!params.seg_channels || !params.dapi_channel) && !params.seg_scales) { segmentation_subpaths = [ '' ] // empty subpath - the input image container contains the array data } else if (params.segmentation_subpaths) { // in this case the subpaths parameters must match exactly the container datasets segmentation_subpaths = as_list(params.segmentation_subpaths) } else { - def seg_channels = as_list(params.seg_channels) + def seg_channels = params.seg_channels + ? as_list(params.seg_channels) + : as_list(params.dapi_channel) + def seg_scales = as_list(params.seg_scales) segmentation_subpaths = [seg_channels, seg_scales].combinations() diff --git a/workflows/spot_extraction.nf b/workflows/spot_extraction.nf index e2dea65d..34e5c2b6 100644 --- a/workflows/spot_extraction.nf +++ b/workflows/spot_extraction.nf @@ -166,8 +166,13 @@ def get_spot_subpaths(meta) { } else { // all but the last channel which typically is DAPI def all_channels = as_list(params.channels) - // this may throw an exception if the channel list is empty or a singleton - spot_channels = all_channels[0..-2] // all but the last channel + if (params.dapi_channel) { + spot_channels = all_channels.findAll { it != params.dapi_channel } + } else { + // automatically consider DAPI the last channel + // this may throw an exception if the channel list is empty or a singleton + spot_channels = all_channels[0..-2] // all but the last channel + } log.debug "Spot channels: $spot_channels (all from ${params.channels} except the last one)" } def spot_scales = as_list(params.spot_scales) From 32e585724642d0a8eb9ce01b78a9f72f18e04d81 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Mon, 5 May 2025 12:07:55 -0400 Subject: [PATCH 62/78] renamed spots_counts module --- modules/local/spots/{sizes => counts}/main.nf | 4 ++-- workflows/spots_features.nf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename modules/local/spots/{sizes => counts}/main.nf (95%) diff --git a/modules/local/spots/sizes/main.nf b/modules/local/spots/counts/main.nf similarity index 95% rename from modules/local/spots/sizes/main.nf rename to modules/local/spots/counts/main.nf index a8771e02..a3473cb7 100644 --- a/modules/local/spots/sizes/main.nf +++ b/modules/local/spots/counts/main.nf @@ -1,4 +1,4 @@ -process SPOTS_SIZES { +process SPOTS_COUNTS { tag { meta.id } container { task.ext.container ?: 'ghcr.io/janeliascicomp/easifish-spots-utils:v1' } cpus { ncpus } @@ -40,7 +40,7 @@ process SPOTS_SIZES { CMD=( python - /opt/scripts/spots-utils/labeled-spots-sizes.py + /opt/scripts/spots-utils/labeled-spots-counts.py --image-container \${full_input_image_path} --image-subpath ${input_dataset} --labels-container \${full_labels_path} diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index c03ba4c0..f3a34869 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -1,5 +1,5 @@ include { SPOTS_REGIONPROPS } from '../modules/local/spots/regionprops/main' -include { SPOTS_SIZES } from '../modules/local/spots/sizes/main' +include { SPOTS_COUNTS } from '../modules/local/spots/counts/main' workflow MEASURE_SPOTS { take: @@ -69,7 +69,7 @@ workflow MEASURE_SPOTS { } } - def spots_sizes_outputs = SPOTS_SIZES( + def spots_sizes_outputs = SPOTS_COUNTS( spots_sizes_input, params.spots_sizes_cores, params.spots_sizes_mem_gb, From b5070a120c7c4649759d86b8963c1cda13b982a9 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Mon, 5 May 2025 12:13:53 -0400 Subject: [PATCH 63/78] renamed spots counts --- conf/spots-features-modules.config | 6 +++--- workflows/spots_features.nf | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/conf/spots-features-modules.config b/conf/spots-features-modules.config index 4ce67539..2e2de7ff 100644 --- a/conf/spots-features-modules.config +++ b/conf/spots-features-modules.config @@ -1,9 +1,9 @@ params { - spots_sizes_cores = 1 - spots_sizes_mem_gb = 2 + spots_counts_cores = 1 + spots_counts_mem_gb = 2 cells_regionprops_cores = 1 cells_regionprops_mem_gb = 2 - spots_sizes_subdir = 'cell-sizes' + spots_counts_subdir = 'cell-counts' cells_regionprops_subdir = 'cell-props' dapi_channel = '' bleeding_channel = '' diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index f3a34869..70e318b2 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -17,7 +17,7 @@ workflow MEASURE_SPOTS { r } - def spots_sizes_input = ch_spots + def spots_counts_input = ch_spots | filter { def (meta_spots, meta_reg, spots_input_image, spots_input_dataset, source_spots, final_spots) = it if (!final_spots) { @@ -53,7 +53,7 @@ workflow MEASURE_SPOTS { .collect { def (meta_spots, spots_image_container, spots_dataset, source_spots, final_spots) = it def spots_dir = file(final_spots).parent - def spots_sizes_output_dir = file("${outputdir}/${params.spots_sizes_subdir}/${meta_spots.id}") + def spots_counts_output_dir = file("${outputdir}/${params.spots_counts_subdir}/${meta_spots.id}") def adjusted_spots_dataset = sync_image_scale_with_labels_scale(spots_dataset, seg_input_dataset) def r = [ @@ -62,21 +62,21 @@ workflow MEASURE_SPOTS { seg_labels, seg_input_dataset, spots_dir, '*coord.csv', - spots_sizes_output_dir, + spots_counts_output_dir, ] log.debug "Spots sizes input: $r" r } } - def spots_sizes_outputs = SPOTS_COUNTS( - spots_sizes_input, - params.spots_sizes_cores, - params.spots_sizes_mem_gb, + def spots_counts_outputs = SPOTS_COUNTS( + spots_counts_input, + params.spots_counts_cores, + params.spots_counts_mem_gb, ) emit: - done = spots_sizes_outputs.results + done = spots_counts_outputs.results } workflow EXTRACT_CELL_REGIONPROPS { From c2089c58f2e567590aee2525ee71380263905443 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Mon, 5 May 2025 13:15:08 -0400 Subject: [PATCH 64/78] renamed measure spots wf --- workflows/easifish.nf | 6 +++--- workflows/spots_features.nf | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/workflows/easifish.nf b/workflows/easifish.nf index b78a2509..7bb201f9 100644 --- a/workflows/easifish.nf +++ b/workflows/easifish.nf @@ -25,7 +25,7 @@ include { REGISTRATION } from './registration' include { SEGMENTATION } from './segmentation' include { SPOT_EXTRACTION } from './spot_extraction' include { WARP_SPOTS } from './warp_spots' -include { MEASURE_SPOTS } from './spots_features' +include { SPOTS_STATS } from './spots_features' include { EXTRACT_CELL_REGIONPROPS } from './spots_features' @@ -170,13 +170,13 @@ workflow EASIFISH { warped_spots_results.subscribe { log.debug "Warped spots results: $it " } - def sized_spots_results = MEASURE_SPOTS( + def spots_stats_results = SPOTS_STATS( warped_spots_results, segmentation_results, outdir, ) - sized_spots_results.subscribe { log.debug "Spots sizes: $it " } + spots_stats_results.subscribe { log.debug "Spots stats: $it " } def cell_regionprops = EXTRACT_CELL_REGIONPROPS( registration_results, diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 70e318b2..ac494600 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -1,7 +1,7 @@ include { SPOTS_REGIONPROPS } from '../modules/local/spots/regionprops/main' include { SPOTS_COUNTS } from '../modules/local/spots/counts/main' -workflow MEASURE_SPOTS { +workflow SPOTS_STATS { take: ch_spots // channel: [ meta_spots, meta_reg, spots_input_image, spots_input_dataset, spots, warped_spots ] ch_segmentation // channel: [ meta, seg_input_image, seg_input_dataset, seg_labels ] From ce6c87f6ae0aa0061145f50af236bb0399feeaef Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Mon, 5 May 2025 17:29:39 -0400 Subject: [PATCH 65/78] logging --- workflows/warp_spots.nf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index dff229bd..bdacb4f9 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -27,7 +27,9 @@ workflow WARP_SPOTS { ) = it def id = meta_reg.mov_id - [ id, meta_reg, inv_transform_path, inv_transform_name, inv_transform_subpath ] + def r = [ id, meta_reg, inv_transform_path, inv_transform_name, inv_transform_subpath ] + log.debug "Registration results considered for warping: $r" + r } def registration_fix = registration_results From 3c1acc2f5a567dc5533e81b85d559e9a7ed61861 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Mon, 5 May 2025 20:12:06 -0400 Subject: [PATCH 66/78] combine transform with all spots --- workflows/warp_spots.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/warp_spots.nf b/workflows/warp_spots.nf index bdacb4f9..28486e58 100644 --- a/workflows/warp_spots.nf +++ b/workflows/warp_spots.nf @@ -77,7 +77,7 @@ workflow WARP_SPOTS { def spots_warp_input = spots | filter { it[2] /* spots_file must be defined */} - | join(transform, by: 0) + | combine(transform, by: 0) | map { def ( id, From c19b51cad804a6041ca76afe0a20900c60054971 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 6 May 2025 09:33:31 -0400 Subject: [PATCH 67/78] logging to track why not cell props are processed --- workflows/spots_features.nf | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index ac494600..62ec4456 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -113,11 +113,16 @@ workflow EXTRACT_CELL_REGIONPROPS { } | unique { it[0] } // unique by id + fixed_images.subscribe { log.debug "Fixed images for regionprops: $it" } + registered_images.subscribe { log.debug "Registered images for regionprops: $it" } + def ch_segmentation_with_id = ch_segmentation | map { def (meta, seg_input_image, seg_input_dataset, seg_labels) = it def id = meta.id - [ id, meta, seg_input_image, seg_input_dataset, seg_labels ] + def r = [ id, meta, seg_input_image, seg_input_dataset, seg_labels ] + log.debug "Segmentation data for regionprops: $r" + r } def regionprops_inputs = fixed_images From eee01854b0700258e94d2417959a4fb244eda8dc Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 6 May 2025 12:03:38 -0400 Subject: [PATCH 68/78] combine all spot channels to extract region properties --- conf/spots-features-modules.config | 16 +++---- workflows/spots_features.nf | 74 +++++++++++++++++++++++++++--- workflows/util_functions.nf | 2 +- 3 files changed, 76 insertions(+), 16 deletions(-) diff --git a/conf/spots-features-modules.config b/conf/spots-features-modules.config index 2e2de7ff..154dd929 100644 --- a/conf/spots-features-modules.config +++ b/conf/spots-features-modules.config @@ -1,10 +1,10 @@ params { - spots_counts_cores = 1 - spots_counts_mem_gb = 2 - cells_regionprops_cores = 1 - cells_regionprops_mem_gb = 2 - spots_counts_subdir = 'cell-counts' - cells_regionprops_subdir = 'cell-props' - dapi_channel = '' - bleeding_channel = '' + spots_counts_cores = 1 + spots_counts_mem_gb = 2 + spots_props_cores = 1 + spots_props_mem_gb = 2 + spots_counts_subdir = 'spot-counts' + spots_props_subdir = 'spot-props' + dapi_channel = '' + bleeding_channel = '' } diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 62ec4456..ccf6ba5a 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -127,15 +127,29 @@ workflow EXTRACT_CELL_REGIONPROPS { def regionprops_inputs = fixed_images | concat(registered_images) + | flatMap { + def (id, image_container, image_dataset) = it + log.debug "Images for regionprops: $it" + get_spot_subpaths(id).collect { subpath, result_name -> + def r = [ + id, + image_container, + subpath, + result_name, + ] + log.debug "Image for regionprops: $r" + r + } + } | combine(ch_segmentation_with_id, by: 0) | map { def (id, - image_container, image_dataset, + image_container, image_dataset, result_name, meta, seg_input_image, seg_input_dataset, seg_labels) = it log.debug "Combined cell images with segmentation: $it" - def regionprops_output_dir = file("${outdir}/${params.cells_regionprops_subdir}/${meta.id}") + def regionprops_output_dir = file("${outdir}/${params.spots_props_subdir}/${meta.id}") def adjusted_image_dataset = sync_image_scale_with_labels_scale(image_dataset, seg_input_dataset) def dapi_dataset = params.dapi_channel @@ -145,8 +159,6 @@ workflow EXTRACT_CELL_REGIONPROPS { ? change_dataset_channel(adjusted_image_dataset, params.bleeding_channel) : '' - def dataset_ch = get_dataset_channel(adjusted_image_dataset) - def r = [ meta, image_container, adjusted_image_dataset, @@ -154,7 +166,7 @@ workflow EXTRACT_CELL_REGIONPROPS { dapi_dataset, bleeding_dataset, regionprops_output_dir, - "${meta.id}-${dataset_ch}-regionprops.csv", + result_name, ] log.debug "Cell regionprops input: $r" r @@ -162,8 +174,8 @@ workflow EXTRACT_CELL_REGIONPROPS { SPOTS_REGIONPROPS( regionprops_inputs, - params.cells_regionprops_cores, - params.cells_regionprops_mem_gb, + params.spots_props_cores, + params.spots_props_mem_gb, ) emit: @@ -193,3 +205,51 @@ def get_dataset_channel(image_dataset) { def image_dataset_comps = image_dataset.split('/') return image_dataset_comps ? image_dataset_comps[-2] : '' } + +def get_spot_subpaths(id) { + if (!params.spot_subpaths && !params.spot_channels && !params.spot_scales) { + return [ + ['', ''], // empty subpath, empty resultnane - the input image container contains the array dataset + ] + } else if (params.spot_subpaths) { + // in this case the subpaths parameters must match exactly the container datasets + return as_list(params.spot_subpaths) + .collect { subpath -> + def spot_ch = get_dataset_channel(subpath) + [ + "${id}/${subpath}", + "${id}-${spot_ch}-props.csv", + ] + } + } else { + def spot_channels; + if (params.spot_channels) { + spot_channels = as_list(params.spot_channels) + log.debug "Use specified spot channels: $spot_channels" + } else { + // all but the last channel which typically is DAPI + def all_channels = as_list(params.channels) + if (params.dapi_channel) { + spot_channels = all_channels.findAll { it != params.dapi_channel } + } else { + // automatically consider DAPI the last channel + // this may throw an exception if the channel list is empty or a singleton + spot_channels = all_channels[0..-2] // all but the last channel + } + log.debug "Spot channels: $spot_channels (all from ${params.channels} except the last one)" + } + def spot_scales = as_list(params.spot_scales) + + return [spot_channels, spot_scales].combinations() + .collect { ch, scale -> + // when channel and scale is used we also prepend the stitched dataset + def dataset = "${ch}/${scale}" + def r = [ + "${id}/${dataset}" + "${id}-${ch}-props.csv", + ] + log.debug "Spot dataset: $r" + r + } + } +} diff --git a/workflows/util_functions.nf b/workflows/util_functions.nf index 3cc24bd4..c4b52fdb 100644 --- a/workflows/util_functions.nf +++ b/workflows/util_functions.nf @@ -1,7 +1,7 @@ def as_list(v) { def vlist if (v instanceof Collection) { - vlist = deformation_entries + vlist = v } else if (v) { vlist = v.tokenize(',').collect { it.trim() } } else { From da2ae4a8e3524a0259416c6bc239df10d6f1936b Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 6 May 2025 12:14:49 -0400 Subject: [PATCH 69/78] fixed syntax error --- workflows/spots_features.nf | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index ccf6ba5a..b4313216 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -193,19 +193,6 @@ def sync_image_scale_with_labels_scale(image_dataset, labels_dataset) { return image_dataset_comps.join('/') } -def change_dataset_channel(image_dataset, channel) { - def image_dataset_comps = image_dataset.split('/') - if (image_dataset_comps) { - image_dataset_comps[-2] = channel - } - return image_dataset_comps.join('/') -} - -def get_dataset_channel(image_dataset) { - def image_dataset_comps = image_dataset.split('/') - return image_dataset_comps ? image_dataset_comps[-2] : '' -} - def get_spot_subpaths(id) { if (!params.spot_subpaths && !params.spot_channels && !params.spot_scales) { return [ @@ -245,7 +232,7 @@ def get_spot_subpaths(id) { // when channel and scale is used we also prepend the stitched dataset def dataset = "${ch}/${scale}" def r = [ - "${id}/${dataset}" + "${id}/${dataset}", "${id}-${ch}-props.csv", ] log.debug "Spot dataset: $r" @@ -253,3 +240,16 @@ def get_spot_subpaths(id) { } } } + +def change_dataset_channel(image_dataset, channel) { + def image_dataset_comps = image_dataset.split('/') + if (image_dataset_comps) { + image_dataset_comps[-2] = channel + } + return image_dataset_comps.join('/') +} + +def get_dataset_channel(image_dataset) { + def image_dataset_comps = image_dataset.split('/') + return image_dataset_comps ? image_dataset_comps[-2] : '' +} From 1e4e05090c35dadf0a53713cba8a99fc3c48a872 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 6 May 2025 12:17:47 -0400 Subject: [PATCH 70/78] missed include --- workflows/spots_features.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index b4313216..e7063f0b 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -1,5 +1,6 @@ include { SPOTS_REGIONPROPS } from '../modules/local/spots/regionprops/main' include { SPOTS_COUNTS } from '../modules/local/spots/counts/main' +include { as_list } from './util_functions' workflow SPOTS_STATS { take: From 7b7c90fdda6bc92fa9a41b8ccb16fe11482e5084 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 6 May 2025 13:15:10 -0400 Subject: [PATCH 71/78] use the proper round id to create the dataset --- workflows/spots_features.nf | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index e7063f0b..68724c69 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -97,7 +97,9 @@ workflow EXTRACT_CELL_REGIONPROPS { transform_name, transform_subpath, inv_transform_output, inv_transform_name, inv_transform_subpath) = it - [ reg_meta.fix_id, warped, warped_subpath ] + // include the registered round id + // so that it can be used for properly creating the dataset + [ reg_meta.fix_id, reg_meta.mov_id, warped, warped_subpath ] } def fixed_images = ch_registration @@ -110,13 +112,12 @@ workflow EXTRACT_CELL_REGIONPROPS { transform_name, transform_subpath, inv_transform_output, inv_transform_name, inv_transform_subpath) = it - [ reg_meta.fix_id, fix, fix_subpath ] + // the fixed round id is included + // to make the result compatible with the one for moving rounds + [ reg_meta.fix_id, reg_meta.fix_id, fix, fix_subpath ] } | unique { it[0] } // unique by id - fixed_images.subscribe { log.debug "Fixed images for regionprops: $it" } - registered_images.subscribe { log.debug "Registered images for regionprops: $it" } - def ch_segmentation_with_id = ch_segmentation | map { def (meta, seg_input_image, seg_input_dataset, seg_labels) = it @@ -129,9 +130,9 @@ workflow EXTRACT_CELL_REGIONPROPS { def regionprops_inputs = fixed_images | concat(registered_images) | flatMap { - def (id, image_container, image_dataset) = it + def (id, image_id, image_container, image_dataset) = it log.debug "Images for regionprops: $it" - get_spot_subpaths(id).collect { subpath, result_name -> + get_spot_subpaths(image_id).collect { subpath, result_name -> def r = [ id, image_container, From c17581d5601f25ea9d1b27039b4cf7a80b5f9dc2 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 6 May 2025 13:24:36 -0400 Subject: [PATCH 72/78] renamed vars --- conf/spots-features-modules.config | 4 ++-- workflows/spots_features.nf | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/conf/spots-features-modules.config b/conf/spots-features-modules.config index 154dd929..2c13bf6f 100644 --- a/conf/spots-features-modules.config +++ b/conf/spots-features-modules.config @@ -3,8 +3,8 @@ params { spots_counts_mem_gb = 2 spots_props_cores = 1 spots_props_mem_gb = 2 - spots_counts_subdir = 'spot-counts' - spots_props_subdir = 'spot-props' + spots_counts_subdir = 'spots-counts' + spots_props_subdir = 'spots-props' dapi_channel = '' bleeding_channel = '' } diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 68724c69..847a4523 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -130,11 +130,12 @@ workflow EXTRACT_CELL_REGIONPROPS { def regionprops_inputs = fixed_images | concat(registered_images) | flatMap { - def (id, image_id, image_container, image_dataset) = it + def (join_id, image_id, image_container, image_dataset) = it log.debug "Images for regionprops: $it" get_spot_subpaths(image_id).collect { subpath, result_name -> def r = [ - id, + join_id, + image_id, image_container, subpath, result_name, @@ -145,13 +146,13 @@ workflow EXTRACT_CELL_REGIONPROPS { } | combine(ch_segmentation_with_id, by: 0) | map { - def (id, - image_container, image_dataset, result_name, + def (join_id, + image_id, image_container, image_dataset, result_name, meta, seg_input_image, seg_input_dataset, seg_labels) = it log.debug "Combined cell images with segmentation: $it" - def regionprops_output_dir = file("${outdir}/${params.spots_props_subdir}/${meta.id}") + def regionprops_output_dir = file("${outdir}/${params.spots_props_subdir}/${image_id}") def adjusted_image_dataset = sync_image_scale_with_labels_scale(image_dataset, seg_input_dataset) def dapi_dataset = params.dapi_channel From 95b3b03aef385fd97ff028a08b4917e337e2f36e Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 6 May 2025 14:46:41 -0400 Subject: [PATCH 73/78] renamed spots props module --- .../spots/{regionprops => props}/main.nf | 4 ++-- workflows/easifish.nf | 20 +++++++++---------- workflows/spots_features.nf | 12 +++++------ 3 files changed, 18 insertions(+), 18 deletions(-) rename modules/local/spots/{regionprops => props}/main.nf (94%) diff --git a/modules/local/spots/regionprops/main.nf b/modules/local/spots/props/main.nf similarity index 94% rename from modules/local/spots/regionprops/main.nf rename to modules/local/spots/props/main.nf index 522928b9..2779d342 100644 --- a/modules/local/spots/regionprops/main.nf +++ b/modules/local/spots/props/main.nf @@ -1,4 +1,4 @@ -process SPOTS_REGIONPROPS { +process SPOTS_PROPS { tag { meta.id } container { task.ext.container ?: 'ghcr.io/janeliascicomp/easifish-spots-utils:v1' } cpus { ncpus } @@ -41,7 +41,7 @@ process SPOTS_REGIONPROPS { CMD=( python - /opt/scripts/spots-utils/labeled-cells-regionprops.py + /opt/scripts/spots-utils/labeled-spots-props.py --image-container \${full_input_image_path} --image-subpath ${input_dataset} --labels-container \${full_labels_path} diff --git a/workflows/easifish.nf b/workflows/easifish.nf index 7bb201f9..f3dd02c0 100644 --- a/workflows/easifish.nf +++ b/workflows/easifish.nf @@ -19,14 +19,14 @@ include { */ -include { INPUT_CHECK } from '../subworkflows/local/input_check' -include { STITCHING } from '../subworkflows/local/stitching' -include { REGISTRATION } from './registration' -include { SEGMENTATION } from './segmentation' -include { SPOT_EXTRACTION } from './spot_extraction' -include { WARP_SPOTS } from './warp_spots' -include { SPOTS_STATS } from './spots_features' -include { EXTRACT_CELL_REGIONPROPS } from './spots_features' +include { INPUT_CHECK } from '../subworkflows/local/input_check' +include { STITCHING } from '../subworkflows/local/stitching' +include { REGISTRATION } from './registration' +include { SEGMENTATION } from './segmentation' +include { SPOT_EXTRACTION } from './spot_extraction' +include { WARP_SPOTS } from './warp_spots' +include { SPOTS_STATS } from './spots_features' +include { EXTRACT_SPOTS_PROPS } from './spots_features' def validate_params() { @@ -178,13 +178,13 @@ workflow EASIFISH { spots_stats_results.subscribe { log.debug "Spots stats: $it " } - def cell_regionprops = EXTRACT_CELL_REGIONPROPS( + def spots_props = EXTRACT_SPOTS_PROPS( registration_results, segmentation_results, outdir, ) - cell_regionprops.subscribe { log.debug "Cell regionprops: $it " } + spots_props.subscribe { log.debug "Spots props: $it " } } diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 847a4523..1ab6ab5a 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -1,6 +1,6 @@ -include { SPOTS_REGIONPROPS } from '../modules/local/spots/regionprops/main' -include { SPOTS_COUNTS } from '../modules/local/spots/counts/main' -include { as_list } from './util_functions' +include { SPOTS_PROPS } from '../modules/local/spots/props/main' +include { SPOTS_COUNTS } from '../modules/local/spots/counts/main' +include { as_list } from './util_functions' workflow SPOTS_STATS { take: @@ -80,7 +80,7 @@ workflow SPOTS_STATS { done = spots_counts_outputs.results } -workflow EXTRACT_CELL_REGIONPROPS { +workflow EXTRACT_SPOTS_PROPS { take: ch_registration ch_segmentation @@ -175,14 +175,14 @@ workflow EXTRACT_CELL_REGIONPROPS { r } - SPOTS_REGIONPROPS( + def spots_props_results = SPOTS_PROPS( regionprops_inputs, params.spots_props_cores, params.spots_props_mem_gb, ) emit: - done = SPOTS_REGIONPROPS.out.results + done = spots_props_results.results } From 700d40e676655916e5306208eb5974a602cc1882 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 7 May 2025 09:42:04 -0400 Subject: [PATCH 74/78] enable all steps --- conf/cellpose_logging_config.ini | 2 +- examples/medium_sample_params.json | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/conf/cellpose_logging_config.ini b/conf/cellpose_logging_config.ini index 30082f07..d9ca6b7f 100644 --- a/conf/cellpose_logging_config.ini +++ b/conf/cellpose_logging_config.ini @@ -18,7 +18,7 @@ qualname=cellpose propagate=0 [logger_distributed_cellpose] -level=INFO +level=DEBUG handlers=consoleHandler qualname=distributed_cellpose propagate=0 diff --git a/examples/medium_sample_params.json b/examples/medium_sample_params.json index 95e6ba6d..0a86b471 100644 --- a/examples/medium_sample_params.json +++ b/examples/medium_sample_params.json @@ -7,7 +7,7 @@ "registration_subdir": "registration", "segmentation_subdir": "segmentation", - "skip_stitching": true, + "skip_stitching": false, "skip_registration": false, "skip_global_align": false, "skip_local_align": false, @@ -35,7 +35,7 @@ "cellpose_verbose": true, "cellpose_dask_workers": 20, "cellpose_dask_worker_cpus": 1, - "cellpose_merge_with_iou_only": true, + "cellpose_merge_with_iou_only": false, "cellpose_worker_runtime_opts": "--nv", "spark_workers": 5, @@ -51,6 +51,8 @@ "multiscale_spark_worker_cores": 20, "multiscale_spark_gb_per_core": 5, + "rsfish_spark_workers": 4, + "dapi_channel": "c2", "bleeding_channel": "c3", From 866c8428830217f49971dfd0ffe50a919c1fca23 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Tue, 13 May 2025 13:55:24 -0400 Subject: [PATCH 75/78] changed cellpose_use_model_and_size to cellpose_eval_with_model_only and now it defaults to false --- conf/segmentation-modules.config | 4 ++-- examples/segtest_params.json | 2 +- nextflow_schema.json | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conf/segmentation-modules.config b/conf/segmentation-modules.config index fe6f24e9..2310ff82 100644 --- a/conf/segmentation-modules.config +++ b/conf/segmentation-modules.config @@ -28,7 +28,7 @@ params { cellpose_verbose = false cellpose_test = false - cellpose_use_model_and_size = false + cellpose_eval_with_model_only = false cellpose_diameter = 30 cellpose_eval_channels = '0,0' cellpose_min_size = 5 @@ -112,7 +112,7 @@ process { optional_arg_value('--worker-cpus', params.cellpose_dask_worker_cpus), - bool_arg('--eval-model-with-size', params.cellpose_use_model_and_size), + bool_arg('--use-model-only-to-eval', params.cellpose_eval_with_model_only), bool_arg('--test-mode', params.cellpose_test), bool_arg('--verbose', params.cellpose_verbose), diff --git a/examples/segtest_params.json b/examples/segtest_params.json index 3e556dda..11e164c8 100644 --- a/examples/segtest_params.json +++ b/examples/segtest_params.json @@ -30,7 +30,7 @@ "cellpose_diameter": 40, "cellpose_min_size": 15000, "cellpose_preprocessing_steps": "", - "cellpose_use_model_and_size": false, + "cellpose_eval_with_model_only": false, "distributed_cellpose_blocksize": "256,256,256", "cellpose_blocks_overlap": "90,90,90", diff --git a/nextflow_schema.json b/nextflow_schema.json index 345bf8ed..7658f2a0 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -822,9 +822,9 @@ "description": "Control the merging algorithm for neighboring regions from adjacent blocks. If true use only IOU of the labels across adjacent faces.", "hidden": true }, - "cellpose_use_model_and_size": { + "cellpose_eval_with_model_only": { "type": "boolean", - "description": "If true use both CellposeModel and SizeModel to segment the volume, otherwise only use CellposeModel", + "description": "If true segment only with CellposeModel otherwise use both CellposeModel and SizeModel to segment the volume", "hidden": true }, "cellpose_merge_iou_depth": { From 82240206e7c632a9bdac0b5fa45dec47b82efda7 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 14 May 2025 09:29:54 -0400 Subject: [PATCH 76/78] change log level --- conf/cellpose_logging_config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cellpose_logging_config.ini b/conf/cellpose_logging_config.ini index d9ca6b7f..30082f07 100644 --- a/conf/cellpose_logging_config.ini +++ b/conf/cellpose_logging_config.ini @@ -18,7 +18,7 @@ qualname=cellpose propagate=0 [logger_distributed_cellpose] -level=DEBUG +level=INFO handlers=consoleHandler qualname=distributed_cellpose propagate=0 From 433b84edd55bd22ec400e2f675f0095bcfa6d937 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 14 May 2025 14:20:02 -0400 Subject: [PATCH 77/78] changelog update --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60ab3dd5..5e31c9da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # JaneliaSciComp/easifish: Changelog +* Added spots count and spots intensities + * Added spots warping using Bigstream * Added RS-FISH module for spot extraction From 6d87dee33a7a8b79521436c1bf99a4e44aa9feb1 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Wed, 14 May 2025 15:58:50 -0400 Subject: [PATCH 78/78] added missing params --- nextflow_schema.json | 166 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/nextflow_schema.json b/nextflow_schema.json index 7658f2a0..ec57a315 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -514,6 +514,166 @@ }, "help_text": "Registration parameters" }, + "spot_extraction": { + "title": "Spot extraction", + "type": "object", + "description": "Spot extraction options", + "help_text": "Spot extraction arguments", + "fa_icon": "far fa-map", + "properties": { + "skip_spot_extraction": { + "type": "boolean", + "description": "If set skip spot extraction" + }, + "spot_extraction_subdir": { + "type": "string", + "fa_icon": "fas fa-folder-open", + "description": "Spot extraction results sub-directory" + }, + "spot_extraction_ids": { + "type": "string", + "description": "rounds used for extracting spots" + }, + "spot_channels": { + "type": "string", + "description": "Comma delimited list of spot channels. This together with spot_scales is used for determining the datasets used for spot extraction." + }, + "spot_scales": { + "type": "string", + "description": "image scale used for spot extraction. This together with spot_channels is used for determining the datasets used for spot extraction." + }, + "spot_subpaths": { + "type": "string", + "description": "alternative for defining the datasets used for spot extraction. If this is defined it takes priority over spot_channels and spot_scales." + }, + "rsfish_min_intensity": { + "type": "integer", + "description": "Minimal intensity of the image. Default: 0", + "fa_icon": "fas fa-angle-double-down" + }, + "rsfish_max_intensity": { + "type": "integer", + "description": "Maximal intensity of the image. Default: 4096", + "fa_icon": "fas fa-angle-double-up" + }, + "rsfish_anisotropy": { + "type": "number", + "description": "The anisotropy factor. Default: 0.7", + "help_text": "Scaling of z relative to xy. Can be determined using the RS-FISH anisotropy plugin in Fiji.", + "fa_icon": "fas fa-arrows-alt" + }, + "rsfish_sigma": { + "type": "number", + "description": "Sigma value for Difference-of-Gaussian (DoG) calculation. Default 1.5", + "fa_icon": "fab fa-etsy" + }, + "rsfish_threshold": { + "type": "number", + "description": "Threshold value for Difference-of-Gaussian (DoG) calculation. Default: 0.007", + "fa_icon": "fas fa-level-down-alt" + }, + "rsfish_background": { + "type": "integer", + "description": "Background subtraction method, 0 == None, 1 == Mean, 2==Median, 3==RANSAC on Mean, 4==RANSAC on Median. Default: 0 (None)", + "fa_icon": "fas fa-level-down-alt" + }, + "rsfish_intensity": { + "type": "integer", + "description": "Intensity calculation method, 0 == Linear Interpolation, 1 == Gaussian fit (on inlier pixels), 2 == Integrate spot intensities (on candidate pixels). Default: 0 (Linear Interpolation)", + "fa_icon": "fas fa-level-down-alt" + }, + "rsfish_spark_workers": { + "type": "integer", + "fa_icon": "fas fa-cogs", + "description": "Number of Spark workers to use for spot extraction.", + "default": 2 + }, + "rsfish_min_spark_workers": { + "type": "integer", + "fa_icon": "fas fa-cogs", + "description": "Minimum number of Spark workers needed to start spot extraction pipeline.", + "default": 1 + }, + "rsfish_spark_worker_cores": { + "type": "integer", + "fa_icon": "fas fa-microchip", + "description": "Number of cores allocated to each Spark worker.", + "default": 5 + }, + "rsfish_spark_gb_per_core": { + "type": "integer", + "fa_icon": "fas fa-cog", + "description": "Size of memory (in GB) that is allocated for each core of a Spark worker.", + "help_text": "The total memory usage for extracting spots from one acquisition will be workers * worker_cores * gb_per_core.", + "default": 5 + }, + "rsfish_spark_driver_cores": { + "type": "integer", + "fa_icon": "fas fa-microchip", + "description": "Number of cores allocated for the Spark driver. Default: 1", + "default": 1 + }, + "rsfish_spark_driver_mem_gb": { + "type": "integer", + "fa_icon": "fas fa-memory", + "description": "Amount of memory to allocate for the Spark driver. Default: 12g", + "default": 4 + } + } + }, + "spots_features": { + "title": "Spots stats and characteristics", + "type": "object", + "description": "Various spots characteristcs, such as counts, area, intensity", + "fa_icon": "far fa-map", + "properties": { + "spots_counts_subdir": { + "type": "string", + "fa_icon": "fas fa-folder-open", + "description": "data sub-directory used for spots counts" + }, + "spots_props_subdir": { + "type": "string", + "fa_icon": "fas fa-folder-open", + "description": "data sub-directory used for spots properties" + }, + "dapi_channel": { + "type": "string", + "fa_icon": "fas fa-asterisk", + "description": "Name of the DAPI channel.", + "help_text": "The DAPI channel is used as a reference channel for registration, segmentation, and spot extraction." + }, + "bleed_channel": { + "type": "string", + "fa_icon": "fas fa-asterisk", + "description": "Channel (other than DAPI) that needs bleedthrough correction." + }, + "spots_counts_cores": { + "type": "integer", + "fa_icon": "fas fa-microchip", + "description": "Number of cores needed for getting spots counts.", + "default": 1 + }, + "spots_counts_mem_gb": { + "type": "integer", + "fa_icon": "fas fa-memory", + "description": "Amount of memory needed for getting spots counts", + "default": 2 + }, + "spots_props_cores": { + "type": "integer", + "fa_icon": "fas fa-microchip", + "description": "Number of cores needed for getting spots properties (intensity).", + "default": 1 + }, + "spots_props_mem_gb": { + "type": "integer", + "fa_icon": "fas fa-memory", + "description": "Amount of memory needed for getting spots properties (intensity)", + "default": 2 + } + } + }, "institutional_config_options": { "title": "Institutional config options", "type": "object", @@ -1030,6 +1190,12 @@ { "$ref": "#/definitions/segmentation" }, + { + "$ref": "#/definitions/spot_extraction" + }, + { + "$ref": "#/definitions/spots_features" + }, { "$ref": "#/definitions/generic_options" }