From b2eb2f1c1ac8db98a397ab11db93ee17c9785a03 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Thu, 13 Nov 2025 10:53:26 -0500 Subject: [PATCH 1/8] fixes for main brain - mainly schema fixes --- bin/download.sh | 2 +- conf/segmentation-modules.config | 3 +-- examples/medium_sample_params.json | 2 +- examples/segtest_params.json | 2 +- examples/tiny_ome_sample_params.json | 2 +- examples/tiny_sample_params.json | 2 +- nextflow_schema.json | 35 +++++++++++++++++++++++++++- workflows/spots_features.nf | 4 ++-- 8 files changed, 42 insertions(+), 10 deletions(-) diff --git a/bin/download.sh b/bin/download.sh index b2b63f11..ee4bda10 100755 --- a/bin/download.sh +++ b/bin/download.sh @@ -6,7 +6,7 @@ md5=$3 if [ ! -e $filepath ]; then echo "Downloading $uri to $filepath" - curl -skL $uri -o $filepath + curl -skL --user-agent 'Mozilla/5.0' $uri -o $filepath fi if [ "$md5" ]; then diff --git a/conf/segmentation-modules.config b/conf/segmentation-modules.config index e68421fc..7c3b8d40 100644 --- a/conf/segmentation-modules.config +++ b/conf/segmentation-modules.config @@ -54,8 +54,7 @@ params { cellpose_dashboard_port = 0 cellpose_worker_runtime_opts = '' - cellpose_worker_cluster_opts = '' - cellpose_worker_lsf_opts = '-q gpu_l4 -gpu "num=1"' + cellpose_worker_cluster_opts = '-q gpu_l4 -gpu "num=1"' } process { diff --git a/examples/medium_sample_params.json b/examples/medium_sample_params.json index 0a86b471..270354a9 100644 --- a/examples/medium_sample_params.json +++ b/examples/medium_sample_params.json @@ -57,5 +57,5 @@ "bleeding_channel": "c3", "lsf_opts": "-P scicompsoft", - "cellpose_worker_lsf_opts": "-q gpu_l4 -gpu 'num=1'" + "cellpose_worker_cluster_opts": "-q gpu_l4 -gpu 'num=1'" } diff --git a/examples/segtest_params.json b/examples/segtest_params.json index 11e164c8..e00cc462 100644 --- a/examples/segtest_params.json +++ b/examples/segtest_params.json @@ -45,5 +45,5 @@ "cellpose_dask_workers": 8, "runtime_opts": "", "lsf_opts": "-P scicompsoft", - "cellpose_worker_lsf_opts": "-q gpu_l4 -gpu 'num=1'" + "cellpose_worker_cluster_opts": "-q gpu_l4 -gpu 'num=1'" } diff --git a/examples/tiny_ome_sample_params.json b/examples/tiny_ome_sample_params.json index e3680b29..d23d42c5 100644 --- a/examples/tiny_ome_sample_params.json +++ b/examples/tiny_ome_sample_params.json @@ -58,5 +58,5 @@ "bleeding_channel": "c0", "lsf_opts": "-P scicompsoft", - "cellpose_worker_lsf_opts": "-q gpu_l4 -gpu 'num=1'" + "cellpose_worker_cluster_opts": "-q gpu_l4 -gpu 'num=1'" } diff --git a/examples/tiny_sample_params.json b/examples/tiny_sample_params.json index 12f2e551..488d5c57 100644 --- a/examples/tiny_sample_params.json +++ b/examples/tiny_sample_params.json @@ -55,5 +55,5 @@ "bleeding_channel": "c0", "lsf_opts": "-P scicompsoft", - "cellpose_worker_lsf_opts": "-q gpu_l4 -gpu 'num=1'" + "cellpose_worker_cluster_opts": "-q gpu_l4 -gpu 'num=1'" } diff --git a/nextflow_schema.json b/nextflow_schema.json index ec57a315..226da9d0 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -1038,12 +1038,42 @@ "cellpose_worker_runtime_opts": { "type": "string" }, - "cellpose_worker_lsf_opts": { + "cellpose_worker_cluster_opts": { "type": "string", "default": "-q gpu_l4 -gpu \"num=1" } } }, + "warp_spots": { + "title": "Warp spots parameters", + "type": "object", + "description": "Parameters for warping spots", + "fa_icon": "far fa-map", + "properties": { + "skip_warp_spots": { + "type": "boolean", + "description": "If set skip spots warping", + "default": false + }, + "warped_spots_subdir": { + "type": "string", + "description": "Subdirectory for warped spots results", + "default": "warped-spots" + }, + "warp_spots_cpus": { + "type": "integer", + "fa_icon": "fas fa-microchip", + "description": "Number of cpus needed for warping spots.", + "default": 1 + }, + "warp_spots_mem_in_gb": { + "type": "integer", + "fa_icon": "fas fa-memory", + "description": "Amount of memory needed for warping spots", + "default": 15 + } + } + }, "generic_options": { "title": "Generic options", "type": "object", @@ -1193,6 +1223,9 @@ { "$ref": "#/definitions/spot_extraction" }, + { + "$ref": "#/definitions/warp_spots" + }, { "$ref": "#/definitions/spots_features" }, diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index 1ab6ab5a..b3d6771b 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -158,8 +158,8 @@ workflow EXTRACT_SPOTS_PROPS { def dapi_dataset = params.dapi_channel ? change_dataset_channel(adjusted_image_dataset, params.dapi_channel) : '' - def bleeding_dataset = params.bleeding_channel - ? change_dataset_channel(adjusted_image_dataset, params.bleeding_channel) + def bleeding_dataset = params.bleed_channel + ? change_dataset_channel(adjusted_image_dataset, params.bleed_channel) : '' def r = [ From b9003488ac7adf8986e784dfe40a50b185f770bc Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Thu, 13 Nov 2025 11:00:16 -0500 Subject: [PATCH 2/8] renamed bleed channel --- nextflow_schema.json | 2 +- workflows/spots_features.nf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 226da9d0..4413d83e 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -643,7 +643,7 @@ "description": "Name of the DAPI channel.", "help_text": "The DAPI channel is used as a reference channel for registration, segmentation, and spot extraction." }, - "bleed_channel": { + "bleeding_channel": { "type": "string", "fa_icon": "fas fa-asterisk", "description": "Channel (other than DAPI) that needs bleedthrough correction." diff --git a/workflows/spots_features.nf b/workflows/spots_features.nf index b3d6771b..1ab6ab5a 100644 --- a/workflows/spots_features.nf +++ b/workflows/spots_features.nf @@ -158,8 +158,8 @@ workflow EXTRACT_SPOTS_PROPS { def dapi_dataset = params.dapi_channel ? change_dataset_channel(adjusted_image_dataset, params.dapi_channel) : '' - def bleeding_dataset = params.bleed_channel - ? change_dataset_channel(adjusted_image_dataset, params.bleed_channel) + def bleeding_dataset = params.bleeding_channel + ? change_dataset_channel(adjusted_image_dataset, params.bleeding_channel) : '' def r = [ From 34515e6a15dafeb7a502c1f0bef77958a9f47731 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Thu, 13 Nov 2025 11:07:13 -0500 Subject: [PATCH 3/8] allow report overwrite --- nextflow.config | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nextflow.config b/nextflow.config index e319fa6f..9547c8fa 100644 --- a/nextflow.config +++ b/nextflow.config @@ -208,8 +208,9 @@ timeline { file = "${params.outdir}/pipeline_info/execution_timeline_${trace_timestamp}.html" } report { - enabled = true - file = "${params.outdir}/pipeline_info/execution_report_${trace_timestamp}.html" + enabled = true + overwrite = true + file = "${params.outdir}/pipeline_info/execution_report_${trace_timestamp}.html" } trace { enabled = true From 4eebbd1fe3da711fec7a6371c1803b3224e6151a Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Thu, 13 Nov 2025 11:24:16 -0500 Subject: [PATCH 4/8] added 2 more missing parameters --- nextflow_schema.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nextflow_schema.json b/nextflow_schema.json index 4413d83e..5a0905a5 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -534,6 +534,10 @@ "type": "string", "description": "rounds used for extracting spots" }, + "distributed_spot_extraction": { + "type": "boolean", + "description": "if set uses a spark cluster to run spot extraction otherwise it uses spark-local" + }, "spot_channels": { "type": "string", "description": "Comma delimited list of spot channels. This together with spot_scales is used for determining the datasets used for spot extraction." @@ -1191,6 +1195,11 @@ "runtime_opts": { "type": "string", "description": "container run options" + }, + "lsf_opts": { + "type": "string", + "description": "LSF cluster options when running an LSF executor", + "hidden": true } } } From e68c0e5e9f89279cdfa7c9a748f8b913c89c212a Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Thu, 13 Nov 2025 11:36:21 -0500 Subject: [PATCH 5/8] formatting --- conf/segmentation-modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/segmentation-modules.config b/conf/segmentation-modules.config index 7c3b8d40..c7e3046e 100644 --- a/conf/segmentation-modules.config +++ b/conf/segmentation-modules.config @@ -54,7 +54,7 @@ params { cellpose_dashboard_port = 0 cellpose_worker_runtime_opts = '' - cellpose_worker_cluster_opts = '-q gpu_l4 -gpu "num=1"' + cellpose_worker_cluster_opts = '-q gpu_l4 -gpu "num=1"' } process { From e84d52f4a8b7cac7644400da12977a91412562e9 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Thu, 13 Nov 2025 12:24:14 -0500 Subject: [PATCH 6/8] updated some default float values --- examples/tiny_sample_params.json | 2 +- nextflow_schema.json | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/examples/tiny_sample_params.json b/examples/tiny_sample_params.json index 488d5c57..cb9c7501 100644 --- a/examples/tiny_sample_params.json +++ b/examples/tiny_sample_params.json @@ -1,6 +1,6 @@ { "channels": "c0,c1", - "input": "examples/remote_samplesheets/tiny_samplesheet.csv", + "input": "${projectDir}/examples/remote_samplesheets/tiny_samplesheet.csv", "indir": "/nrs/scicompsoft/goinac/multifish/tiny", "outdir": "/nrs/scicompsoft/goinac/multifish/tiny/results", diff --git a/nextflow_schema.json b/nextflow_schema.json index 5a0905a5..535640aa 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -440,7 +440,7 @@ }, "bigstream_inv_step": { "type": "number", - "default": 1.0, + "default": 1, "hidden": true, "description": "Step used for generating inverse displacement vector" }, @@ -870,11 +870,11 @@ }, "sample_anisotropy": { "type": "number", - "default": 1.0 + "default": 1 }, "sample_expansion_factor": { "type": "number", - "default": 1.0 + "default": 1 }, "sample_voxel_spacing": { "type": "string" @@ -962,7 +962,7 @@ }, "cellprob_threshold": { "type": "number", - "default": 1.0, + "default": 1, "description": "Probabilty threshold to control cellpose mask size." }, "cellpose_iou_threshold": { @@ -1195,11 +1195,6 @@ "runtime_opts": { "type": "string", "description": "container run options" - }, - "lsf_opts": { - "type": "string", - "description": "LSF cluster options when running an LSF executor", - "hidden": true } } } From fc843324f71947acbb90fb57caad3bb7f01bdde2 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Thu, 13 Nov 2025 14:30:48 -0500 Subject: [PATCH 7/8] fixed id schema url --- nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 535640aa..b21927b3 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/JaneliaSciComp/easifish/master/nextflow_schema.json", + "$id": "https://raw.githubusercontent.com/JaneliaSciComp/easifish/main/nextflow_schema.json", "title": "JaneliaSciComp/easifish pipeline parameters", "description": "Lightsheet microscopy image reconstruction", "type": "object", From dc1d98c4a6cf22e84651af5e4fc43dfab0c25869 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Thu, 13 Nov 2025 14:31:43 -0500 Subject: [PATCH 8/8] use http for schema not https --- nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index b21927b3..01331142 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "http://json-schema.org/draft-07/schema", "$id": "https://raw.githubusercontent.com/JaneliaSciComp/easifish/main/nextflow_schema.json", "title": "JaneliaSciComp/easifish pipeline parameters", "description": "Lightsheet microscopy image reconstruction",