From 5d2efacc0b95158c4676fda72d936e0712afcf2c Mon Sep 17 00:00:00 2001 From: Kishori Konwar Date: Tue, 28 Jul 2020 18:47:06 +0000 Subject: [PATCH 01/11] increased the HDD to 3500 GB, from 500 GB originally, and updated the new sctools to write the intermediate BAM files of SplitCellByBarcode in SAM format. The goal is to speed-up the step but at the expense of larger sized intermediate files in the step --- library/tasks/SplitBamByCellBarcode.wdl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library/tasks/SplitBamByCellBarcode.wdl b/library/tasks/SplitBamByCellBarcode.wdl index 8733539b1..77856ebb9 100644 --- a/library/tasks/SplitBamByCellBarcode.wdl +++ b/library/tasks/SplitBamByCellBarcode.wdl @@ -6,13 +6,13 @@ task SplitBamByCellBarcode { Float size_in_mb = 1024.0 # runtime values - String docker = "quay.io/humancellatlas/secondary-analysis-sctools:v0.3.5" + String docker = "quay.io/humancellatlas/secondary-analysis-sctools:v0.3.9" Int machine_mem_mb = 15258 Int cpu = 16 # we can calculate disk size for arrays of input files in WDL 1.0 - Int disk = 500 + Int disk = 3500 # by default request non preemptible machine to make sure the slow cell barcode split step completes Int preemptible = 0 } From 0366569714c404cadcb74f037299d234bc87387b Mon Sep 17 00:00:00 2001 From: Kishori Konwar Date: Wed, 29 Jul 2020 11:47:01 +0000 Subject: [PATCH 02/11] minor formatting --- library/tasks/SplitBamByCellBarcode.wdl | 1 + 1 file changed, 1 insertion(+) diff --git a/library/tasks/SplitBamByCellBarcode.wdl b/library/tasks/SplitBamByCellBarcode.wdl index 77856ebb9..f8fabd676 100644 --- a/library/tasks/SplitBamByCellBarcode.wdl +++ b/library/tasks/SplitBamByCellBarcode.wdl @@ -13,6 +13,7 @@ task SplitBamByCellBarcode { # we can calculate disk size for arrays of input files in WDL 1.0 Int disk = 3500 + # by default request non preemptible machine to make sure the slow cell barcode split step completes Int preemptible = 0 } From b0ab763c53348329e9d9fee1ffcfabf39052e536 Mon Sep 17 00:00:00 2001 From: Kishori Konwar Date: Thu, 30 Jul 2020 17:12:42 +0000 Subject: [PATCH 03/11] fixed pr error --- test/optimus/pr/ValidateOptimus.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/optimus/pr/ValidateOptimus.wdl b/test/optimus/pr/ValidateOptimus.wdl index 78fb04857..5a542f702 100644 --- a/test/optimus/pr/ValidateOptimus.wdl +++ b/test/optimus/pr/ValidateOptimus.wdl @@ -265,7 +265,7 @@ task GenerateReport { fail=false echo Bam Validation: ~{bam_validation_result} - if [ "~{bam_validation_result}" == "FAIL"]; then + if [ "~{bam_validation_result}" == "FAIL" ]; then fail=true fi From b40d146f8bb744c02c3db41cdbd702290b2951b1 Mon Sep 17 00:00:00 2001 From: Kishori Konwar Date: Fri, 31 Jul 2020 17:51:40 +0000 Subject: [PATCH 04/11] fixed pr for optimus_mouse --- test/optimus_mouse/pr/test_inputs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/optimus_mouse/pr/test_inputs.json b/test/optimus_mouse/pr/test_inputs.json index 682330478..9e9e225cd 100644 --- a/test/optimus_mouse/pr/test_inputs.json +++ b/test/optimus_mouse/pr/test_inputs.json @@ -1,5 +1,5 @@ { - "TestOptimusPR.expected_bam_hash": "8ff09ba3b0df8ac5a4c8276cde239b72", + "TestOptimusPR.expected_bam_hash": "d832a4fa8362c61ae1fc8096d85690fd", "TestOptimusPR.expected_matrix_hash": "79d0e000e9e707e16fce1ecc1559cd5d", "TestOptimusPR.expected_cell_metric_hash": "627baa8511cea26b33f12e4f8021fc5d", "TestOptimusPR.expected_gene_metric_hash": "30ad387acf91764a937c1adf3bc8bb66", From 991cf4e24e141442657a71038a063364503ddab4 Mon Sep 17 00:00:00 2001 From: Kishori Konwar Date: Mon, 3 Aug 2020 14:14:16 +0000 Subject: [PATCH 05/11] fixed validation results --- test/optimus_snrna/pr/test_inputs.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/optimus_snrna/pr/test_inputs.json b/test/optimus_snrna/pr/test_inputs.json index 3b63bda88..23615e31a 100644 --- a/test/optimus_snrna/pr/test_inputs.json +++ b/test/optimus_snrna/pr/test_inputs.json @@ -1,9 +1,9 @@ { "TestOptimusPR.expected_bam_hash": "77bb8a250b56d913a0d75c63f67c17da", "TestOptimusPR.reference_matrix": "gs://hca-dcp-sc-pipelines-test-data/optimus-test-data/referenceMatrix_snrna.rds", - "TestOptimusPR.expected_cell_metric_hash": "8df13b6298f1baccf95d18b4f50b7089", - "TestOptimusPR.expected_gene_metric_hash": "71fd4a40176b4e3728d6a3ac4317764b", - "TestOptimusPR.expected_loom_file_checksum": "6cd0d7f1cb4c92a215d7b21ff50dc72d", + "TestOptimusPR.expected_cell_metric_hash": "fee06b8f0dfae528ef13d06283214190", + "TestOptimusPR.expected_gene_metric_hash": "2ef1173b79a899eab55f28785533f4f0", + "TestOptimusPR.expected_loom_file_checksum": "3dd004e0ef286bbf193945df6dbe7739", "TestOptimusPR.r1_fastq": [ "gs://hca-dcp-sc-pipelines-test-data/smallDatasets/chemistry_X10_V2/nuclei_900/nuclei_900_S1_L001_R1_001.filtered.fastq.gz", "gs://hca-dcp-sc-pipelines-test-data/smallDatasets/chemistry_X10_V2/nuclei_900/nuclei_900_S1_L002_R1_001.filtered.fastq.gz" From 7736f0cb5cfe647532a1f30d693bc9948948c364 Mon Sep 17 00:00:00 2001 From: Kishori Konwar Date: Mon, 3 Aug 2020 19:18:38 +0000 Subject: [PATCH 06/11] fixed tests --- test/options.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/options.json b/test/options.json index 720239d58..79d53b3ed 100644 --- a/test/options.json +++ b/test/options.json @@ -1,5 +1,5 @@ { "monitoring_script": "gs://hca-dcp-mint-test-data/accessories/monitoring/cellranger/monitoring.sh", - "read_from_cache": true, + "read_from_cache": false, "write_to_cache": true } From 4d09c7b28c2997c44f325068cf3a374dd786cc59 Mon Sep 17 00:00:00 2001 From: Kishori Konwar Date: Mon, 3 Aug 2020 22:17:27 +0000 Subject: [PATCH 07/11] test set to read from cache true --- test/options.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/options.json b/test/options.json index 79d53b3ed..8383a99c4 100644 --- a/test/options.json +++ b/test/options.json @@ -1,5 +1,5 @@ { "monitoring_script": "gs://hca-dcp-mint-test-data/accessories/monitoring/cellranger/monitoring.sh", - "read_from_cache": false, - "write_to_cache": true + "read_from_cache": true, + "write_to_cache": false } From ff57e22cf8fa16425fdabb02fc5175a124079ee8 Mon Sep 17 00:00:00 2001 From: Kishori Konwar Date: Mon, 3 Aug 2020 22:19:40 +0000 Subject: [PATCH 08/11] run limited tests --- .circleci/config.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b799938e..c7c7656fa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -188,17 +188,17 @@ workflows: version: 2 test_all: jobs: - - test_optimus +# - test_optimus - test_optimus_snrna # - test_emptyDropsWrapper - - test_optimus_mouse - - test_smartseq2 - - test_smartseq2_single_end - - test_npz2rds - - test_snap_atac - - test_sc_atac - - test_bulk_rna - - test_optimus_v3 - - test_smartseq2_multisample - - test_smartseq2_multisample_single_end +# - test_optimus_mouse +# - test_smartseq2 +# - test_smartseq2_single_end +# - test_npz2rds +# - test_snap_atac +# - test_sc_atac +# - test_bulk_rna +# - test_optimus_v3 +# - test_smartseq2_multisample +# - test_smartseq2_multisample_single_end From 083a3e1fe716d7ef923c03944a4027ebc57877ed Mon Sep 17 00:00:00 2001 From: Kishori Konwar Date: Tue, 4 Aug 2020 12:40:00 +0000 Subject: [PATCH 09/11] matrix --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c7c7656fa..1c7aa4e59 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -201,4 +201,3 @@ workflows: # - test_optimus_v3 # - test_smartseq2_multisample # - test_smartseq2_multisample_single_end - From b459f84c74a4afb42a2324d0a03ad008f146a889 Mon Sep 17 00:00:00 2001 From: Kishori Konwar Date: Tue, 4 Aug 2020 13:11:02 +0000 Subject: [PATCH 10/11] matrix --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c7aa4e59..99e376e31 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -188,16 +188,16 @@ workflows: version: 2 test_all: jobs: -# - test_optimus + - test_optimus - test_optimus_snrna # - test_emptyDropsWrapper -# - test_optimus_mouse + - test_optimus_mouse # - test_smartseq2 # - test_smartseq2_single_end # - test_npz2rds # - test_snap_atac # - test_sc_atac # - test_bulk_rna -# - test_optimus_v3 + - test_optimus_v3 # - test_smartseq2_multisample # - test_smartseq2_multisample_single_end From a5a4cc44c4d9e049c399c2dce83b49503d7a31a0 Mon Sep 17 00:00:00 2001 From: Kishori Konwar Date: Tue, 4 Aug 2020 13:51:26 +0000 Subject: [PATCH 11/11] matrix --- .circleci/config.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 99e376e31..d8040e4b2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -188,16 +188,16 @@ workflows: version: 2 test_all: jobs: - - test_optimus - - test_optimus_snrna +# - test_optimus +# - test_optimus_snrna # - test_emptyDropsWrapper - test_optimus_mouse -# - test_smartseq2 -# - test_smartseq2_single_end + - test_smartseq2 + - test_smartseq2_single_end # - test_npz2rds # - test_snap_atac # - test_sc_atac # - test_bulk_rna - - test_optimus_v3 -# - test_smartseq2_multisample -# - test_smartseq2_multisample_single_end +# - test_optimus_v3 + - test_smartseq2_multisample + - test_smartseq2_multisample_single_end