Update all nf-core modules (#100)

* Update Bcftools annotate * Update Bcftools concat * Update Bcftools convert * Update Bcftools index * Update Bcftools norm * Update bcftools pluginsplit * Remove unused bcftools query * Update bcftools view * Update dumpsoftwareversions * Update Gawk * Update glimpse2 chunk * Update glimpse2 concordance * Update glimpse2 ligate splitreference * Update glimpse2 phase * Update Gunzip * Update multiqc * Update samtools faidx * Update samtools merge * Update samtools index * Update samtools reheader * Update samtools view * Update modules.json * Patch samtools view * Update bcftools * Patch bcftools concat * Fix getfilesextension for null file, add debug and fix variable * Update gawk * Fix samtools index * Update changelog * Remove uncessary process * Fix chunking * Update snapshot * Update snapshot * Update snapshot * Update checkFileIndex function to collect the file before checking --------- Co-authored-by: LouisLeNezet <[email protected]>
nf-core · Jul 12, 2024 · 930c996 · 930c996
1 parent 48eb141
commit 930c996
Show file tree

Hide file tree

Showing 142 changed files with 4,362 additions and 1,245 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -44,6 +44,7 @@ Initial release of nf-core/phaseimpute, created with the [nf-core](https://nf-co
 - [#93](https://github.com/nf-core/phaseimpute/pull/93) - Fix fai combination to fasta.
 - [#96](https://github.com/nf-core/phaseimpute/pull/96) - Simplify csv export
 - [#96](https://github.com/nf-core/phaseimpute/pull/96) - Use only legend file as posfile for all imputation workflow.
+- [#100](https://github.com/nf-core/phaseimpute/pull/100) - Update bcftools, samtools, ... nf-core modules. All indexing is now done with the file creation for most of them.
 - [#101](https://github.com/nf-core/phaseimpute/pull/101) - Set `--compute_freq` as `false` by default.
 
 ### `Fixed`

diff --git a/conf/steps/imputation_glimpse1.config b/conf/steps/imputation_glimpse1.config
@@ -44,14 +44,10 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_GLIMPSE1:BAM_GL_BCFTOOLS:BCFTOOLS_ANNOTATE' {
-        ext.args   = "--set-id '%CHROM:%POS:%REF:%ALT' -Oz"
+        ext.args   = ["--set-id '%CHROM:%POS:%REF:%ALT'", "-Oz", "--write-index=tbi"].join(' ')
         ext.prefix = { "${meta.id}.annotate" }
     }
 
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_GLIMPSE1:BAM_GL_BCFTOOLS:BCFTOOLS_INDEX' {
-        ext.args = "--tbi"
-    }
-
     // Impute the variants
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_GLIMPSE1:GLIMPSE_PHASE' {
         ext.args   = ["--impute-reference-only-variants"].join(' ')
@@ -85,11 +81,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_GLIMPSE1:BCFTOOLS_CONCAT' {
-        ext.args   = ["--ligate", "--output-type z"].join(' ')
+        ext.args   = ["--ligate", "--output-type z", "--write-index=tbi"].join(' ')
         ext.prefix = { "${meta.id}.glimpse1" }
     }
-
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_GLIMPSE1:BCFTOOLS_INDEX' {
-        ext.args     = "--tbi"
-    }
 }
diff --git a/conf/steps/imputation_glimpse2.config b/conf/steps/imputation_glimpse2.config
@@ -50,12 +50,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_GLIMPSE2:BCFTOOLS_CONCAT' {
-        ext.args   = ["--ligate", "--output-type z"].join(' ')
+        ext.args   = ["--ligate", "--output-type z", "--write-index=tbi"].join(' ')
         ext.prefix = { "${meta.id}.glimpse2" }
     }
-
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_GLIMPSE2:BCFTOOLS_INDEX' {
-        ext.args     = "--tbi"
-    }
-
 }
diff --git a/conf/steps/imputation_quilt.config b/conf/steps/imputation_quilt.config
@@ -26,23 +26,18 @@ process {
         publishDir = [enabled: false]
     }
 
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_QUILT:BCFTOOLS_INDEX_1' {
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_QUILT:BCFTOOLS_INDEX' {
         ext.args     = "--tbi"
         publishDir = [enabled: false]
     }
 
     // Annotate quilt imputed VCFs
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_QUILT:BCFTOOLS_ANNOTATE' {
-        ext.args     = "--set-id '%CHROM:%POS:%REF:%ALT' -Oz"
+        ext.args     = ["--set-id '%CHROM:%POS:%REF:%ALT'", "-Oz", "--write-index=tbi"].join(' ')
         ext.prefix = { "${meta.id}_${meta.chr}_${meta.chunk}.impute.annotate" }
         publishDir = [ enabled: false ]
     }
 
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_IMPUTE_QUILT:BCFTOOLS_INDEX_2' {
-        ext.args     = "--tbi"
-        publishDir = [ enabled: false ]
-    }
-
     // Concatenate quilt imputed VCFs
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_QUILT:.*' {
         publishDir = [
@@ -53,12 +48,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_QUILT:BCFTOOLS_CONCAT' {
-        ext.args = ["--ligate", "--output-type z",].join(' ')
+        ext.args = ["--ligate", "--output-type z", "--write-index=tbi"].join(' ')
         ext.prefix = { "${meta.id}.quilt" }
     }
-
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_QUILT:BCFTOOLS_INDEX' {
-        ext.args     = "--tbi"
-    }
-
 }
diff --git a/conf/steps/imputation_stitch.config b/conf/steps/imputation_stitch.config
@@ -37,11 +37,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_STITCH:BCFTOOLS_CONCAT' {
-        ext.args   = ["--ligate", "--output-type z"].join(' ')
-    }
-
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_STITCH:BCFTOOLS_INDEX' {
-        ext.args   = "--tbi"
+        ext.args   = ["--ligate", "--output-type z", "--write-index=tbi"].join(' ')
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_SAMPLES_BCFTOOLS:.*' {
@@ -53,7 +49,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_SAMPLES_BCFTOOLS:BCFTOOLS_PLUGINSPLIT' {
-        ext.args   = "-Oz"
+        ext.args   = ["--output-type z", "--write-index=tbi"].join(' ')
         ext.suffix = ".stitch"
     }
 

diff --git a/conf/steps/panel_prep.config b/conf/steps/panel_prep.config
@@ -22,7 +22,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CHR_CHECK:VCF_CHR_RENAME:BCFTOOLS_ANNOTATE' {
-        ext.args   = ["-Oz", "--no-version"].join(' ')
+        ext.args   = ["-Oz", "--no-version", "--write-index=tbi"].join(' ')
         ext.prefix = { "${meta.id}_chrrename" }
         publishDir = [ enabled: false ]
     }
@@ -41,23 +41,17 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_NORMALIZE_BCFTOOLS:BCFTOOLS_NORM' {
-        ext.args      = '-m +any --no-version --output-type z'
+        ext.args      = ["-m +any", "--no-version", "--output-type z", "--write-index=tbi"].join(' ')
         ext.prefix    = { "${meta.id}_${meta.chr}_multiallelic" }
         publishDir = [ enabled: false ]
     }
 
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_NORMALIZE_BCFTOOLS:BCFTOOLS_INDEX_1' {
-        ext.args     = "--tbi"
-        publishDir = [enabled: false]
-    }
-
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_NORMALIZE_BCFTOOLS:BCFTOOLS_VIEW' {
-        ext.args   = {
-            ["-v snps", "-m 2", "-M 2",
+        ext.args   = [
+            "-v snps", "-m 2", "-M 2",
             params.remove_samples ? "-s^${params.remove_samples}" : '',
-            "-Oz"
-            ].join(' ')
-        }
+            "--output-type z", "--write-index=tbi"
+        ].join(' ')
         ext.prefix = { "${meta.id}_${meta.chr}_biallelic_snps" }
         publishDir = [
             path: { "${params.outdir}/prep_panel/normalized" },
@@ -66,15 +60,6 @@ process {
         ]
     }
 
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_NORMALIZE_BCFTOOLS:BCFTOOLS_INDEX_2' {
-        ext.args     = "--tbi"
-        publishDir = [
-            path: { "${params.outdir}/prep_panel/normalized" },
-            saveAs: { filename -> params.compute_freq ? null : filename.equals('versions.yml') ? null : filename },
-            enabled: true
-        ]
-    }
-
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_NORMALIZE_BCFTOOLS:VCFLIB_VCFFIXUP' {
         ext.prefix   = { "${meta.id}_${meta.chr}" }
         publishDir = [
@@ -84,7 +69,7 @@ process {
         ]
     }
 
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_NORMALIZE_BCFTOOLS:BCFTOOLS_INDEX_3' {
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_NORMALIZE_BCFTOOLS:BCFTOOLS_INDEX' {
         ext.args     = "--tbi"
         publishDir = [
             path: { "${params.outdir}/prep_panel/normalized" },
@@ -135,6 +120,7 @@ process {
             "-M 2",
             "-v snps",
             "--output-type z",
+            "--write-index=tbi",
             "--no-version"
         ].join(' ')
         ext.prefix = { "${meta.id}_${meta.chr}_glimpse1_sites" }
@@ -156,62 +142,16 @@ process {
         ]
     }
 
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_SITES_EXTRACT_BCFTOOLS:BCFTOOLS_QUERY' {
-        ext.args   = ["-f'%CHROM\t%POS\t%REF,%ALT\\n'"].join(' ')
-        ext.prefix = { "${meta.id}_${meta.chr}_glimpse1_sites_tsv" }
-        publishDir = [ enabled: false ]
-    }
-
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_SITES_EXTRACT_BCFTOOLS:GAWK' {
-        ext.args2  = "'{ gsub(\",\", \"\\t\") ; key = \$1 FS \$2 } !seen[key]++'" // Remove duplicates
-        ext.prefix = { "${meta.id}_${meta.chr}_posfile_stitch" }
-        ext.suffix = "txt"
-        publishDir = [
-            path: { "${params.outdir}/prep_panel/posfile/" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-            enabled: true
-        ]
-    }
-
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_SITES_EXTRACT_BCFTOOLS:TABIX_BGZIP' {
-        ext.prefix = { "${meta.id}_${meta.chr}_glimpse1_sites_tsv" }
-        publishDir = [
-            path: { "${params.outdir}/prep_panel/sites/tsv/" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-            enabled: true
-        ]
-    }
-
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_SITES_EXTRACT_BCFTOOLS:TABIX_TABIX' {
-        ext.args = [
-            "-s1",
-            "-b2",
-            "-e2"
-        ].join(' ')
-        ext.prefix = { "${meta.id}_${meta.chr}_glimpse1_sites_tsv_gz" }
-        publishDir = [
-            path: { "${params.outdir}/prep_panel/sites/tsv/" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-            enabled: true
-        ]
-    }
-
     // Subworkflow: Concat phased panel
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_PANEL:.*' {
         publishDir = [ enabled: false ]
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_PANEL:BCFTOOLS_CONCAT' {
+        ext.args   = ["--ligate", "--output-type z", "--write-index=tbi"].join(' ')
         ext.prefix = { "${meta.id}.panel" }
     }
 
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_PANEL:BCFTOOLS_INDEX' {
-        ext.args = "--tbi"
-    }
-
     // Subworkflow: Make chunks
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CHUNK_GLIMPSE:.*' {
         publishDir = [

diff --git a/conf/steps/simulation.config b/conf/steps/simulation.config
@@ -17,15 +17,15 @@ process {
         tag = {"${meta.id} ${meta.chr}"}
     }
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_REGION:SAMTOOLS_VIEW' {
-        ext.args   = "--output-fmt bam"
+        ext.args   = ["--output-fmt bam", "--write-index"].join(' ')
         ext.prefix = { "${meta.id}_R${meta.region.replace(':','_')}" }
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_REGION:SAMTOOLS_MERGE' {
         ext.prefix = { "${meta.id}" }
         tag = {"${meta.id} ${meta.chr}"}
     }
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_REGION:SAMTOOLS_INDEX_2' {
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_REGION:SAMTOOLS_INDEX' {
         ext.args = ""
     }
 
@@ -48,7 +48,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_DOWNSAMPLE:SAMTOOLS_VIEW' {
-        ext.args   = "--output-fmt bam"
+        ext.args   = ["--output-fmt bam", "--write-index"].join(' ')
         ext.prefix = { "${meta.id}_depth_${meta.depth}x" }
         publishDir = [
             path: { "${params.outdir}/simulation/" },
@@ -57,12 +57,4 @@ process {
         ]
         tag = {"${meta.id} ${meta.chr}"}
     }
-
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:BAM_DOWNSAMPLE:SAMTOOLS_INDEX' {
-        publishDir = [
-            path: { "${params.outdir}/simulation/" },
-            mode: params.publish_dir_mode,
-            saveAs: {  filename -> filename.equals('versions.yml') ? null : filename  }
-        ]
-    }
 }
diff --git a/conf/steps/validation.config b/conf/steps/validation.config
@@ -39,14 +39,10 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:GL_TRUTH:BCFTOOLS_ANNOTATE' {
-        ext.args   = ["--set-id '%CHROM:%POS:%REF:%ALT'", "-Oz"].join(' ')
+        ext.args   = ["--set-id '%CHROM:%POS:%REF:%ALT'", "-Oz", "--write-index=tbi"].join(' ')
         ext.prefix = { "${meta.id}_${meta.chr}.annotate" }
     }
 
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:GL_TRUTH:BCFTOOLS_INDEX' {
-        ext.args = "--tbi"
-    }
-
     // Concatenate the truth set
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_TRUTH:.*' {
         ext.prefix = { "${meta.id}.A-truth" }
@@ -58,11 +54,7 @@ process {
     }
 
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_TRUTH:BCFTOOLS_CONCAT' {
-        ext.args = ["--ligate", "--output-type z",].join(' ')
-    }
-
-    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:CONCAT_TRUTH:BCFTOOLS_INDEX' {
-        ext.args = "--tbi"
+        ext.args   = ["--ligate", "--output-type z", "--write-index=tbi"].join(' ')
     }
 
     // Validation subworkflow

diff --git a/conf/test_panelprep.config b/conf/test_panelprep.config
@@ -32,6 +32,14 @@ params {
 }
 
 process {
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CHUNK_GLIMPSE:GLIMPSE_CHUNK' {
+        ext.args = ["--window-size 10000", "--window-count 400", "--buffer-size 5000", "--buffer-count 30"].join(' ')
+    }
+
+    withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_CHUNK_GLIMPSE:GLIMPSE2_CHUNK' {
+        ext.args = ["--window-mb 0.01", "--window-cm 0.01", "--window-count 200", "--buffer-mb 0.005", "--buffer-cm 0.005", "--buffer-count 30"].join(' ')
+    }
+
     withName: 'NFCORE_PHASEIMPUTE:PHASEIMPUTE:VCF_PHASE_SHAPEIT5:GLIMPSE2_CHUNK' {
         ext.args = ["--window-mb 0.01", "--window-cm 0.01", "--window-count 200", "--buffer-mb 0.005", "--buffer-cm 0.005", "--buffer-count 30"].join(' ')
     }