Merge pull request #216 from genomic-medicine-sweden/cgl-pipeline-dev

Add cron to pipelines
genomic-medicine-sweden · Nov 9, 2023 · f7188a9 · f7188a9
2 parents a6c022c + 10b4ac3
commit f7188a9
Show file tree

Hide file tree

Showing 10 changed files with 58 additions and 8 deletions.
diff --git a/Makefile b/Makefile
@@ -162,7 +162,7 @@ install: download_or_build_containers \
 update_databases: update_amrfinderplus \
 	update_mlst_db \
 	update_blast_db \
-	update_virulencefinder_db \
+	update_finder_dbs \
 	saureus_all \
 	ecoli_all \
 	kpneumoniae_all \
@@ -244,7 +244,7 @@ $(MLSTDB_DIR)/blast:
 VIRULENCEFINDERDB_DIR := $(ASSETS_DIR)/virulencefinder_db
 KMA_DIR := $(ASSETS_DIR)/kma
 
-update_virulencefinder_db: $(VIRULENCEFINDERDB_DIR)/stx.name
+update_finder_dbs: $(VIRULENCEFINDERDB_DIR)/stx.name
 
 $(ASSETS_DIR)/virulencefinder_db/stx.name: | check-and-reinit-git-submodules
 	$(call log_message,"Starting update of VirulenceFinder database")

diff --git a/configs/nextflow.base.config b/configs/nextflow.base.config
@@ -15,6 +15,7 @@ params {
 	containerDir = "${root}/container"
 	args = ""
 	prefix = ""
+	cronCopy = false
 	useKraken = true
 	useSkesa = true // To use spades set useSkesa = false
 }
@@ -116,6 +117,9 @@ process {
 	withName: chewbbaca_split_results {
 		publishDir = [ path: "${params.outdir}/${params.speciesDir}/chewbbaca", mode: 'copy', overwrite: true ]
 	}
+	withName: copy_to_cron {
+		publishDir = [ path: "${params.outdir}/${params.speciesDir}/cron", mode: 'copy', overwrite: true ]
+	}
 	withName: create_analysis_result {
 		container = "${params.containerDir}/pythonScripts.sif"
 		publishDir = [ path: "${params.outdir}/${params.speciesDir}/analysis_result", mode: 'copy', overwrite: true ]

diff --git a/configs/nextflow.hopper.config b/configs/nextflow.hopper.config
@@ -25,6 +25,7 @@ params {
 	args = ""
 	prefix = ""
 	// PIPELINE OPTIONS //
+	cronCopy = true
 	useKraken = true
 	useSkesa = true // To use spades set useSkesa = false
 	queue='grace-high'
@@ -129,12 +130,15 @@ process {
 	withName: chewbbaca_split_results {
 		publishDir = [ path: "${params.outdir}/${params.speciesDir}/chewbbaca", mode: 'copy', overwrite: true ]
 	}
+	withName: copy_to_cron {
+		publishDir = [ path: "/fs1/results/cron/jasen", mode: 'copy', overwrite: true ]
+	}
 	withName: create_analysis_result {
 		container = "${params.containerDir}/pythonScripts.sif"
 		publishDir = [ path: "${params.outdir}/${params.speciesDir}/analysis_result", mode: 'copy', overwrite: true ]
 	}
 	withName: export_to_cdm {
-		publishDir = [ path: "/fs1/results_dev/cron/qc", mode: 'copy', overwrite: true ]
+		publishDir = [ path: "/fs1/results/cron/qc", mode: 'copy', overwrite: true ]
 	}
 	withName: freebayes {
 		container = "${params.containerDir}/freebayes.sif"

diff --git a/configs/nextflow.ngp.config b/configs/nextflow.ngp.config
@@ -1,6 +1,6 @@
 params {
-        queue = "all.q"
-        penv = "mpi"
+	queue = "all.q"
+	penv = "mpi"
 	root = "/gms_home/xsylis/code/JASEN" //edit
 	amrfinderDb = "${root}/assets/amrfinder_db/latest"
 	resfinderDb = "${params.root}/assets/resfinder_db"
@@ -17,6 +17,7 @@ params {
 	containerDir = "${root}/container"
 	args = ""
 	prefix = ""
+	cronCopy = false
 	useKraken = false
 	useSkesa = false // To use spades set useSkesa = false
 }
@@ -121,6 +122,9 @@ process {
 	withName: chewbbaca_split_results {
 		publishDir = [ path: "${params.outdir}/${params.speciesDir}/chewbbaca", mode: 'copy', overwrite: true ]
 	}
+	withName: copy_to_cron {
+		publishDir = [ path: "${params.outdir}/${params.speciesDir}/cron", mode: 'copy', overwrite: true ]
+	}
 	withName: create_analysis_result {
 		container = "${params.containerDir}/pythonScripts.sif"
 		publishDir = [ path: "${params.outdir}/${params.speciesDir}/analysis_result", mode: 'copy', overwrite: true ]

diff --git a/configs/nextflow.selftest.config b/configs/nextflow.selftest.config
@@ -15,6 +15,7 @@ params {
 	containerDir = "${root}/container"
 	args = ""
 	prefix = ""
+	copyCron = false
 	useKraken = false
 	useSkesa = false // To use spades set useSkesa = false
 }
@@ -120,6 +121,9 @@ process {
 	withName: chewbbaca_split_results {
 		publishDir = [ path: "${params.outdir}/${params.speciesDir}/chewbbaca", mode: 'copy', overwrite: true ]
 	}
+	withName: copy_to_cron {
+		publishDir = [ path: "${params.outdir}/${params.speciesDir}/cron", mode: 'copy', overwrite: true ]
+	}
 	withName: create_analysis_result {
 		container = "${params.containerDir}/pythonScripts.sif"
 		publishDir = [ path: "${params.outdir}/${params.speciesDir}/analysis_result", mode: 'copy', overwrite: true ]

diff --git a/nextflow-modules/modules/cron/main.nf b/nextflow-modules/modules/cron/main.nf
@@ -0,0 +1,25 @@
+process copy_to_cron {
+  tag "${sampleName}"
+  scratch params.scratch
+
+  input:
+    tuple val(sampleName), val(prp)
+
+  output:
+    tuple val(sampleName), path(output), emit: json
+
+  when:
+    params.cronCopy
+
+  script:
+    output = "${sampleName}_result.json"
+    """
+    cp ${prp} ${output}
+    """
+
+  stub:
+    output = "${sampleName}_result.json"
+    """
+    touch $output
+    """
+}
diff --git a/workflows/escherichia_coli.nf b/workflows/escherichia_coli.nf
@@ -10,6 +10,7 @@ include { bwa_mem as bwa_mem_dedup                  } from '../nextflow-modules/
 include { chewbbaca_allelecall                      } from '../nextflow-modules/modules/chewbbaca/main'
 include { chewbbaca_create_batch_list               } from '../nextflow-modules/modules/chewbbaca/main'
 include { chewbbaca_split_results                   } from '../nextflow-modules/modules/chewbbaca/main'
+include { copy_to_cron                              } from '../nextflow-modules/modules/cron/main'
 include { create_analysis_result                    } from '../nextflow-modules/modules/prp/main'
 include { export_to_cdm                             } from '../nextflow-modules/modules/cmd/main'
 include { freebayes                                 } from '../nextflow-modules/modules/freebayes/main'
@@ -120,7 +121,6 @@ workflow CALL_ESCHERICHIA_COLI {
             .join(ch_metadata)
             .join(ch_empty)
             .join(ch_empty)
-            .join(ch_empty)
             .set{ combinedOutput }
 
         if ( params.useKraken ) {
@@ -137,6 +137,8 @@ workflow CALL_ESCHERICHIA_COLI {
             create_analysis_result(combinedOutput)
         }
 
+        copy_to_cron(create_analysis_result.out.json)
+
         ch_versions = ch_versions.mix(amrfinderplus.out.versions)
         ch_versions = ch_versions.mix(bwa_index.out.versions)
         ch_versions = ch_versions.mix(bwa_mem_dedup.out.versions)

diff --git a/workflows/klebsiella_pneumoniae.nf b/workflows/klebsiella_pneumoniae.nf
@@ -10,6 +10,7 @@ include { bwa_mem as bwa_mem_dedup                  } from '../nextflow-modules/
 include { chewbbaca_allelecall                      } from '../nextflow-modules/modules/chewbbaca/main'
 include { chewbbaca_create_batch_list               } from '../nextflow-modules/modules/chewbbaca/main'
 include { chewbbaca_split_results                   } from '../nextflow-modules/modules/chewbbaca/main'
+include { copy_to_cron                              } from '../nextflow-modules/modules/cron/main'
 include { create_analysis_result                    } from '../nextflow-modules/modules/prp/main'
 include { export_to_cdm                             } from '../nextflow-modules/modules/cmd/main'
 include { freebayes                                 } from '../nextflow-modules/modules/freebayes/main'
@@ -120,7 +121,6 @@ workflow CALL_KLEBSIELLA_PNEUMONIAE {
             .join(ch_metadata)
             .join(ch_empty)
             .join(ch_empty)
-            .join(ch_empty)
             .set{ combinedOutput }
 
         if ( params.useKraken ) {
@@ -137,6 +137,8 @@ workflow CALL_KLEBSIELLA_PNEUMONIAE {
             create_analysis_result(combinedOutput)
         }
 
+        copy_to_cron(create_analysis_result.out.json)
+
         ch_versions = ch_versions.mix(amrfinderplus.out.versions)
         ch_versions = ch_versions.mix(bwa_index.out.versions)
         ch_versions = ch_versions.mix(bwa_mem_dedup.out.versions)

diff --git a/workflows/mycobacterium_tuberculosis.nf b/workflows/mycobacterium_tuberculosis.nf
@@ -4,6 +4,7 @@ nextflow.enable.dsl=2
 
 include { get_meta                  } from '../methods/get_meta'
 include { bracken                   } from '../nextflow-modules/modules/bracken/main'
+include { copy_to_cron              } from '../nextflow-modules/modules/cron/main'
 include { create_analysis_result    } from '../nextflow-modules/modules/prp/main'
 include { kraken                    } from '../nextflow-modules/modules/kraken/main'
 include { mykrobe                   } from '../nextflow-modules/modules/mykrobe/main'
@@ -67,6 +68,8 @@ workflow CALL_MYCOBACTERIUM_TUBERCULOSIS {
             create_analysis_result(combinedOutput)
         }
 
+        copy_to_cron(create_analysis_result.out.json)
+
         ch_versions = ch_versions.mix(CALL_BACTERIAL_BASE.out.versions)
         ch_versions = ch_versions.mix(mykrobe.out.versions)
         ch_versions = ch_versions.mix(snippy.out.versions)

diff --git a/workflows/staphylococcus_aureus.nf b/workflows/staphylococcus_aureus.nf
@@ -10,6 +10,7 @@ include { bwa_mem as bwa_mem_dedup                  } from '../nextflow-modules/
 include { chewbbaca_allelecall                      } from '../nextflow-modules/modules/chewbbaca/main'
 include { chewbbaca_create_batch_list               } from '../nextflow-modules/modules/chewbbaca/main'
 include { chewbbaca_split_results                   } from '../nextflow-modules/modules/chewbbaca/main'
+include { copy_to_cron                              } from '../nextflow-modules/modules/cron/main'
 include { create_analysis_result                    } from '../nextflow-modules/modules/prp/main'
 include { export_to_cdm                             } from '../nextflow-modules/modules/cmd/main'
 include { freebayes                                 } from '../nextflow-modules/modules/freebayes/main'
@@ -120,7 +121,6 @@ workflow CALL_STAPHYLOCOCCUS_AUREUS {
             .join(ch_metadata)
             .join(ch_empty)
             .join(ch_empty)
-            .join(ch_empty)
             .set{ combinedOutput }
 
         if ( params.useKraken ) {
@@ -137,6 +137,8 @@ workflow CALL_STAPHYLOCOCCUS_AUREUS {
             create_analysis_result(combinedOutput)
         }
 
+        copy_to_cron(create_analysis_result.out.json)
+
         ch_versions = ch_versions.mix(amrfinderplus.out.versions)
         ch_versions = ch_versions.mix(bwa_index.out.versions)
         ch_versions = ch_versions.mix(bwa_mem_dedup.out.versions)