Use methods.setup_process_afterscript() for process logs

CHANGELOG.md

@@ -12,8 +12,11 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 ---
 
 ## [1.0.1] - 2024-05-29
+
 ### [Changed]
 - Update Nextflow configuration test workflows
+- Use `methods.setup_process_afterscript()` for process logs
+
 ### [Fixed]
 - Resolve interval path for original given intervals to ensure the index file for intervals is found when present
 

config/methods.config

@@ -130,6 +130,7 @@ methods {
         methods.setup_docker_cpus()
         methods.verify_input_deletion()
         methods.set_recal_tables()
+        methods.setup_process_afterscript()
 
         json_extractor.store_object_as_json(
             params,

module/base-recalibration.nf

@@ -40,10 +40,7 @@ process run_BaseRecalibrator_GATK {
       mode: "copy",
       pattern: "*.grp"
 
-    publishDir path: "${params.log_output_dir}/process-log",
-      pattern: ".command.*",
-      mode: "copy",
-      saveAs: { "${task.process.replace(':', '/')}-${sample_id}/log${file(it).getName()}" }
+    ext log_dir_suffix: { "-${sample_id}" }
 
     input:
     path(reference_fasta)
@@ -60,7 +57,6 @@ process run_BaseRecalibrator_GATK {
     tuple path(indelrealigned_bams), path(indelrealigned_bams_bai), val(sample_id)
 
     output:
-    path(".command.*")
     tuple val(sample_id), path("${sample_id}_recalibration_table.grp"), emit: recalibration_table
 
     script:
@@ -115,10 +111,7 @@ process run_ApplyBQSR_GATK {
       enabled: params.save_intermediate_files,
       pattern: "*_recalibrated-*"
 
-    publishDir path: "${params.log_output_dir}/process-log",
-      pattern: ".command.*",
-      mode: "copy",
-      saveAs: { "${task.process.replace(':', '/')}-${interval_id}/log${file(it).getName()}" }
+    ext log_dir_suffix: { "-${interval_id}" }
 
     input:
     path(reference_fasta)
@@ -133,7 +126,6 @@ process run_ApplyBQSR_GATK {
           path(recalibration_table)
 
     output:
-    path(".command.*")
     path("*_recalibrated-*"), emit: output_ch_apply_bqsr
     tuple path(indelrealigned_bam), path(indelrealigned_bam_index), emit: output_ch_deletion
 

module/checksum.nf

@@ -16,16 +16,12 @@ process calculate_sha512 {
       pattern: "*.sha512",
       saveAs: { filename -> (filename.endsWith(".bai.sha512") && !filename.endsWith(".bam.bai.sha512")) ? "${file(file(filename).baseName).baseName}.bam.bai.sha512" : "${filename}"}
 
-    publishDir path: "${params.log_output_dir}/process-log",
-      pattern: ".command.*",
-      mode: "copy",
-      saveAs: { "${task.process.replace(':', '/')}/${task.process.split(':')[-1]}-${task.index}/log${file(it).getName()}" }
+    ext log_dir_suffix: { "/${task.process.split(':')[-1]}-${task.index}" }
 
     input:
     path(file_for_calc)
 
     output:
-    path(".command.*")
     path("*.sha512"), emit: sha512_sum
 
     script:

module/delete-input.nf

@@ -29,17 +29,13 @@ process check_deletion_status {
     container params.docker_image_pipeval
     containerOptions "--volume ${get_root_directory(params.metapipeline_final_output_dir)}:${get_root_directory(params.metapipeline_final_output_dir)}"
 
-    publishDir path: "${params.log_output_dir}/process-log",
-        pattern: ".command.*",
-        mode: "copy",
-        saveAs: { "${task.process.replace(':', '/')}-${task.index}/log${file(it).getName()}" }
+    ext log_dir_suffix: { "-${task.index}" }
 
     input:
     path(file_to_check)
 
     output:
     tuple path(file_to_check), env(DELETE_INPUT), emit: file_to_delete
-    path(".command.*")
 
     when:
     params.metapipeline_delete_input_bams

module/indel-realignment.nf

@@ -28,10 +28,7 @@ process run_RealignerTargetCreator_GATK {
       enabled: params.save_intermediate_files,
       pattern: "*_RTC_*.intervals"
 
-    publishDir path: "${params.log_output_dir}/process-log",
-      pattern: ".command.*",
-      mode: "copy",
-      saveAs: { "${task.process.replace(':', '/')}-${interval_id}/log${file(it).getName()}" }
+    ext log_dir_suffix: { "-${interval_id}" }
 
     input:
     path(reference_fasta)
@@ -45,7 +42,6 @@ process run_RealignerTargetCreator_GATK {
     tuple path(bam), path(bam_index), val(interval_id), path(interval)
 
     output:
-    path(".command.*")
     tuple path(bam), path(bam_index), val(interval_id), path(interval), path("${output_rtc_intervals}"), emit: ir_targets
 
     script:
@@ -101,10 +97,7 @@ process run_IndelRealigner_GATK {
       pattern: "*indelrealigned*",
       saveAs: { filename -> (file(filename).getExtension() == "bai") ? "${file(filename).baseName}.bam.bai" : "${filename}" }
 
-    publishDir path: "${params.log_output_dir}/process-log",
-      pattern: ".command.*",
-      mode: "copy",
-      saveAs: { "${task.process.replace(':', '/')}-${interval_id}/log${file(it).getName()}" }
+    ext log_dir_suffix: { "-${interval_id}" }
 
     input:
     path(reference_fasta)
@@ -117,7 +110,6 @@ process run_IndelRealigner_GATK {
     tuple path(bam), path(bam_index), val(interval_id), path(scatter_intervals), path(target_intervals_RTC)
 
     output:
-    path(".command.*")
     tuple path("${output_filename}.bam"), path("${output_filename}.bai"), val(interval_id), path(scatter_intervals), val(has_unmapped), emit: output_ch_indel_realignment
 
     script:

module/merge-bam.nf

@@ -36,16 +36,12 @@ process run_MergeSamFiles_Picard {
         enabled: params.parallelize_by_chromosome,
         pattern: "${output_file_name}"
 
-    publishDir path: "${params.log_output_dir}/process-log",
-        pattern: ".command.*",
-        mode: "copy",
-        saveAs: { "${task.process.replace(':', '/')}-${sample_id}/log${file(it).getName()}" }
+    ext log_dir_suffix: { "-${sample_id}" }
 
     input:
     tuple val(sample_id), path(bams)
 
     output:
-    path(".command.*")
     tuple val(sample_id), path(output_file_name), emit: merged_bam
     path(bams), emit: output_ch_deletion
 
@@ -101,10 +97,7 @@ process deduplicate_records_SAMtools {
         mode: "copy",
         pattern: "${output_file_name}"
 
-    publishDir path: "${params.log_output_dir}/process-log",
-        pattern: ".command.*",
-        mode: "copy",
-        saveAs: { "${task.process.replace(':', '/')}-${sample_id}/log${file(it).getName()}" }
+    ext log_dir_suffix: { "-${sample_id}" }
 
     when:
     !params.parallelize_by_chromosome
@@ -113,7 +106,6 @@ process deduplicate_records_SAMtools {
     tuple val(sample_id), path(bam)
 
     output:
-    path(".command.*")
     tuple val(sample_id), path(output_file_name), emit: dedup_bam
     path(bam), emit: bam_for_deletion
 
@@ -156,16 +148,12 @@ process run_index_SAMtools {
         mode: "copy",
         pattern: "*.bai"
 
-    publishDir path: "${params.log_output_dir}/process-log",
-        pattern: ".command.*",
-        mode: "copy",
-        saveAs: { "${task.process.replace(':', '/')}-${sample_id}/log${file(it).getName()}" }
+    ext log_dir_suffix: { "-${sample_id}" }
 
     input:
     tuple val(sample_id), path(bam)
 
     output:
-    path(".command.*")
     tuple val(sample_id), path(bam), path("${bam}.bai"), emit: indexed_out
 
     script:

module/split-intervals.nf

@@ -20,10 +20,6 @@ process run_SplitIntervals_GATK {
                mode: "copy",
                pattern: "*-contig.interval_list",
                enabled: params.save_intermediate_files
-    publishDir "${params.log_output_dir}/process-log",
-               mode: "copy",
-               pattern: ".command.*",
-               saveAs: { "${task.process.replace(':', '/')}/log${file(it).getName()}" }
 
     input:
     path intervals
@@ -33,7 +29,6 @@ process run_SplitIntervals_GATK {
 
     output:
     path "*-contig.interval_list", emit: interval_list
-    path ".command.*"
 
     script:
     """

module/summary-qc.nf

@@ -23,10 +23,7 @@ process run_GetPileupSummaries_GATK {
       mode: "copy",
       pattern: '*.table'
 
-    publishDir path: "${params.log_output_dir}/process-log",
-      pattern: ".command.*",
-      mode: "copy",
-      saveAs: { "${task.process.replace(':', '/')}-${sample_id}/log${file(it).getName()}" }
+    ext log_dir_suffix: { "-${sample_id}" }
 
     input:
     path(reference_fasta)
@@ -38,7 +35,6 @@ process run_GetPileupSummaries_GATK {
     tuple val(sample_id), path(bam), path(bam_index)
 
     output:
-    path(".command.*")
     tuple val(sample_id), path("*getpileupsummaries.table"), emit: pileupsummaries
 
     script:
@@ -82,16 +78,12 @@ process run_CalculateContamination_GATK {
       mode: "copy",
       pattern: '*.table'
 
-    publishDir path: "${params.log_output_dir}/process-log",
-      pattern: ".command.*",
-      mode: "copy",
-      saveAs: { "${task.process.replace(':', '/')}-${sample_id}/log${file(it).getName()}" }
+    ext log_dir_suffix: { "-${sample_id}" }
 
     input:
     tuple val(normal_id), path(normal_pileup), val(tumor_id), path(tumor_pileup), val(sample_type)
 
     output:
-    path(".command.*")
     path("*-tumor-segmentation.table")
     path("*_alone.table"), emit: contamination
     path("*_with-matched-normal.table"), emit: tumor_normal_matched_contamination optional true
@@ -148,10 +140,7 @@ process run_DepthOfCoverage_GATK {
       mode: "copy",
       pattern: '*_DOC*'
 
-    publishDir path: "${params.log_output_dir}/process-log",
-      pattern: ".command.*",
-      mode: "copy",
-      saveAs: { "${task.process.replace(':', '/')}-${sample_id}/log${file(it).getName()}" }
+    ext log_dir_suffix: { "-${sample_id}" }
 
     input:
     path(reference_fasta)
@@ -161,7 +150,6 @@ process run_DepthOfCoverage_GATK {
     tuple val(sample_id), path(bam), path(bam_index)
 
     output:
-    path(".command.*")
     path("*_DOC*")
 
     when:

test/configtest-F16.json

@@ -241,6 +241,12 @@
       "work_dir": "/scratch/851543"
     },
     "process": {
+      "afterScript": {
+        "1": "",
+        "2": "",
+        "3": "",
+        "closure": ""
+      },
       "cache": false,
       "containerOptions": {
         "1": "--cpu-shares 1024 --cpus $task.cpus",
@@ -261,6 +267,21 @@
         "closure": "terminate"
       },
       "executor": "local",
+      "ext": {
+        "capture_logs": true,
+        "commonAfterScript": {
+          "1": "",
+          "2": "",
+          "3": "",
+          "closure": ""
+        },
+        "log_dir": {
+          "1": "ext",
+          "2": "ext",
+          "3": "ext",
+          "closure": "ext"
+        }
+      },
       "maxRetries": "1",
       "memory": "31 GB",
       "withLabel:process_high": {

test/configtest-F32.json

@@ -241,6 +241,12 @@
       "work_dir": "/scratch/851543"
     },
     "process": {
+      "afterScript": {
+        "1": "",
+        "2": "",
+        "3": "",
+        "closure": ""
+      },
       "cache": false,
       "containerOptions": {
         "1": "--cpu-shares 1024 --cpus $task.cpus",
@@ -261,6 +267,21 @@
         "closure": "terminate"
       },
       "executor": "local",
+      "ext": {
+        "capture_logs": true,
+        "commonAfterScript": {
+          "1": "",
+          "2": "",
+          "3": "",
+          "closure": ""
+        },
+        "log_dir": {
+          "1": "ext",
+          "2": "ext",
+          "3": "ext",
+          "closure": "ext"
+        }
+      },
       "maxRetries": "1",
       "memory": "64 GB",
       "withLabel:process_high": {