Merge pull request #553 from nf-core/nf-core-template-merge-2.11

Important! Template update for nf-core/tools v2.11

.github/CONTRIBUTING.md

@@ -27,6 +27,9 @@ If you're not used to this workflow with git, you can start with some [docs from
 
 ## Tests
 
+You can optionally test your changes by running the pipeline locally. Then it is recommended to use the `debug` profile to
+receive warnings about process selectors and other debug info. Example: `nextflow run . -profile debug,test,docker --outdir <OUTDIR>`.
+
 When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests.
 Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then.
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -19,6 +19,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/mag/
 - [ ] If necessary, also make a PR on the nf-core/mag _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.
 - [ ] Make sure your code lints (`nf-core lint`).
 - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir <OUTDIR>`).
+- [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir <OUTDIR>`).
 - [ ] Usage Documentation in `docs/usage.md` is updated.
 - [ ] Output Documentation in `docs/output.md` is updated.
 - [ ] `CHANGELOG.md` is updated.

.github/workflows/ci.yml

@@ -33,7 +33,7 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
 
       - name: Check out pipeline code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v1

.github/workflows/fix-linting.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # Use the @nf-core-bot token to check out so we can push later
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           token: ${{ secrets.nf_core_bot_auth_token }}
 
@@ -24,7 +24,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
 
       - name: Install Prettier
         run: npm install -g prettier @prettier/plugin-php

.github/workflows/linting.yml

@@ -14,9 +14,9 @@ jobs:
   EditorConfig:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
 
       - name: Install editorconfig-checker
         run: npm install -g editorconfig-checker
@@ -27,9 +27,9 @@ jobs:
   Prettier:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
 
       - name: Install Prettier
         run: npm install -g prettier
@@ -40,7 +40,7 @@ jobs:
   PythonBlack:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Check code lints with Black
         uses: psf/black@stable
@@ -71,7 +71,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v1

.gitpod.yml

@@ -4,7 +4,9 @@ tasks:
     command: |
       pre-commit install --install-hooks
       nextflow self-update
-
+  - name: unset JAVA_TOOL_OPTIONS
+    command: |
+      unset JAVA_TOOL_OPTIONS
 vscode:
   extensions: # based on nf-core.nf-core-extensionpack
     - codezombiech.gitignore # Language support for .gitignore files

README.md

@@ -39,11 +39,8 @@ Furthermore, the pipeline creates various reports in the results directory speci
 
 ## Usage
 
-:::note
-If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how
-to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline)
-with `-profile test` before running the workflow on actual data.
-:::
+> [!NOTE]
+> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data.
 
 ```bash
 nextflow run nf-core/mag -profile <docker/singularity/podman/shifter/charliecloud/conda/institute> --input '*_R{1,2}.fastq.gz' --outdir <OUTDIR>
@@ -55,11 +52,9 @@ or
 nextflow run nf-core/mag -profile <docker/singularity/podman/shifter/charliecloud/conda/institute> --input samplesheet.csv --outdir <OUTDIR>
 ```
 
-:::warning
-Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those
-provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_;
-see [docs](https://nf-co.re/usage/configuration#custom-configuration-files).
-:::
+> [!WARNING]
+> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_;
+> see [docs](https://nf-co.re/usage/configuration#custom-configuration-files).
 
 For more details and further functionality, please refer to the [usage documentation](https://nf-co.re/mag/usage) and the [parameter documentation](https://nf-co.re/mag/parameters).
 
@@ -110,7 +105,7 @@ If you use nf-core/mag for your analysis, please cite the preprint as follows:
 >
 > NAR Genom Bioinform. 2022 Feb 2;4(1):lqac007. doi: [10.1093/nargab/lqac007](https://doi.org/10.1093/nargab/lqac007).
 
-additionally you can cite the pipeline directly with the following doi: [10.5281/zenodo.3589527](https://doi.org/10.5281/zenodo.3589527)
+Additionally you can cite the pipeline directly with the following doi: [10.5281/zenodo.3589527](https://doi.org/10.5281/zenodo.3589527)
 
 An extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file.
 

assets/multiqc_config.yml

@@ -1,9 +1,7 @@
 report_comment: >
-
-  This report has been generated by the <a href="https://github.com/nf-core/mag/releases/tag/dev" target="_blank">nf-core/mag</a>
+  This report has been generated by the <a href="https://github.com/nf-core/mag/tree/dev" target="_blank">nf-core/mag</a>
   analysis pipeline. For information about how to interpret these results, please see the
   <a href="https://nf-co.re/mag/dev/docs/output" target="_blank">documentation</a>.
-
 report_section_order:
   "nf-core-mag-methods-description":
     order: -1000

assets/slackreport.json

@@ -3,7 +3,7 @@
         {
             "fallback": "Plain-text summary of the attachment.",
             "color": "<% if (success) { %>good<% } else { %>danger<%} %>",
-            "author_name": "nf-core/mag v${version} - ${runName}",
+            "author_name": "nf-core/mag ${version} - ${runName}",
             "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico",
             "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>",
             "fields": [

conf/modules.config

@@ -750,7 +750,7 @@ process {
     }
 
     withName: 'MULTIQC' {
-        ext.args   = params.multiqc_title ? "--title \"$params.multiqc_title\"" : ''
+        ext.args   = { params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' }
         publishDir = [
             path: { "${params.outdir}/multiqc" },
             mode: params.publish_dir_mode,

conf/test.config

@@ -26,7 +26,7 @@ params {
     skip_krona                    = true
     min_length_unbinned_contigs   = 1
     max_unbinned_contigs          = 2
-    busco_db                      = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz"
+    busco_db                      = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz"
     busco_clean                   = true
     skip_gtdbtk                   = true
     skip_concoct                  = true

conf/test_adapterremoval.config

@@ -27,7 +27,7 @@ params {
     skip_krona                  = true
     min_length_unbinned_contigs = 1
     max_unbinned_contigs        = 2
-    busco_db                    = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz"
+    busco_db                    = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz"
     skip_gtdbtk                 = true
     clip_tool                   = 'adapterremoval'
     skip_concoct                = true

conf/test_ancient_dna.config

@@ -26,7 +26,7 @@ params {
     skip_krona                           = true
     min_length_unbinned_contigs          = 1
     max_unbinned_contigs                 = 2
-    busco_db                             = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz"
+    busco_db                             = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz"
     skip_gtdbtk                          = true
     ancient_dna                          = true
     binning_map_mode                     = 'own'

conf/test_bbnorm.config

@@ -32,7 +32,7 @@ params {
     skip_krona                    = true
     min_length_unbinned_contigs   = 1
     max_unbinned_contigs          = 2
-    busco_db                      = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz"
+    busco_db                      = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz"
     busco_clean                   = true
     skip_gtdbtk                   = true
     bbnorm                        = true

conf/test_binrefinement.config

@@ -27,7 +27,7 @@ params {
     skip_krona                    = true
     min_length_unbinned_contigs   = 1
     max_unbinned_contigs          = 2
-    busco_db                      = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz"
+    busco_db                      = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz"
     skip_gtdbtk                   = true
     refine_bins_dastool           = true
     refine_bins_dastool_threshold = 0

conf/test_host_rm.config

@@ -24,7 +24,7 @@ params {
     input                       = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.host_rm.csv'
     min_length_unbinned_contigs = 1
     max_unbinned_contigs        = 2
-    busco_db                    = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz"
+    busco_db                    = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz"
     skip_gtdbtk                 = true
     skip_concoct                = true
 }

conf/test_hybrid.config

@@ -23,7 +23,7 @@ params {
     input                       = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.hybrid.csv'
     min_length_unbinned_contigs = 1
     max_unbinned_contigs        = 2
-    busco_db                    = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz"
+    busco_db                    = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz"
     skip_gtdbtk                 = true
     skip_concoct                = true
 }

docs/usage.md

@@ -39,7 +39,7 @@ At a minimum CSV file should contain the following columns:
 
 The path to `long_reads` and `short_reads_2` is optional. Valid examples could look like the following:
 
-```bash
+```csv title="samplesheet.csv"
 sample,group,short_reads_1,short_reads_2,long_reads
 sample1,0,data/sample1_R1.fastq.gz,data/sample1_R2.fastq.gz,data/sample1.fastq.gz
 sample2,0,data/sample2_R1.fastq.gz,data/sample2_R2.fastq.gz,data/sample2.fastq.gz
@@ -48,15 +48,15 @@ sample3,1,data/sample3_R1.fastq.gz,data/sample3_R2.fastq.gz,
 
 or
 
-```bash
+```csv title="samplesheet.csv"
 sample,group,short_reads_1,short_reads_2,long_reads
 sample1,0,data/sample1.fastq.gz,,
 sample2,0,data/sample2.fastq.gz,,
 ```
 
 or to additionally to perform run merging of two runs of sample1:
 
-```bash
+```csv title="samplesheet.csv"
 sample,run,group,short_reads_1,short_reads_2,long_reads
 sample1,1,0,data/sample1_R1.fastq.gz,data/sample1_R2.fastq.gz,data/sample1.fastq.gz
 sample1,2,0,data/sample1_R1.fastq.gz,data/sample1_R2.fastq.gz,data/sample1.fastq.gz
@@ -87,7 +87,7 @@ The assembly CSV file should contain the following columns:
 
 Where `id` is the ID of the assembly, group is the assembly/binning group (see samplesheet information section for more details), `assembler` is the assembler used to produce the assembly (one of `MEGAHIT`, `SPAdes`, or `SPAdesHybrid`), and `fasta` is the path to the assembly fasta file. Input fasta files can be compressed or uncompressed, but compressed assemblies will be automatically uncompressed for use within the pipeline. The exact information required for each supplied assembly depends on whether the assemblies provided are single assemblies or group-wise co-assemblies. For the following example `--input` CSV:
 
-```bash
+```csv title="samplesheet.csv"
 sample,group,short_reads_1,short_reads_2,long_reads
 sample1,0,data/sample1_R1.fastq.gz,data/sample1_R2.fastq.gz,
 sample2,0,data/sample2_R1.fastq.gz,data/sample2_R2.fastq.gz,
@@ -96,7 +96,7 @@ sample3,1,data/sample3_R1.fastq.gz,data/sample3_R2.fastq.gz,
 
 If the assemblies are single assemblies, then the `id` and `group` columns should match those supplied in the `-input` read CSV files for each read set:
 
-```bash
+```csv title="samplesheet.csv"
 id,group,assembler,fasta
 sample1,0,MEGAHIT,MEGAHIT-sample1.contigs.fa.gz
 sample1,0,SPAdes,SPAdes-sample1.fasta.gz
@@ -108,7 +108,7 @@ sample3,1,SPAdes,SPAdes-sample3.contigs.fasta.gz
 
 If the assemblies are co-assemblies, the parameter `--coassemble_group` should additionally be specified. In this case, the `id` column should uniquely identify the assembly, while `group` should match those specified in the `--input` CSV file:
 
-```bash
+```csv title="samplesheet.csv"
 id,group,assembler,fasta
 group-0,0,MEGAHIT,MEGAHIT-group-0.contigs.fa.gz
 group-0,0,SPAdes,SPAdes-group-0.contigs.fasta.gz

lib/NfcoreTemplate.groovy

@@ -4,6 +4,7 @@
 
 import org.yaml.snakeyaml.Yaml
 import groovy.json.JsonOutput
+import nextflow.extension.FilesEx
 
 class NfcoreTemplate {
 
@@ -143,12 +144,14 @@ class NfcoreTemplate {
             try {
                 if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') }
                 // Try to send HTML e-mail using sendmail
+                def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html")
+                sendmail_tf.withWriter { w -> w << sendmail_html }
                 [ 'sendmail', '-t' ].execute() << sendmail_html
                 log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-"
             } catch (all) {
                 // Catch failures and try with plaintext
                 def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ]
-                if ( mqc_report.size() <= max_multiqc_email_size.toBytes() ) {
+                if ( mqc_report != null && mqc_report.size() <= max_multiqc_email_size.toBytes() ) {
                     mail_cmd += [ '-A', mqc_report ]
                 }
                 mail_cmd.execute() << email_html
@@ -157,14 +160,16 @@ class NfcoreTemplate {
         }
 
         // Write summary e-mail HTML to a file
-        def output_d = new File("${params.outdir}/pipeline_info/")
-        if (!output_d.exists()) {
-            output_d.mkdirs()
-        }
-        def output_hf = new File(output_d, "pipeline_report.html")
+        def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html")
         output_hf.withWriter { w -> w << email_html }
-        def output_tf = new File(output_d, "pipeline_report.txt")
+        FilesEx.copyTo(output_hf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.html");
+        output_hf.delete()
+
+        // Write summary e-mail TXT to a file
+        def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt")
         output_tf.withWriter { w -> w << email_txt }
+        FilesEx.copyTo(output_tf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.txt");
+        output_tf.delete()
     }
 
     //
@@ -229,15 +234,14 @@ class NfcoreTemplate {
     // Dump pipeline parameters in a json file
     //
     public static void dump_parameters(workflow, params) {
-        def output_d = new File("${params.outdir}/pipeline_info/")
-        if (!output_d.exists()) {
-            output_d.mkdirs()
-        }
-
         def timestamp  = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')
-        def output_pf  = new File(output_d, "params_${timestamp}.json")
+        def filename   = "params_${timestamp}.json"
+        def temp_pf    = new File(workflow.launchDir.toString(), ".${filename}")
         def jsonStr    = JsonOutput.toJson(params)
-        output_pf.text = JsonOutput.prettyPrint(jsonStr)
+        temp_pf.text   = JsonOutput.prettyPrint(jsonStr)
+
+        FilesEx.copyTo(temp_pf.toPath(), "${params.outdir}/pipeline_info/params_${timestamp}.json")
+        temp_pf.delete()
     }
 
     //

modules.json

@@ -78,7 +78,7 @@
                     },
                     "custom/dumpsoftwareversions": {
                         "branch": "master",
-                        "git_sha": "05c280924b6c768d484c7c443dad5e605c4ff4b4",
+                        "git_sha": "bba7e362e4afead70653f84d8700588ea28d0f9e",
                         "installed_by": ["modules"]
                     },
                     "dastool/dastool": {
@@ -98,7 +98,7 @@
                     },
                     "fastqc": {
                         "branch": "master",
-                        "git_sha": "9a4517e720bc812e95b56d23d15a1653b6db4f53",
+                        "git_sha": "65ad3e0b9a4099592e1102e92e10455dc661cf53",
                         "installed_by": ["modules"]
                     },
                     "freebayes": {
@@ -168,7 +168,7 @@
                     },
                     "multiqc": {
                         "branch": "master",
-                        "git_sha": "a6e11ac655e744f7ebc724be669dd568ffdc0e80",
+                        "git_sha": "4ab13872435962dadc239979554d13709e20bf29",
                         "installed_by": ["modules"]
                     },
                     "prodigal": {