Merge remote-tracking branch 'upstream/dev' into rename-parameters

genomic-medicine-sweden · Nov 14, 2024 · 04cff8a · 04cff8a
2 parents c5facbd + 42ec814
commit 04cff8a
Show file tree

Hide file tree

Showing 186 changed files with 6,049 additions and 3,271 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,6 +11,8 @@ on:
 
 env:
   NXF_ANSI_LOG: false
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
+  NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
   NFTEST_VER: "0.9.2"
 
 concurrency:
@@ -54,30 +56,21 @@ jobs:
           git remote add upstream ${{ github.event.repository.clone_url }}
           git fetch upstream ${{ github.event.pull_request.base.ref }}
 
-      - name: Install Nextflow
+      - name: Set up Nextflow
         uses: nf-core/setup-nextflow@v2
         with:
           version: "${{ matrix.NXF_VER }}"
 
       - uses: nf-core/setup-nf-test@v1
         with:
           version: ${{ env.NFTEST_VER }}
+          install-pdiff: true
 
       - uses: actions/setup-python@v4
         with:
           python-version: "3.11"
           architecture: "x64"
 
-      - name: Cache pdiff
-        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4
-        id: cache-pip-pdiff
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-pdiff
-
-      - name: Install pdiff
-        run: python -m pip install --upgrade pip pdiff cryptography
-
       - name: Run nf-test
         run: |
           nf-test test \

diff --git a/.github/workflows/keep_a_changelog.yml b/.github/workflows/keep_a_changelog.yml
@@ -0,0 +1,15 @@
+name: "Changelog Reminder"
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
+
+jobs:
+  # Enforces the update of a changelog file on every pull request
+  changelog:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dangoslen/changelog-enforcer@v3
+        with:
+          changeLogPath: "CHANGELOG.md"
+          skipLabels: "Skip-Changelog"
diff --git a/.github/workflows/template_version_comment.yml b/.github/workflows/template_version_comment.yml
@@ -10,9 +10,11 @@ jobs:
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Read template version from .nf-core.yml
-        uses: pietrobolcato/action-read-yaml@1.0.0
+        uses: nichmor/minimal-read-yaml@v0.0.2
         id: read_yml
         with:
           config: ${{ github.workspace }}/.nf-core.yml
@@ -24,20 +26,20 @@ jobs:
 
       - name: Check nf-core outdated
         id: nf_core_outdated
-        run: pip list --outdated | grep nf-core
+        run: echo "OUTPUT=$(pip list --outdated | grep nf-core)" >> ${GITHUB_ENV}
 
       - name: Post nf-core template version comment
         uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2
         if: |
-          ${{ steps.nf_core_outdated.outputs.stdout }} =~ 'nf-core'
+          contains(env.OUTPUT, 'nf-core')
         with:
           repo-token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }}
           allow-repeats: false
           message: |
-            ## :warning: Newer version of the nf-core template is available.
-
-            Your pipeline is using an old version of the nf-core template: ${{ steps.read_yml.outputs['nf_core_version'] }}.
-            Please update your pipeline to the latest version.
-
-            For more documentation on how to update your pipeline, please see the [nf-core documentation](https://github.com/nf-core/tools?tab=readme-ov-file#sync-a-pipeline-with-the-template) and [Synchronisation documentation](https://nf-co.re/docs/contributing/sync).
-          #
+            > [!WARNING]
+            > Newer version of the nf-core template is available.
+            >
+            > Your pipeline is using an old version of the nf-core template: ${{ steps.read_yml.outputs['nf_core_version'] }}.
+            > Please update your pipeline to the latest version.
+            >
+            > For more documentation on how to update your pipeline, please see the [nf-core documentation](https://github.com/nf-core/tools?tab=readme-ov-file#sync-a-pipeline-with-the-template) and [Synchronisation documentation](https://nf-co.re/docs/contributing/sync).
diff --git a/.gitignore b/.gitignore
@@ -6,4 +6,5 @@ results/
 testing/
 testing*
 *.pyc
+null/
 .nf-test*
diff --git a/.nf-core.yml b/.nf-core.yml
@@ -27,7 +27,7 @@ lint:
   nextflow_config:
     - manifest.name
     - manifest.homePage
-nf_core_version: 3.0.1
+nf_core_version: 3.0.2
 org_path: null
 repository_type: pipeline
 template:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#450](https://github.com/genomic-medicine-sweden/nallo/pull/450) - Added ranking of SVs (and CNVs)
 - [#451](https://github.com/genomic-medicine-sweden/nallo/pull/451) - Added support for running methylation subworkflow without phasing
 - [#451](https://github.com/genomic-medicine-sweden/nallo/pull/451) - Added nf-test to methylation
+- [#491](https://github.com/genomic-medicine-sweden/nallo/pull/491) - Added a changelog reminder action
 
 ### `Changed`
 
@@ -82,7 +83,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#479](https://github.com/genomic-medicine-sweden/nallo/pull/479) - Replaced bgzip tabix with bcftools sort in rank variants to fix [#457](https://github.com/genomic-medicine-sweden/nallo/issues/457)
 - [#480](https://github.com/genomic-medicine-sweden/nallo/pull/480) - Updated ranking of SVs to work with multiple families per project
 - [#484](https://github.com/genomic-medicine-sweden/nallo/pull/484) - Updated metro map and added SVG version
+- [#485](https://github.com/genomic-medicine-sweden/nallo/pull/485) - Updated repeat expansion annotation to annotate per family instead of per sample
+- [#486](https://github.com/genomic-medicine-sweden/nallo/pull/486) - Updated nf-core modules
 - [#487](https://github.com/genomic-medicine-sweden/nallo/pull/487) - Changed CI tests to only run tests where changes have been made
+- [#489](https://github.com/genomic-medicine-sweden/nallo/pull/489) - Updated nf-core template to 3.0.2
+- [#493](https://github.com/genomic-medicine-sweden/nallo/pull/493) - Refactored `nallo.nf` to remove many nested ifs and easier to follow logic
+- [#493](https://github.com/genomic-medicine-sweden/nallo/pull/493) - Updated rank_variants dependencies with sv_annotation
 
 ### `Removed`
 
@@ -146,16 +152,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Module updates
 
-| Tool       | Old version | New version |
-| ---------- | ----------- | ----------- |
-| fqcrs      | 0.1.0       |
-| severus    |             | 1.1         |
-| longphase  |             | 1.7.3       |
-| genmod     | 3.8.2       | 3.9         |
-| WhatsHap   | 2.2         | 2.3         |
-| SVDB       |             | 2.8.1       |
-| hifiasm    | 0.19.8      | 0.20.0      |
-| HiFiCNV    | 0.1.7       | 1.0.0       |
+| Tool           | Old version | New version |
+| -------------- | ----------- | ----------- |
+| fqcrs          | 0.1.0       |
+| severus        |             | 1.1         |
+| longphase      |             | 1.7.3       |
+| genmod         | 3.8.2       | 3.9         |
+| WhatsHap       | 2.2         | 2.3         |
+| SVDB           |             | 2.8.1       |
+| hifiasm        | 0.19.8      | 0.20.0      |
+| HiFiCNV        | 0.1.7       | 1.0.0       |
+| samtools/faidx | 1.2         | 1.21        |
+| samtools/index | 1.2         | 1.21        |
+| samtools/merge | 1.2         | 1.21        |
+| stranger       | 0.9.1       | 0.9.2       |
+| multiqc        | 1.21        | 1.25.1      |
 
 > [!NOTE]
 > Version has been updated if both old and new version information is present.

diff --git a/conf/modules/annotate_repeat_expansions.config b/conf/modules/annotate_repeat_expansions.config
@@ -23,13 +23,13 @@ process {
     }
 
     withName: '.*:ANNOTATE_REPEAT_EXPANSIONS:COMPRESS_STRANGER' {
-        ext.prefix = { "${meta.id}_repeat_expansion_stranger" }
+        ext.prefix = { "${meta.id}_repeats_annotated" }
         ext.args   = [
             '--output-type z',
             '--write-index=tbi'
         ].join(' ')
         publishDir = [
-            path: { "${params.outdir}/repeat_annotation/stranger/${meta.id}" },
+            path: { "${params.outdir}/repeats/family/${meta.id}" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]

diff --git a/conf/modules/call_repeat_expansions.config b/conf/modules/call_repeat_expansions.config
@@ -31,15 +31,15 @@ process {
     withName: '.*:CALL_REPEAT_EXPANSIONS:SAMTOOLS_SORT_TRGT' {
         ext.prefix = { "${meta.id}_spanning_sorted" }
         publishDir = [
-            path: { "${params.outdir}/repeat_calling/trgt/single_sample/${meta.id}" },
+            path: { "${params.outdir}/repeats/sample/${meta.id}" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
 
     withName: '.*:CALL_REPEAT_EXPANSIONS:SAMTOOLS_INDEX_TRGT' {
         publishDir = [
-            path: { "${params.outdir}/repeat_calling/trgt/single_sample/${meta.id}" },
+            path: { "${params.outdir}/repeats/sample/${meta.id}" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
@@ -52,22 +52,23 @@ process {
             '--write-index=tbi'
         ].join(' ')
         publishDir = [
-            path: { "${params.outdir}/repeat_calling/trgt/single_sample/${meta.id}" },
+            path: { "${params.outdir}/repeats/sample/${meta.id}" },
             mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
 
     withName: '.*:CALL_REPEAT_EXPANSIONS:BCFTOOLS_MERGE' {
+        ext.prefix = { "${meta.id}_repeats" }
         ext.args = [
             '--output-type z',
             '--write-index=tbi',
             '--force-single'
         ].join(' ')
         publishDir = [
-            path: { "${params.outdir}/repeat_calling/trgt/multi_sample/${meta.id}" },
+            path: { "${params.outdir}/repeats/family/${meta.id}" },
             mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+            saveAs: { filename -> filename.equals('versions.yml') || !params.skip_repeat_annotation ? null : filename }
         ]
     }
 

diff --git a/conf/modules/short_variant_calling.config b/conf/modules/short_variant_calling.config
@@ -34,7 +34,7 @@ process {
     }
 
 
-    withName: '.*:SHORT_VARIANT_CALLING:DEEPVARIANT' {
+    withName: '.*:SHORT_VARIANT_CALLING:DEEPVARIANT_RUNDEEPVARIANT' {
         ext.prefix = { intervals ? "${meta.id}_${intervals}_deepvariant" : "${meta.id}_deepvariant" }
         ext.args = { [
             "--sample_name=${meta.id}",

diff --git a/docs/output.md b/docs/output.md
@@ -157,23 +157,27 @@ If the pipeline is run with phasing, the aligned reads will be happlotagged usin
 
 ### Repeats
 
-[TRGT](https://github.com/PacificBiosciences/trgt) is used to call repeats:
-
-| Path                                                      | Description                               |
-| --------------------------------------------------------- | ----------------------------------------- |
-| `repeat_calling/trgt/multi_sample/{project}/*.vcf.gz`     | Merged VCF file for all samples           |
-| `repeat_calling/trgt/multi_sample/{project}/*.vcf.gz.tbi` | Index of the VCF file                     |
-| `repeat_calling/trgt/single_sample/{sample}/*.vcf.gz`     | VCF file with called repeats for a sample |
-| `repeat_calling/trgt/single_sample/{sample}/*.vcf.gz.tbi` | Index of the VCF file                     |
-| `repeat_calling/trgt/single_sample/{sample}/*.bam`        | BAM file with sorted spanning reads       |
-| `repeat_calling/trgt/single_sample/{sample}/*.bai`        | Index of the BAM file                     |
-
-[Stranger](https://github.com/Clinical-Genomics/stranger) is used to annotate them:
-
-| Path                                               | Description                     |
-| -------------------------------------------------- | ------------------------------- |
-| `repeat_annotation/stranger/{sample}/*.vcf.gz`     | Annotated VCF file              |
-| `repeat_annotation/stranger/{sample}/*.vcf.gz.tbi` | Index of the annotated VCF file |
+[TRGT](https://github.com/PacificBiosciences/trgt) is used to call repeats.
+
+!!!note
+
+    Merged variants per family are only output without annotation if `--skip_repeat_annotation` is true. Variants per sample are always output without annotation.
+
+| Path                                                     | Description                               |
+| -------------------------------------------------------- | ----------------------------------------- |
+| `repeats/{family}/{family}_repeat_expansions.vcf.gz`     | Merged VCF file per family                |
+| `repeats/{family}/{family}_repeat_expansions.vcf.gz.tbi` | Index of the VCF file                     |
+| `repeats/sample/{sample}/{sample}_sorted.vcf.gz`         | VCF file with called repeats for a sample |
+| `repeats/sample/{sample}/{sample}_sorted.vcf.gz.tbi`     | Index of the VCF file                     |
+| `repeats/sample/{sample}/{sample}_spanning_sorted.bam`   | BAM file with sorted spanning reads       |
+| `repeats/sample/{sample}/{sample}_spanning_sorted.bai`   | Index of the BAM file                     |
+
+[Stranger](https://github.com/Clinical-Genomics/stranger) is used to annotate repeats.
+
+| Path                                                                         | Description                           |
+| ---------------------------------------------------------------------------- | ------------------------------------- |
+| `repeat_expansions/{family}/{family}_repeat_expansions_annotated.vcf.gz`     | Merged, annotated VCF file per family |
+| `repeat_expansions/{family}/{family}_repeat_expansions_annotated.vcf.gz.tbi` | Index of the VCF file                 |
 
 ### SNVs