From 828db0228652c826f87ff6f5410329231af1bbb8 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Sat, 30 Nov 2024 07:36:06 +0000 Subject: [PATCH 01/89] install subworkflows/nf-core/fastq_align_dedup_bwameth/ --- modules.json | 26 +- modules/nf-core/parabricks/fq2bammeth/main.nf | 68 ++++++ .../nf-core/parabricks/fq2bammeth/meta.yml | 95 ++++++++ .../parabricks/fq2bammeth/tests/main.nf.test | 123 ++++++++++ .../fq2bammeth/tests/main.nf.test.snap | 166 +++++++++++++ .../fq2bammeth/tests/nextflow.config | 8 + nextflow.config | 1 + subworkflows/local/bwameth/main.nf | 136 ----------- subworkflows/local/bwameth/nextflow.config | 35 --- subworkflows/local/bwameth/tests/main.nf.test | 68 ------ .../local/bwameth/tests/main.nf.test.snap | 27 --- .../nextflow.config | 4 - .../fastq_align_dedup_bismark/nextflow.config | 8 - .../nf-core/fastq_align_dedup_bwameth/main.nf | 163 +++++++++++++ .../fastq_align_dedup_bwameth/meta.yml | 116 +++++++++ .../tests/gpu.nf.test | 132 +++++++++++ .../tests/gpu.nf.test.snap | 149 ++++++++++++ .../tests/main.nf.test | 179 ++++++++++++++ .../tests/main.nf.test.snap | 224 ++++++++++++++++++ .../tests/nextflow.config | 14 ++ 20 files changed, 1456 insertions(+), 286 deletions(-) create mode 100644 modules/nf-core/parabricks/fq2bammeth/main.nf create mode 100644 modules/nf-core/parabricks/fq2bammeth/meta.yml create mode 100644 modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test create mode 100644 modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test.snap create mode 100644 modules/nf-core/parabricks/fq2bammeth/tests/nextflow.config delete mode 100644 subworkflows/local/bwameth/main.nf delete mode 100644 subworkflows/local/bwameth/nextflow.config delete mode 100644 subworkflows/local/bwameth/tests/main.nf.test delete mode 100644 subworkflows/local/bwameth/tests/main.nf.test.snap delete mode 100644 subworkflows/nf-core/fasta_index_bismark_bwameth/nextflow.config delete mode 100644 subworkflows/nf-core/fastq_align_dedup_bismark/nextflow.config create mode 100644 subworkflows/nf-core/fastq_align_dedup_bwameth/main.nf create mode 100644 subworkflows/nf-core/fastq_align_dedup_bwameth/meta.yml create mode 100644 subworkflows/nf-core/fastq_align_dedup_bwameth/tests/gpu.nf.test create mode 100644 subworkflows/nf-core/fastq_align_dedup_bwameth/tests/gpu.nf.test.snap create mode 100644 subworkflows/nf-core/fastq_align_dedup_bwameth/tests/main.nf.test create mode 100644 subworkflows/nf-core/fastq_align_dedup_bwameth/tests/main.nf.test.snap create mode 100644 subworkflows/nf-core/fastq_align_dedup_bwameth/tests/nextflow.config diff --git a/modules.json b/modules.json index 4b7d4145..13135a6c 100644 --- a/modules.json +++ b/modules.json @@ -43,7 +43,7 @@ "bwameth/align": { "branch": "master", "git_sha": "dfd89399e3c6e2422c4cdcf0aaa8cf2393dc51c9", - "installed_by": ["modules"] + "installed_by": ["fastq_align_dedup_bwameth", "modules"] }, "bwameth/index": { "branch": "master", @@ -68,22 +68,27 @@ "methyldackel/extract": { "branch": "master", "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1", - "installed_by": ["modules"] + "installed_by": ["fastq_align_dedup_bwameth", "modules"] }, "methyldackel/mbias": { "branch": "master", "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1", - "installed_by": ["modules"] + "installed_by": ["fastq_align_dedup_bwameth", "modules"] }, "multiqc": { "branch": "master", "git_sha": "cf17ca47590cc578dfb47db1c2a44ef86f89976d", "installed_by": ["modules"] }, + "parabricks/fq2bammeth": { + "branch": "master", + "git_sha": "3a897454edabb27d21debfec644e40f7bcfa4c43", + "installed_by": ["fastq_align_dedup_bwameth"] + }, "picard/markduplicates": { "branch": "master", "git_sha": "49f4e50534fe4b64101e62ea41d5dc43b1324358", - "installed_by": ["modules"] + "installed_by": ["fastq_align_dedup_bwameth", "modules"] }, "preseq/lcextrap": { "branch": "master", @@ -108,22 +113,22 @@ "samtools/flagstat": { "branch": "master", "git_sha": "2d20463181b1c38981a02e90d3084b5f9fa8d540", - "installed_by": ["modules"] + "installed_by": ["fastq_align_dedup_bwameth", "modules"] }, "samtools/index": { "branch": "master", "git_sha": "b13f07be4c508d6ff6312d354d09f2493243e208", - "installed_by": ["fastq_align_dedup_bismark", "modules"] + "installed_by": ["fastq_align_dedup_bismark", "fastq_align_dedup_bwameth", "modules"] }, "samtools/sort": { "branch": "master", "git_sha": "b7800db9b069ed505db3f9d91b8c72faea9be17b", - "installed_by": ["fastq_align_dedup_bismark", "modules"] + "installed_by": ["fastq_align_dedup_bismark", "fastq_align_dedup_bwameth", "modules"] }, "samtools/stats": { "branch": "master", "git_sha": "2d20463181b1c38981a02e90d3084b5f9fa8d540", - "installed_by": ["modules"] + "installed_by": ["fastq_align_dedup_bwameth", "modules"] }, "trimgalore": { "branch": "master", @@ -149,6 +154,11 @@ "git_sha": "af84433c2e619cf28e026d4140b1b8763ae3690b", "installed_by": ["subworkflows"] }, + "fastq_align_dedup_bwameth": { + "branch": "master", + "git_sha": "bd5f75ccaf2345269810e66e85de8a70e4de8764", + "installed_by": ["subworkflows"] + }, "utils_nextflow_pipeline": { "branch": "master", "git_sha": "c2b22d85f30a706a3073387f30380704fcae013b", diff --git a/modules/nf-core/parabricks/fq2bammeth/main.nf b/modules/nf-core/parabricks/fq2bammeth/main.nf new file mode 100644 index 00000000..44f26660 --- /dev/null +++ b/modules/nf-core/parabricks/fq2bammeth/main.nf @@ -0,0 +1,68 @@ +process PARABRICKS_FQ2BAMMETH { + tag "$meta.id" + label 'process_high' + label 'process_gpu' + + container "nvcr.io/nvidia/clara/clara-parabricks:4.3.2-1" + + input: + tuple val(meta), path(reads) + tuple val(meta2), path(fasta) + tuple val(meta3), path(index) + path(known_sites) + + output: + tuple val(meta), path("*.bam") , emit: bam + tuple val(meta), path("*.bai") , emit: bai + path("qc_metrics") , emit: qc_metrics, optional:true + path("*.table") , emit: bqsr_table, optional:true + path("duplicate-metrics.txt") , emit: duplicate_metrics, optional:true + path("versions.yml") , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." + } + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def in_fq_command = meta.single_end ? "--in-se-fq $reads" : "--in-fq $reads" + def known_sites_command = known_sites ? known_sites.collect{"--knownSites $it"}.join(' ') : "" + def known_sites_output = known_sites ? "--out-recal-file ${prefix}.table" : "" + def num_gpus = task.accelerator ? "--num-gpus $task.accelerator.request" : '' + """ + ln -sf \$(readlink $fasta) $index/$fasta + pbrun \\ + fq2bam_meth \\ + --ref $index/$fasta \\ + $in_fq_command \\ + --out-bam ${prefix}.bam \\ + $known_sites_command \\ + $known_sites_output \\ + $num_gpus \\ + $args + cat <<-END_VERSIONS > versions.yml + "${task.process}": + pbrun: \$(echo \$(pbrun version 2>&1) | sed 's/^Please.* //' ) + END_VERSIONS + """ + + stub: + // Exit if running this module with -profile conda / -profile mamba + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." + } + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.bam + touch ${prefix}.bam.bai + cat <<-END_VERSIONS > versions.yml + "${task.process}": + pbrun: \$(echo \$(pbrun version 2>&1) | sed 's/^Please.* //' ) + END_VERSIONS + """ +} diff --git a/modules/nf-core/parabricks/fq2bammeth/meta.yml b/modules/nf-core/parabricks/fq2bammeth/meta.yml new file mode 100644 index 00000000..4f295b88 --- /dev/null +++ b/modules/nf-core/parabricks/fq2bammeth/meta.yml @@ -0,0 +1,95 @@ +name: "parabricks_fq2bammeth" +description: VIDIA Clara Parabricks GPU-accelerated fast, accurate algorithm for mapping methylated DNA sequence reads to a reference genome, performing local alignment, and producing alignment for different parts of the query sequence +keywords: + - align + - sort + - bqsr + - duplicates + - bwameth +tools: + - "parabricks": + description: "NVIDIA Clara Parabricks GPU-accelerated genomics tools" + homepage: "https://www.nvidia.com/en-us/clara/genomics/" + documentation: "https://docs.nvidia.com/clara/parabricks/latest/documentation/tooldocs/man_fq2bam_meth.html#fq2bam-meth-reference" + licence: ["custom"] + identifier: "" + +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - reads: + type: file + description: fastq.gz files + pattern: "*.fastq.gz" + - - meta2: + type: map + description: | + Groovy Map containing fasta information + - fasta: + type: file + description: reference fasta file - must be unzipped + pattern: "*.fasta" + - - meta3: + type: map + description: | + Groovy Map containing index information + - index: + type: file + description: reference BWA index + pattern: "*.{amb,ann,bwt,pac,sa}" + - - known_sites: + type: file + description: (optional) known sites file(s) for calculating BQSR. markdups must + be true to perform BQSR. + pattern: "*.vcf.gz" +output: + - bam: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - "*.bam": + type: file + description: Sorted BAM file + pattern: "*.bam" + - bai: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - "*.bai": + type: file + description: index corresponding to sorted BAM file + pattern: "*.bai" + - qc_metrics: + - qc_metrics: + type: directory + description: (optional) optional directory of qc metrics + pattern: "qc_metrics" + - bqsr_table: + - "*.table": + type: file + description: (optional) table from base quality score recalibration calculation, + to be used with parabricks/applybqsr + pattern: "*.table" + - duplicate_metrics: + - duplicate-metrics.txt: + type: file + description: (optional) metrics calculated from marking duplcates in the bam + file + pattern: "*-duplicate-metrics.txt" + - versions: + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@sateeshperi" +maintainers: + - "@sateeshperi" + - "@gallvp" diff --git a/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test b/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test new file mode 100644 index 00000000..459eb631 --- /dev/null +++ b/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test @@ -0,0 +1,123 @@ +nextflow_process { + + name "Test Process PARABRICKS_FQ2BAMMETH" + script "../main.nf" + process "PARABRICKS_FQ2BAMMETH" + config "./nextflow.config" + + tag "bwameth/index" + tag "modules" + tag "parabricks" + tag "parabricks/fq2bammeth" + tag "modules_nfcore" + tag "gpu" + + setup { + run("BWAMETH_INDEX") { + script "../../../bwameth/index/main.nf" + process { + """ + input[0] = Channel.of([ + [ id:'test' ], // meta map + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + """ + } + } + } + + + test("SRR389222 - fastq - se") { + + when { + process { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], // meta map + [ + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = Channel.of([ + [ id:'test' ], // meta map + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = BWAMETH_INDEX.out.index + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + process.out.bai, + process.out.versions + ).match() } + ) + } + } + + test("SRR389222 - fastq - se - stub") { + options '-stub' + + when { + process { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], // meta map + [ + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = Channel.of([ + [ id:'test' ], // meta map + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = BWAMETH_INDEX.out.index + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("E. coli - fastq - pe - stub") { + options '-stub' + + when { + process { + """ + input[0] = Channel.of([ + [ id:'test', single_end:false ], // meta map + [ + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R1.fastq.gz', checkIfExists: true), + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R2.fastq.gz', checkIfExists: true), + ] + ]) + input[1] = Channel.of([ + [ id:'test' ], // meta map + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = BWAMETH_INDEX.out.index + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } +} \ No newline at end of file diff --git a/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test.snap b/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test.snap new file mode 100644 index 00000000..cf06f28c --- /dev/null +++ b/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test.snap @@ -0,0 +1,166 @@ +{ + "SRR389222 - fastq - se - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": true + }, + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + "versions.yml:md5,882d996a64ad457fcaf85e3a75514baf" + ], + "bai": [ + [ + { + "id": "test", + "single_end": true + }, + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "bam": [ + [ + { + "id": "test", + "single_end": true + }, + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "bqsr_table": [ + + ], + "duplicate_metrics": [ + + ], + "qc_metrics": [ + + ], + "versions": [ + "versions.yml:md5,882d996a64ad457fcaf85e3a75514baf" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-12T11:20:10.809580487" + }, + "E. coli - fastq - pe - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + "versions.yml:md5,882d996a64ad457fcaf85e3a75514baf" + ], + "bai": [ + [ + { + "id": "test", + "single_end": false + }, + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "bam": [ + [ + { + "id": "test", + "single_end": false + }, + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "bqsr_table": [ + + ], + "duplicate_metrics": [ + + ], + "qc_metrics": [ + + ], + "versions": [ + "versions.yml:md5,882d996a64ad457fcaf85e3a75514baf" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-12T11:20:18.927598721" + }, + "SRR389222 - fastq - se": { + "content": [ + "a7f7ca7b5eb503ab58790d64a0273ed6", + [ + [ + { + "id": "test", + "single_end": true + }, + "test.bam.bai:md5,9ec20dda17ee0441fee0598ec0e2152c" + ] + ], + [ + "versions.yml:md5,882d996a64ad457fcaf85e3a75514baf" + ] + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.04.4" + }, + "timestamp": "2024-10-26T21:29:14.709022" + } +} \ No newline at end of file diff --git a/modules/nf-core/parabricks/fq2bammeth/tests/nextflow.config b/modules/nf-core/parabricks/fq2bammeth/tests/nextflow.config new file mode 100644 index 00000000..6ad9d535 --- /dev/null +++ b/modules/nf-core/parabricks/fq2bammeth/tests/nextflow.config @@ -0,0 +1,8 @@ +process { + withName: 'PARABRICKS_FQ2BAMMETH' { + ext.args = '--low-memory' + } + // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 + // Parabricks’s fq2bam requires 24GB of memory. + // Using --low-memory for testing +} \ No newline at end of file diff --git a/nextflow.config b/nextflow.config index 13acfffb..b671d1dd 100644 --- a/nextflow.config +++ b/nextflow.config @@ -75,6 +75,7 @@ params { nomeseq = false // bwa-meth options + use_gpu = false min_depth = 0 ignore_flags = false methyl_kit = false diff --git a/subworkflows/local/bwameth/main.nf b/subworkflows/local/bwameth/main.nf deleted file mode 100644 index 04608995..00000000 --- a/subworkflows/local/bwameth/main.nf +++ /dev/null @@ -1,136 +0,0 @@ -/* - * bwameth subworkflow - */ -include { BWAMETH_ALIGN } from '../../../modules/nf-core/bwameth/align/main' -include { SAMTOOLS_SORT } from '../../../modules/nf-core/samtools/sort/main' -include { SAMTOOLS_INDEX as SAMTOOLS_INDEX_ALIGNMENTS } from '../../../modules/nf-core/samtools/index/main' -include { SAMTOOLS_FLAGSTAT } from '../../../modules/nf-core/samtools/flagstat/main' -include { SAMTOOLS_STATS } from '../../../modules/nf-core/samtools/stats/main' -include { PICARD_MARKDUPLICATES } from '../../../modules/nf-core/picard/markduplicates/main' -include { SAMTOOLS_INDEX as SAMTOOLS_INDEX_DEDUPLICATED } from '../../../modules/nf-core/samtools/index/main' -include { METHYLDACKEL_EXTRACT } from '../../../modules/nf-core/methyldackel/extract/main' -include { METHYLDACKEL_MBIAS } from '../../../modules/nf-core/methyldackel/mbias/main' - -workflow BWAMETH { - take: - reads // channel: [ val(meta), [ reads ] ] - bwameth_index // channel: /path/to/bwa_index/ - fasta // channel: /path/to/genome.fa - fasta_index // channel: /path/to/genome.fa.fai - skip_deduplication // boolean: whether to deduplicate alignments - - main: - ch_versions = Channel.empty() - - /* - * Align with bwameth - */ - BWAMETH_ALIGN ( - reads, - fasta, - bwameth_index - ) - BWAMETH_ALIGN.out.bam.dump(tag: 'BWAMETH_ALIGN: bam') - - ch_versions = ch_versions.mix(BWAMETH_ALIGN.out.versions) - - /* - * Sort raw output BAM - */ - SAMTOOLS_SORT ( - BWAMETH_ALIGN.out.bam, - [[:],[]] // Empty map and list as is optional input but required for nextflow - ) - SAMTOOLS_SORT.out.bam.dump(tag: 'BWAMETH/SAMTOOLS_SORT: bam') - - ch_versions = ch_versions.mix(SAMTOOLS_SORT.out.versions) - - /* - * Run samtools index on alignment - */ - SAMTOOLS_INDEX_ALIGNMENTS (SAMTOOLS_SORT.out.bam) - SAMTOOLS_INDEX_ALIGNMENTS.out.bai.dump(tag: 'BWAMETH/SAMTOOLS_INDEX_ALIGNMENTS: bai') - - ch_versions = ch_versions.mix(SAMTOOLS_INDEX_ALIGNMENTS.out.versions) - - /* - * Run samtools flagstat and samtools stats - */ - SAMTOOLS_FLAGSTAT(BWAMETH_ALIGN.out.bam.join(SAMTOOLS_INDEX_ALIGNMENTS.out.bai)) - SAMTOOLS_FLAGSTAT.out.flagstat.dump(tag: 'BWAMETH/SAMTOOLS_FLAGSTAT: flagstat') - - SAMTOOLS_STATS( - BWAMETH_ALIGN.out.bam.join(SAMTOOLS_INDEX_ALIGNMENTS.out.bai), - [[:],[]] - ) - SAMTOOLS_STATS.out.stats.dump(tag: 'BWAMETH/SAMTOOLS_STATS: flagstat') - - ch_versions = ch_versions.mix(SAMTOOLS_FLAGSTAT.out.versions) - ch_versions = ch_versions.mix(SAMTOOLS_STATS.out.versions) - - if (skip_deduplication) { - alignments = SAMTOOLS_SORT.out.bam - bam_index = SAMTOOLS_INDEX_ALIGNMENTS.out.bai - picard_metrics = Channel.empty() - picard_version = Channel.empty() - } else { - /* - * Run Picard MarkDuplicates - */ - PICARD_MARKDUPLICATES ( - SAMTOOLS_SORT.out.bam, - fasta, - fasta_index.map{ index -> [[:], index]}, - ) - PICARD_MARKDUPLICATES.out.bam.dump(tag: 'BWAMETH/PICARD_MARKDUPLICATES: bam') - /* - * Run samtools index on deduplicated alignment - */ - SAMTOOLS_INDEX_DEDUPLICATED (PICARD_MARKDUPLICATES.out.bam) - - alignments = PICARD_MARKDUPLICATES.out.bam - bam_index = SAMTOOLS_INDEX_DEDUPLICATED.out.bai - picard_metrics = PICARD_MARKDUPLICATES.out.metrics - picard_version = PICARD_MARKDUPLICATES.out.versions - ch_versions = ch_versions.mix(PICARD_MARKDUPLICATES.out.versions) - } - - - /* - * Extract per-base methylation and plot methylation bias - */ - - METHYLDACKEL_EXTRACT( - alignments.join(bam_index), - fasta.map{ meta, fasta_file -> [fasta_file]}, - fasta_index - ) - METHYLDACKEL_EXTRACT.out.bedgraph.dump(tag: 'BWAMETH/METHYLDACKEL_EXTRACT: bedgraph') - METHYLDACKEL_EXTRACT.out.methylkit.dump(tag: 'BWAMETH/METHYLDACKEL_EXTRACT: methylkit') - - METHYLDACKEL_MBIAS( - alignments.join(bam_index), - fasta.map{ meta, fasta_file -> [fasta_file]}, - fasta_index - ) - METHYLDACKEL_MBIAS.out.txt.dump(tag: 'BWAMETH/METHYLDACKEL_MBIAS: txt') - - ch_versions = ch_versions.mix(METHYLDACKEL_EXTRACT.out.versions) - ch_versions = ch_versions.mix(METHYLDACKEL_MBIAS.out.versions) - - /* - * Collect MultiQC inputs - */ - picard_metrics.collect{ it[1] } - .mix(SAMTOOLS_FLAGSTAT.out.flagstat.collect{ it[1] }) - .mix(SAMTOOLS_STATS.out.stats.collect{ it[1] }) - .mix(METHYLDACKEL_EXTRACT.out.bedgraph.collect{ it[1] }) - .mix(METHYLDACKEL_MBIAS.out.txt.collect{ it[1] }) - .set{ multiqc_files } - - emit: - bam = SAMTOOLS_SORT.out.bam // channel: [ val(meta), [ bam ] ] ## sorted, non-deduplicated (raw) BAM from aligner - dedup = alignments // channel: [ val(meta), [ bam ] ] ## sorted, possibly deduplicated BAM - mqc = multiqc_files // path: *{html,txt} - versions = ch_versions // path: *.version.txt -} diff --git a/subworkflows/local/bwameth/nextflow.config b/subworkflows/local/bwameth/nextflow.config deleted file mode 100644 index 19f4011e..00000000 --- a/subworkflows/local/bwameth/nextflow.config +++ /dev/null @@ -1,35 +0,0 @@ - -includeConfig "../../../conf/modules/bwameth_align.config" -includeConfig "../../../conf/modules/samtools_sort.config" -includeConfig "../../../conf/modules/samtools_flagstat.config" -includeConfig "../../../conf/modules/samtools_stats.config" -includeConfig "../../../conf/modules/picard_markduplicates.config" -includeConfig "../../../conf/modules/methyldackel_extract.config" -includeConfig "../../../conf/modules/methyldackel_mbias.config" - -process { - - withName: SAMTOOLS_INDEX_ALIGNMENTS { - ext.args = "" - publishDir = [ - [ - path: { "${params.outdir}/${params.aligner}/alignments/" }, - mode: params.publish_dir_mode, - pattern: "*.bam.bai", - enabled: params.save_align_intermeds - ] - ] - } - - withName: SAMTOOLS_INDEX_DEDUPLICATED { - ext.args = "" - publishDir = [ - [ - path: { "${params.outdir}/${params.aligner}/deduplicated/" }, - mode: params.publish_dir_mode, - pattern: "*.bam.bai" - ] - ] - } - -} diff --git a/subworkflows/local/bwameth/tests/main.nf.test b/subworkflows/local/bwameth/tests/main.nf.test deleted file mode 100644 index 21aadf8a..00000000 --- a/subworkflows/local/bwameth/tests/main.nf.test +++ /dev/null @@ -1,68 +0,0 @@ -nextflow_workflow { - - name "Test Workflow BWAMETH" - script "../main.nf" - workflow "BWAMETH" - config "../nextflow.config" - - setup { - run("UNTAR") { - script "../../../../modules/nf-core/untar/main.nf" - process { - """ - input[0] = [ - [:], - file(params.pipelines_testdata_base_path + 'reference/Bwameth_Index.tar.gz', checkIfExists: true) - ] - """ - } - } - } - - test("Bwameth | save_reference") { - - when { - params { - aligner = "bwameth" - publish_dir_mode = 'copy' - outdir = "results" - save_reference = true - save_align_intermeds = false - min_depth = 0 - comprehensive = false - ignore_flags = false - methyl_kit = false - skip_deduplication = false - } - workflow { - """ - input[0] = Channel.of([ - [ id:'test', single_end:true ], - file(params.pipelines_testdata_base_path + 'testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) - ]) - input[1] = UNTAR.out.untar - input[2] = Channel.of([ - [:], - file(params.pipelines_testdata_base_path + 'reference/genome.fa', checkIfExists: true) - ]) - input[3] = Channel.of(file(params.pipelines_testdata_base_path + 'reference/genome.fa.fai', checkIfExists: true)) - input[4] = params.skip_deduplication - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert snapshot( - workflow.out.bam.collect { bam(it[1]).getReadsMD5() }, - workflow.out.dedup.collect { bam(it[1]).getReadsMD5() }, - //workflow.out.mqc, - workflow.out.versions - ).match() } - ) - } - - } - -} diff --git a/subworkflows/local/bwameth/tests/main.nf.test.snap b/subworkflows/local/bwameth/tests/main.nf.test.snap deleted file mode 100644 index 3834e919..00000000 --- a/subworkflows/local/bwameth/tests/main.nf.test.snap +++ /dev/null @@ -1,27 +0,0 @@ -{ - "Bwameth | save_reference": { - "content": [ - [ - "37ec1c6338cc3fee7ab1cb2d48dba38" - ], - [ - "37ec1c6338cc3fee7ab1cb2d48dba38" - ], - [ - "versions.yml:md5,0906211fa5d0f6fd287c24407a882ae6", - "versions.yml:md5,204d3bc1f6fff3c1f7ea1f014f256fd7", - "versions.yml:md5,2b54b53cdedbbd547bc6bc3623dcc622", - "versions.yml:md5,38a8061118fdcb6215a31e60419df107", - "versions.yml:md5,5e8c932b160898dbcf29469f4ea6c978", - "versions.yml:md5,88592f164662fc6e5389899dbf472f53", - "versions.yml:md5,b935a799534fa7094fe81a723df04ecf", - "versions.yml:md5,c3658cdda0f57c6d7f0c57e820921eb4" - ] - ], - "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" - }, - "timestamp": "2024-10-21T18:23:18.006459111" - } -} \ No newline at end of file diff --git a/subworkflows/nf-core/fasta_index_bismark_bwameth/nextflow.config b/subworkflows/nf-core/fasta_index_bismark_bwameth/nextflow.config deleted file mode 100644 index ee09ec49..00000000 --- a/subworkflows/nf-core/fasta_index_bismark_bwameth/nextflow.config +++ /dev/null @@ -1,4 +0,0 @@ -includeConfig "../../../conf/modules/gunzip.config" -includeConfig "../../../conf/modules/bismark_genomepreparation.config" -includeConfig "../../../conf/modules/samtools_faidx.config" -includeConfig "../../../conf/modules/bwameth_index.config" diff --git a/subworkflows/nf-core/fastq_align_dedup_bismark/nextflow.config b/subworkflows/nf-core/fastq_align_dedup_bismark/nextflow.config deleted file mode 100644 index c8e3aa92..00000000 --- a/subworkflows/nf-core/fastq_align_dedup_bismark/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -includeConfig "../../../conf/modules/bismark_align.config" -includeConfig "../../../conf/modules/bismark_deduplicate.config" -includeConfig "../../../conf/modules/samtools_sort.config" -includeConfig "../../../conf/modules/samtools_index.config" -includeConfig "../../../conf/modules/bismark_methylationextractor.config" -includeConfig "../../../conf/modules/bismark_coverage2cytosine.config" -includeConfig "../../../conf/modules/bismark_report.config" -includeConfig "../../../conf/modules/bismark_summary.config" \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_align_dedup_bwameth/main.nf b/subworkflows/nf-core/fastq_align_dedup_bwameth/main.nf new file mode 100644 index 00000000..c0cc67b8 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bwameth/main.nf @@ -0,0 +1,163 @@ +include { BWAMETH_ALIGN } from '../../../modules/nf-core/bwameth/align/main' +include { PARABRICKS_FQ2BAMMETH } from '../../../modules/nf-core/parabricks/fq2bammeth/main' +include { SAMTOOLS_SORT } from '../../../modules/nf-core/samtools/sort/main' +include { SAMTOOLS_INDEX as SAMTOOLS_INDEX_ALIGNMENTS } from '../../../modules/nf-core/samtools/index/main' +include { SAMTOOLS_FLAGSTAT } from '../../../modules/nf-core/samtools/flagstat/main' +include { SAMTOOLS_STATS } from '../../../modules/nf-core/samtools/stats/main' +include { PICARD_MARKDUPLICATES } from '../../../modules/nf-core/picard/markduplicates/main' +include { SAMTOOLS_INDEX as SAMTOOLS_INDEX_DEDUPLICATED } from '../../../modules/nf-core/samtools/index/main' +include { METHYLDACKEL_EXTRACT } from '../../../modules/nf-core/methyldackel/extract/main' +include { METHYLDACKEL_MBIAS } from '../../../modules/nf-core/methyldackel/mbias/main' + +workflow FASTQ_ALIGN_DEDUP_BWAMETH { + + take: + ch_reads // channel: [ val(meta), [ reads ] ] + ch_fasta // channel: [ val(meta), [ fasta ] ] + ch_fasta_index // channel: [ val(meta), [ fasta index ] ] + ch_bwameth_index // channel: [ val(meta), [ bwameth index ] ] + skip_deduplication // boolean: whether to deduplicate alignments + + main: + + ch_alignment = Channel.empty() + ch_alignment_index = Channel.empty() + ch_samtools_flagstat = Channel.empty() + ch_samtools_stats = Channel.empty() + ch_methydackel_extract_bedgraph = Channel.empty() + ch_methydackel_extract_methylkit = Channel.empty() + ch_methydackel_mbias = Channel.empty() + ch_picard_metrics = Channel.empty() + ch_multiqc_files = Channel.empty() + ch_versions = Channel.empty() + + /* + * Align with bwameth + */ + if (params.use_gpu) { + /* + * Align with parabricks GPU enabled fq2bammeth implementation of bwameth + */ + PARABRICKS_FQ2BAMMETH ( + ch_reads, + ch_fasta, + ch_bwameth_index, + [] // known sites + ) + ch_alignment = PARABRICKS_FQ2BAMMETH.out.bam + ch_versions = ch_versions.mix(PARABRICKS_FQ2BAMMETH.out.versions) + } else { + /* + * Align with CPU version of bwameth + */ + BWAMETH_ALIGN ( + ch_reads, + ch_fasta, + ch_bwameth_index + ) + ch_alignment = BWAMETH_ALIGN.out.bam + ch_versions = BWAMETH_ALIGN.out.versions + } + + /* + * Sort raw output BAM + */ + SAMTOOLS_SORT ( + ch_alignment, + [[:],[]] // [ [meta], [fasta]] + ) + ch_alignment = SAMTOOLS_SORT.out.bam + ch_versions = ch_versions.mix(SAMTOOLS_SORT.out.versions) + + /* + * Run samtools index on alignment + */ + SAMTOOLS_INDEX_ALIGNMENTS ( + ch_alignment + ) + ch_alignment_index = SAMTOOLS_INDEX_ALIGNMENTS.out.bai + ch_versions = ch_versions.mix(SAMTOOLS_INDEX_ALIGNMENTS.out.versions) + + /* + * Run samtools flagstat + */ + SAMTOOLS_FLAGSTAT ( + ch_alignment.join(ch_alignment_index) + ) + ch_samtools_flagstat = SAMTOOLS_FLAGSTAT.out.flagstat + ch_versions = ch_versions.mix(SAMTOOLS_FLAGSTAT.out.versions) + + /* + * Run samtools stats + */ + SAMTOOLS_STATS ( + ch_alignment.join(ch_alignment_index), + [[:],[]] // [ [meta], [fasta]] + ) + ch_samtools_stats = SAMTOOLS_STATS.out.stats + ch_versions = ch_versions.mix(SAMTOOLS_STATS.out.versions) + + if (!skip_deduplication) { + /* + * Run Picard MarkDuplicates + */ + PICARD_MARKDUPLICATES ( + ch_alignment, + ch_fasta, + ch_fasta_index + ) + /* + * Run samtools index on deduplicated alignment + */ + SAMTOOLS_INDEX_DEDUPLICATED ( + PICARD_MARKDUPLICATES.out.bam + ) + ch_alignment = PICARD_MARKDUPLICATES.out.bam + ch_alignment_index = SAMTOOLS_INDEX_DEDUPLICATED.out.bai + ch_picard_metrics = PICARD_MARKDUPLICATES.out.metrics + ch_versions = ch_versions.mix(PICARD_MARKDUPLICATES.out.versions) + ch_versions = ch_versions.mix(SAMTOOLS_INDEX_DEDUPLICATED.out.versions) + } + + /* + * Extract per-base methylation and plot methylation bias + */ + + METHYLDACKEL_EXTRACT ( + ch_alignment.join(ch_alignment_index), + ch_fasta.map{ meta, fasta_file -> fasta_file }, + ch_fasta_index.map{ meta, fasta_index -> fasta_index } + ) + ch_methydackel_extract_bedgraph = METHYLDACKEL_EXTRACT.out.bedgraph + ch_methydackel_extract_methylkit = METHYLDACKEL_EXTRACT.out.methylkit + ch_versions = ch_versions.mix(METHYLDACKEL_EXTRACT.out.versions) + + METHYLDACKEL_MBIAS ( + ch_alignment.join(ch_alignment_index), + ch_fasta.map{ meta, fasta_file -> fasta_file }, + ch_fasta_index.map{ meta, fasta_index -> fasta_index } + ) + ch_methydackel_mbias = METHYLDACKEL_MBIAS.out.txt + ch_versions = ch_versions.mix(METHYLDACKEL_MBIAS.out.versions) + + /* + * Collect MultiQC inputs + */ + ch_multiqc_files = ch_picard_metrics.collect{ meta, metrics -> metrics } + .mix(ch_samtools_flagstat.collect{ meta, flagstat -> flagstat }) + .mix(ch_samtools_stats.collect{ meta, stats -> stats }) + .mix(ch_methydackel_extract_bedgraph.collect{ meta, bedgraph -> bedgraph }) + .mix(ch_methydackel_mbias.collect{ meta, txt -> txt }) + + emit: + bam = ch_alignment // channel: [ val(meta), [ bam ] ] + bai = ch_alignment_index // channel: [ val(meta), [ bai ] ] + samtools_flagstat = ch_samtools_flagstat // channel: [ val(meta), [ flagstat ] ] + samtools_stats = ch_samtools_stats // channel: [ val(meta), [ stats ] ] + methydackel_extract_bedgraph = ch_methydackel_extract_bedgraph // channel: [ val(meta), [ bedgraph ] ] + methydackel_extract_methylkit = ch_methydackel_extract_methylkit // channel: [ val(meta), [ methylkit ] ] + methydackel_mbias = ch_methydackel_mbias // channel: [ val(meta), [ mbias ] ] + picard_metrics = ch_picard_metrics // channel: [ val(meta), [ metrics ] ] + multiqc = ch_multiqc_files // channel: [ *{html,txt} ] + versions = ch_versions // channel: [ versions.yml ] +} diff --git a/subworkflows/nf-core/fastq_align_dedup_bwameth/meta.yml b/subworkflows/nf-core/fastq_align_dedup_bwameth/meta.yml new file mode 100644 index 00000000..a66ea024 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bwameth/meta.yml @@ -0,0 +1,116 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "fastq_align_dedup_bwameth" +description: Performs alignment of BS-Seq reads using bwameth or parabricks/fq2bammeth, sort and deduplicate +keywords: + - bwameth + - alignment + - 3-letter genome + - map + - methylation + - 5mC + - methylseq + - bisulphite + - bisulfite + - fastq + - bam +components: + - bwameth/align + - parabricks/fq2bammeth + - samtools/sort + - samtools/index + - samtools/flagstat + - samtools/stats + - picard/markduplicates + - samtools/index + - methyldackel/extract + - methyldackel/mbias +input: + - ch_reads: + description: | + List of input FastQ files of size 1 and 2 for single-end and paired-end data, + respectively. + Structure: [ val(meta), [ path(reads) ] ] + pattern: "*.{fastq,fastq.gz}" + - ch_fasta: + type: file + description: | + Structure: [ val(meta), path(fasta) ] + pattern: "*.{fa,fa.gz}" + - ch_fasta_index: + type: file + description: | + Structure: [ val(meta), path(fasta index) ] + - ch_bwameth_index: + description: | + Bismark genome index files + Structure: [ val(meta), path(index) ] + pattern: "BismarkIndex" + - skip_deduplication: + type: boolean + description: | + Skip deduplication of aligned reads +output: + - bam: + type: file + description: | + Channel containing BAM files + Structure: [ val(meta), path(bam) ] + pattern: "*.bam" + - bai: + type: file + description: | + Channel containing indexed BAM (BAI) files + Structure: [ val(meta), path(bai) ] + pattern: "*.bai" + - samtools_flagstat: + type: file + description: | + File containing samtools flagstat output + Structure: [ val(meta), path(flagstat) ] + pattern: "*.flagstat" + - samtools_stats: + type: file + description: | + File containing samtools stats output + Structure: [ val(meta), path(stats) ] + pattern: "*.{stats}" + - methydackel_extract_bedgraph: + type: file + description: | + bedGraph file, containing per-base methylation metrics + Structure: [ val(meta), path(bedgraph) ] + pattern: "*.bedGraph" + - methydackel_extract_methylkit: + type: file + description: | + methylKit file, containing per-base methylation metrics + Structure: [ val(meta), path(methylKit) ] + pattern: "*.methylKit" + - methydackel_mbias: + type: file + description: | + Text file containing methylation bias + Structure: [ val(meta), path(mbias) ] + pattern: "*.{txt}" + - picard_metrics: + type: file + description: | + Duplicate metrics file generated by picard + Structure: [ val(meta), path(metrics) ] + pattern: "*.{metrics.txt}" + - multiqc: + type: file + description: | + Channel containing MultiQC report aggregating results across samples. + Structure: [ val(meta), path(multiqc_report.html) ] + pattern: "*.html" + - versions: + type: file + description: | + File containing software versions + Structure: [ path(versions.yml) ] + pattern: "versions.yml" +authors: + - "@sateeshperi" +maintainers: + - "@sateeshperi" diff --git a/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/gpu.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/gpu.nf.test new file mode 100644 index 00000000..b156de6b --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/gpu.nf.test @@ -0,0 +1,132 @@ +nextflow_workflow { + + name "Test Subworkflow FASTQ_ALIGN_DEDUP_BWAMETH" + script "../main.nf" + workflow "FASTQ_ALIGN_DEDUP_BWAMETH" + config "./nextflow.config" + + tag "gpu" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/fastq_align_dedup_bwameth" + tag "bwameth/align" + tag "parabricks/fq2bammeth" + tag "samtools/sort" + tag "samtools/index" + tag "samtools/flagstat" + tag "samtools/stats" + tag "picard/markduplicates" + tag "samtools/index" + tag "methyldackel/extract" + tag "methyldackel/mbias" + tag "untar" + + setup { + run("UNTAR") { + script "../../../../modules/nf-core/untar/main.nf" + process { + """ + input[0] = [ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/Bwameth_Index.tar.gz', checkIfExists: true) + ] + """ + } + } + } + + test("Params: parabricks/fq2bammeth single-end | use_gpu") { + + when { + params { + skip_deduplication = false + use_gpu = true + } + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa.fai', checkIfExists: true) + ]) + input[3] = UNTAR.out.untar + input[4] = params.skip_deduplication + """ + } + } + + then { + assertAll( + { assert workflow.success}, + { assert snapshot( + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.samtools_flagstat, + workflow.out.samtools_stats, + workflow.out.methydackel_extract_bedgraph, + workflow.out.methydackel_extract_methylkit, + workflow.out.methydackel_mbias, + workflow.out.picard_metrics.collect { meta, metrics -> file(metrics).name }, + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } + ) + } + } + + test("Params: parabricks/fq2bammeth single-end | use_gpu | skip_deduplication") { + + when { + params { + skip_deduplication = true + use_gpu = true + } + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa.fai', checkIfExists: true) + ]) + input[3] = UNTAR.out.untar + input[4] = params.skip_deduplication + """ + } + } + + then { + assertAll( + { assert workflow.success}, + { assert snapshot( + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.samtools_flagstat, + workflow.out.samtools_stats, + workflow.out.methydackel_extract_bedgraph, + workflow.out.methydackel_extract_methylkit, + workflow.out.methydackel_mbias, + workflow.out.picard_metrics.collect { meta, metrics -> file(metrics).name }, + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } + ) + } + } + +} diff --git a/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/gpu.nf.test.snap b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/gpu.nf.test.snap new file mode 100644 index 00000000..a1602b30 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/gpu.nf.test.snap @@ -0,0 +1,149 @@ +{ + "Params: parabricks/fq2bammeth single-end | use_gpu": { + "content": [ + [ + "a7f7ca7b5eb503ab58790d64a0273ed6" + ], + [ + "test.markdup.sorted.bam.bai" + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,897d500a710a56a7098172167fa71108" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,9aac964b859fda8239aa0eae16382d56" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.markdup.sorted_CpG.bedGraph:md5,f2fe02f180456f5f4922a2a8aa559fca" + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.mbias.txt:md5,fce04d733e066d0b933cedc602e2af81" + ] + ], + [ + "test.markdup.sorted.MarkDuplicates.metrics.txt" + ], + [ + "test.flagstat", + "test.markdup.sorted.MarkDuplicates.metrics.txt", + "test.markdup.sorted_CpG.bedGraph", + "test.mbias.txt", + "test.stats" + ], + [ + "versions.yml:md5,36bd052d24ec766084f6aa2fb8a6ae4c", + "versions.yml:md5,45239309d0c40b5f0a56eba4347f09be", + "versions.yml:md5,4a6bb9a47d944ab197c823ae0ae61092", + "versions.yml:md5,8b72c7013fa6f632d28933b60ad1f2ea", + "versions.yml:md5,a80a57d29a4d72830f033bc0326b1abf", + "versions.yml:md5,b6492c12bfae23b6e279f4abfd4780e5", + "versions.yml:md5,baba90d5bd57679b913be2abd531ae15", + "versions.yml:md5,ddbe480ff81df55c6d95f911e7b6dc8a", + "versions.yml:md5,e9602257141b65a907ad9036e8a32a83" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-17T06:56:47.211389" + }, + "Params: parabricks/fq2bammeth single-end | use_gpu | skip_deduplication": { + "content": [ + [ + "a7f7ca7b5eb503ab58790d64a0273ed6" + ], + [ + "test.sorted.bam.bai" + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,897d500a710a56a7098172167fa71108" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,9aac964b859fda8239aa0eae16382d56" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.sorted_CpG.bedGraph:md5,b0cb426020f8beb45b4e8f09b9a17bfa" + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.mbias.txt:md5,fce04d733e066d0b933cedc602e2af81" + ] + ], + [ + + ], + [ + "test.flagstat", + "test.mbias.txt", + "test.sorted_CpG.bedGraph", + "test.stats" + ], + [ + "versions.yml:md5,36bd052d24ec766084f6aa2fb8a6ae4c", + "versions.yml:md5,4a6bb9a47d944ab197c823ae0ae61092", + "versions.yml:md5,8b72c7013fa6f632d28933b60ad1f2ea", + "versions.yml:md5,a80a57d29a4d72830f033bc0326b1abf", + "versions.yml:md5,b6492c12bfae23b6e279f4abfd4780e5", + "versions.yml:md5,baba90d5bd57679b913be2abd531ae15", + "versions.yml:md5,e9602257141b65a907ad9036e8a32a83" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-17T06:57:34.41912" + } +} diff --git a/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/main.nf.test b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/main.nf.test new file mode 100644 index 00000000..6b20d545 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/main.nf.test @@ -0,0 +1,179 @@ +nextflow_workflow { + + name "Test Subworkflow FASTQ_ALIGN_DEDUP_BWAMETH" + script "../main.nf" + workflow "FASTQ_ALIGN_DEDUP_BWAMETH" + config "./nextflow.config" + + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/fastq_align_dedup_bwameth" + tag "bwameth/align" + tag "parabricks/fq2bammeth" + tag "samtools/sort" + tag "samtools/index" + tag "samtools/flagstat" + tag "samtools/stats" + tag "picard/markduplicates" + tag "samtools/index" + tag "methyldackel/extract" + tag "methyldackel/mbias" + tag "untar" + + setup { + run("UNTAR") { + script "../../../../modules/nf-core/untar/main.nf" + process { + """ + input[0] = [ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/Bwameth_Index.tar.gz', checkIfExists: true) + ] + """ + } + } + } + + test("Params: bwameth single-end | default") { + + when { + params { + skip_deduplication = false + use_gpu = false + } + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa.fai', checkIfExists: true) + ]) + input[3] = UNTAR.out.untar + input[4] = params.skip_deduplication + """ + } + } + + then { + assertAll( + { assert workflow.success}, + { assert snapshot( + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.samtools_flagstat, + workflow.out.samtools_stats, + workflow.out.methydackel_extract_bedgraph, + workflow.out.methydackel_extract_methylkit, + workflow.out.methydackel_mbias, + workflow.out.picard_metrics.collect { meta, metrics -> file(metrics).name }, + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } + ) + } + } + + test("Params: bwameth paired-end | default") { + + when { + params { + skip_deduplication = false + use_gpu = false + } + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R1.fastq.gz', checkIfExists: true), + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R2.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa.fai', checkIfExists: true) + ]) + input[3] = UNTAR.out.untar + input[4] = params.skip_deduplication + """ + } + } + + then { + assertAll( + { assert workflow.success}, + { assert snapshot( + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.samtools_flagstat, + workflow.out.samtools_stats, + workflow.out.methydackel_extract_bedgraph, + workflow.out.methydackel_extract_methylkit, + workflow.out.methydackel_mbias, + workflow.out.picard_metrics.collect { meta, metrics -> file(metrics).name }, + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } + ) + } + } + + test("Params: bwameth paired-end | skip_deduplication") { + + when { + params { + skip_deduplication = true + use_gpu = false + } + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R1.fastq.gz', checkIfExists: true), + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R2.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa.fai', checkIfExists: true) + ]) + input[3] = UNTAR.out.untar + input[4] = params.skip_deduplication + """ + } + } + + then { + assertAll( + { assert workflow.success}, + { assert snapshot( + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.samtools_flagstat, + workflow.out.samtools_stats, + workflow.out.methydackel_extract_bedgraph, + workflow.out.methydackel_extract_methylkit, + workflow.out.methydackel_mbias, + workflow.out.picard_metrics.collect { meta, metrics -> file(metrics).name }, + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/main.nf.test.snap new file mode 100644 index 00000000..90c96015 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/main.nf.test.snap @@ -0,0 +1,224 @@ +{ + "Params: bwameth single-end | default": { + "content": [ + [ + "37ec1c6338cc3fee7ab1cb2d48dba38" + ], + [ + "test.markdup.sorted.bam.bai" + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,897d500a710a56a7098172167fa71108" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,9aac964b859fda8239aa0eae16382d56" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.markdup.sorted_CpG.bedGraph:md5,f2fe02f180456f5f4922a2a8aa559fca" + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.mbias.txt:md5,fce04d733e066d0b933cedc602e2af81" + ] + ], + [ + "test.markdup.sorted.MarkDuplicates.metrics.txt" + ], + [ + "test.flagstat", + "test.markdup.sorted.MarkDuplicates.metrics.txt", + "test.markdup.sorted_CpG.bedGraph", + "test.mbias.txt", + "test.stats" + ], + [ + "versions.yml:md5,36bd052d24ec766084f6aa2fb8a6ae4c", + "versions.yml:md5,45239309d0c40b5f0a56eba4347f09be", + "versions.yml:md5,4a6bb9a47d944ab197c823ae0ae61092", + "versions.yml:md5,8b72c7013fa6f632d28933b60ad1f2ea", + "versions.yml:md5,8edf3166176c863b88ba488f8b715aa3", + "versions.yml:md5,a80a57d29a4d72830f033bc0326b1abf", + "versions.yml:md5,b6492c12bfae23b6e279f4abfd4780e5", + "versions.yml:md5,baba90d5bd57679b913be2abd531ae15", + "versions.yml:md5,ddbe480ff81df55c6d95f911e7b6dc8a" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-17T05:42:12.81856451" + }, + "Params: bwameth paired-end | skip_deduplication": { + "content": [ + [ + "cf25656fffc044f2bb7d9f1b3686ecb4" + ], + [ + "test.sorted.bam.bai" + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,4ff87d121ca174953734723938c99081" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,c753c72eb4e1c32f74afb1fbd932fe1f" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.sorted_CpG.bedGraph:md5,285e492823182f5705bf0817e2d088b8" + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.mbias.txt:md5,c1fda203c1b19aca2498efe0fd4cc9e3" + ] + ], + [ + + ], + [ + "test.flagstat", + "test.mbias.txt", + "test.sorted_CpG.bedGraph", + "test.stats" + ], + [ + "versions.yml:md5,36bd052d24ec766084f6aa2fb8a6ae4c", + "versions.yml:md5,4a6bb9a47d944ab197c823ae0ae61092", + "versions.yml:md5,8b72c7013fa6f632d28933b60ad1f2ea", + "versions.yml:md5,8edf3166176c863b88ba488f8b715aa3", + "versions.yml:md5,a80a57d29a4d72830f033bc0326b1abf", + "versions.yml:md5,b6492c12bfae23b6e279f4abfd4780e5", + "versions.yml:md5,baba90d5bd57679b913be2abd531ae15" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-17T05:10:43.907134648" + }, + "Params: bwameth paired-end | default": { + "content": [ + [ + "cf25656fffc044f2bb7d9f1b3686ecb4" + ], + [ + "test.markdup.sorted.bam.bai" + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.flagstat:md5,4ff87d121ca174953734723938c99081" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.stats:md5,c753c72eb4e1c32f74afb1fbd932fe1f" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.markdup.sorted_CpG.bedGraph:md5,c6c73e5abba70ac799500f592fec5c29" + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.mbias.txt:md5,c1fda203c1b19aca2498efe0fd4cc9e3" + ] + ], + [ + "test.markdup.sorted.MarkDuplicates.metrics.txt" + ], + [ + "test.flagstat", + "test.markdup.sorted.MarkDuplicates.metrics.txt", + "test.markdup.sorted_CpG.bedGraph", + "test.mbias.txt", + "test.stats" + ], + [ + "versions.yml:md5,36bd052d24ec766084f6aa2fb8a6ae4c", + "versions.yml:md5,45239309d0c40b5f0a56eba4347f09be", + "versions.yml:md5,4a6bb9a47d944ab197c823ae0ae61092", + "versions.yml:md5,8b72c7013fa6f632d28933b60ad1f2ea", + "versions.yml:md5,8edf3166176c863b88ba488f8b715aa3", + "versions.yml:md5,a80a57d29a4d72830f033bc0326b1abf", + "versions.yml:md5,b6492c12bfae23b6e279f4abfd4780e5", + "versions.yml:md5,baba90d5bd57679b913be2abd531ae15", + "versions.yml:md5,ddbe480ff81df55c6d95f911e7b6dc8a" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-17T05:42:39.183331191" + } +} diff --git a/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/nextflow.config b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/nextflow.config new file mode 100644 index 00000000..55385ec0 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bwameth/tests/nextflow.config @@ -0,0 +1,14 @@ +process { + withName: 'PARABRICKS_FQ2BAMMETH' { + ext.args = '--low-memory' + } + + withName: 'SAMTOOLS_SORT' { + ext.prefix = { "${meta.id}.sorted" } + } + + withName: 'PICARD_MARKDUPLICATES' { + ext.args = "--ASSUME_SORTED true --REMOVE_DUPLICATES false --VALIDATION_STRINGENCY LENIENT --PROGRAM_RECORD_ID 'null' --TMP_DIR tmp" + ext.prefix = { "${meta.id}.markdup.sorted" } + } +} From cb71a6e7b2feb6a157946bdef876eda37d342c95 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Sat, 30 Nov 2024 07:37:08 +0000 Subject: [PATCH 02/89] plug into methylseq and create conf/subworkflows --- .../fasta_index_bismark_bwameth.config | 4 +++ .../fastq_align_dedup_bismark.config | 8 +++++ .../fastq_align_dedup_bwameth.config | 35 +++++++++++++++++++ workflows/methylseq/main.nf | 20 +++++------ workflows/methylseq/nextflow.config | 6 ++-- 5 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 conf/subworkflows/fasta_index_bismark_bwameth.config create mode 100644 conf/subworkflows/fastq_align_dedup_bismark.config create mode 100644 conf/subworkflows/fastq_align_dedup_bwameth.config diff --git a/conf/subworkflows/fasta_index_bismark_bwameth.config b/conf/subworkflows/fasta_index_bismark_bwameth.config new file mode 100644 index 00000000..b170b48d --- /dev/null +++ b/conf/subworkflows/fasta_index_bismark_bwameth.config @@ -0,0 +1,4 @@ +includeConfig "../modules/gunzip.config" +includeConfig "../modules/bismark_genomepreparation.config" +includeConfig "../modules/samtools_faidx.config" +includeConfig "../modules/bwameth_index.config" diff --git a/conf/subworkflows/fastq_align_dedup_bismark.config b/conf/subworkflows/fastq_align_dedup_bismark.config new file mode 100644 index 00000000..7531de5a --- /dev/null +++ b/conf/subworkflows/fastq_align_dedup_bismark.config @@ -0,0 +1,8 @@ +includeConfig "../modules/bismark_align.config" +includeConfig "../modules/bismark_deduplicate.config" +includeConfig "../modules/samtools_sort.config" +includeConfig "../modules/samtools_index.config" +includeConfig "../modules/bismark_methylationextractor.config" +includeConfig "../modules/bismark_coverage2cytosine.config" +includeConfig "../modules/bismark_report.config" +includeConfig "../modules/bismark_summary.config" diff --git a/conf/subworkflows/fastq_align_dedup_bwameth.config b/conf/subworkflows/fastq_align_dedup_bwameth.config new file mode 100644 index 00000000..c68bf509 --- /dev/null +++ b/conf/subworkflows/fastq_align_dedup_bwameth.config @@ -0,0 +1,35 @@ + +includeConfig "../modules/bwameth_align.config" +includeConfig "../modules/samtools_sort.config" +includeConfig "../modules/samtools_flagstat.config" +includeConfig "../modules/samtools_stats.config" +includeConfig "../modules/picard_markduplicates.config" +includeConfig "../modules/methyldackel_extract.config" +includeConfig "../modules/methyldackel_mbias.config" + +process { + + withName: SAMTOOLS_INDEX_ALIGNMENTS { + ext.args = "" + publishDir = [ + [ + path: { "${params.outdir}/${params.aligner}/alignments/" }, + mode: params.publish_dir_mode, + pattern: "*.bam.bai", + enabled: params.save_align_intermeds + ] + ] + } + + withName: SAMTOOLS_INDEX_DEDUPLICATED { + ext.args = "" + publishDir = [ + [ + path: { "${params.outdir}/${params.aligner}/deduplicated/" }, + mode: params.publish_dir_mode, + pattern: "*.bam.bai" + ] + ] + } + +} diff --git a/workflows/methylseq/main.nf b/workflows/methylseq/main.nf index 15d5b70b..be4575f3 100644 --- a/workflows/methylseq/main.nf +++ b/workflows/methylseq/main.nf @@ -4,20 +4,19 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ - +include { paramsSummaryMap } from 'plugin/nf-schema' include { FASTQC } from '../../modules/nf-core/fastqc/main' include { TRIMGALORE } from '../../modules/nf-core/trimgalore/main' include { QUALIMAP_BAMQC } from '../../modules/nf-core/qualimap/bamqc/main' include { PRESEQ_LCEXTRAP } from '../../modules/nf-core/preseq/lcextrap/main' include { MULTIQC } from '../../modules/nf-core/multiqc/main' include { CAT_FASTQ } from '../../modules/nf-core/cat/fastq/main' -include { paramsSummaryMap } from 'plugin/nf-schema' +include { FASTQ_ALIGN_DEDUP_BISMARK } from '../../subworkflows/nf-core/fastq_align_dedup_bismark/main' +include { FASTQ_ALIGN_DEDUP_BWAMETH } from '../../subworkflows/nf-core/fastq_align_dedup_bwameth/main' include { paramsSummaryMultiqc } from '../../subworkflows/nf-core/utils_nfcore_pipeline' include { softwareVersionsToYAML } from '../../subworkflows/nf-core/utils_nfcore_pipeline' include { methodsDescriptionText } from '../../subworkflows/local/utils_nfcore_methylseq_pipeline' include { validateInputSamplesheet } from '../../subworkflows/local/utils_nfcore_methylseq_pipeline' -include { FASTQ_ALIGN_DEDUP_BISMARK } from '../../subworkflows/nf-core/fastq_align_dedup_bismark/main' -include { BWAMETH } from '../../subworkflows/local/bwameth' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -104,17 +103,16 @@ workflow METHYLSEQ { // Aligner: bwameth else if ( params.aligner == 'bwameth' ){ - BWAMETH ( + FASTQ_ALIGN_DEDUP_BWAMETH ( reads, - ch_bwameth_index, ch_fasta, - ch_fasta_index, + ch_fasta_index.map{ index -> [ [:], index ]}, + ch_bwameth_index, params.skip_deduplication || params.rrbs, ) - ch_versions = ch_versions.mix(BWAMETH.out.versions.unique{ it.baseName }) - ch_bam = BWAMETH.out.bam - ch_dedup = BWAMETH.out.dedup - ch_aligner_mqc = BWAMETH.out.mqc + ch_versions = ch_versions.mix(FASTQ_ALIGN_DEDUP_BWAMETH.out.versions.unique{ it.baseName }) + ch_bam = FASTQ_ALIGN_DEDUP_BWAMETH.out.bam + ch_aligner_mqc = FASTQ_ALIGN_DEDUP_BWAMETH.out.multiqc } // diff --git a/workflows/methylseq/nextflow.config b/workflows/methylseq/nextflow.config index 202d7b9c..848b3085 100644 --- a/workflows/methylseq/nextflow.config +++ b/workflows/methylseq/nextflow.config @@ -6,9 +6,9 @@ includeConfig "../../conf/modules/preseq_lcextrap.config" includeConfig "../../conf/modules/qualimap_bamqc.config" // subworkflow configs -includeConfig "../../subworkflows/nf-core/fasta_index_bismark_bwameth/nextflow.config" -includeConfig "../../subworkflows/nf-core/fastq_align_dedup_bismark/nextflow.config" -includeConfig "../../subworkflows/local/bwameth/nextflow.config" +includeConfig "../../conf/subworkflows/fasta_index_bismark_bwameth.config" +includeConfig "../../conf/subworkflows/fastq_align_dedup_bismark.config" +includeConfig "../../conf/subworkflows/fastq_align_dedup_bwameth.config" process { From 3f73e334b5603103ed4d1a11918ff06f3f41731c Mon Sep 17 00:00:00 2001 From: Sateesh Date: Sat, 30 Nov 2024 07:37:30 +0000 Subject: [PATCH 03/89] add use_gpu param to schema --- nextflow_schema.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 358a9db1..d8bc78af 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -374,12 +374,18 @@ } } }, - "bwa_meth_options": { + "bwameth_options": { "title": "bwa-meth options", "type": "object", "description": "", "default": "", "properties": { + "use_gpu": { + "type": "boolean", + "description": "Run Parabricks GPU-accelerated fq2bammeth module for alignment", + "default": false, + "fa_icon": "fas fa-dot-circle" + }, "min_depth": { "type": "integer", "description": "Specify a minimum read coverage for MethylDackel to report a methylation call.", @@ -624,7 +630,7 @@ "$ref": "#/$defs/bismark_options" }, { - "$ref": "#/$defs/bwa_meth_options" + "$ref": "#/$defs/bwameth_options" }, { "$ref": "#/$defs/qualimap_options" From 787319b4d12139c256579cd29953a7c75e4c2eab Mon Sep 17 00:00:00 2001 From: Sateesh Date: Sat, 30 Nov 2024 07:43:16 +0000 Subject: [PATCH 04/89] module & subwf updates --- main.nf | 2 +- modules.json | 4 +- modules/nf-core/bismark/align/main.nf | 2 +- .../utils_nfcore_methylseq_pipeline/main.nf | 3 -- .../nf-core/utils_nfcore_pipeline/main.nf | 44 +--------------- .../tests/main.function.nf.test | 52 ------------------- .../tests/main.function.nf.test.snap | 30 ----------- 7 files changed, 5 insertions(+), 132 deletions(-) diff --git a/main.nf b/main.nf index f87f6d0e..77c8ef37 100644 --- a/main.nf +++ b/main.nf @@ -16,10 +16,10 @@ */ include { FASTA_INDEX_BISMARK_BWAMETH } from './subworkflows/nf-core/fasta_index_bismark_bwameth/main' -include { METHYLSEQ } from './workflows/methylseq/' include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_methylseq_pipeline' include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_methylseq_pipeline' include { getGenomeAttribute } from './subworkflows/local/utils_nfcore_methylseq_pipeline' +include { METHYLSEQ } from './workflows/methylseq/' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/modules.json b/modules.json index 13135a6c..57ebb780 100644 --- a/modules.json +++ b/modules.json @@ -7,7 +7,7 @@ "nf-core": { "bismark/align": { "branch": "master", - "git_sha": "f5a291e827949778a3bb976479d6298b3abf99cd", + "git_sha": "9a1e8bb6a5d205cf7807dcefca872a3314b2f3e6", "installed_by": ["fastq_align_dedup_bismark", "modules"] }, "bismark/coverage2cytosine": { @@ -166,7 +166,7 @@ }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "1b89f75f1aa2021ec3360d0deccd0f6e97240551", + "git_sha": "9a1e8bb6a5d205cf7807dcefca872a3314b2f3e6", "installed_by": ["subworkflows"] }, "utils_nfschema_plugin": { diff --git a/modules/nf-core/bismark/align/main.nf b/modules/nf-core/bismark/align/main.nf index df8b7e06..4aaa1be2 100644 --- a/modules/nf-core/bismark/align/main.nf +++ b/modules/nf-core/bismark/align/main.nf @@ -45,7 +45,7 @@ process BISMARK_ALIGN { // Check that we have enough memory try { - def tmem = (task.memory as nextflow.util.MemoryUnit).toBytes() + def tmem = (task.memory as MemoryUnit).toBytes() def mcore = (tmem / mem_per_multicore) as int ccore = Math.min(ccore, mcore) } catch (all) { diff --git a/subworkflows/local/utils_nfcore_methylseq_pipeline/main.nf b/subworkflows/local/utils_nfcore_methylseq_pipeline/main.nf index f242fc0a..c1637ba6 100644 --- a/subworkflows/local/utils_nfcore_methylseq_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_methylseq_pipeline/main.nf @@ -16,9 +16,6 @@ include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' include { imNotification } from '../../nf-core/utils_nfcore_pipeline' include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' -include { workflowCitation } from '../../nf-core/utils_nfcore_pipeline' - - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 4cd33626..228dbff8 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -56,21 +56,6 @@ def checkProfileProvided(nextflow_cli_args) { } } -// -// Citation string for pipeline -// -def workflowCitation() { - def temp_doi_ref = "" - def manifest_doi = workflow.manifest.doi.tokenize(",") - // Handling multiple DOIs - // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers - // Removing ` ` since the manifest.doi is a string and not a proper list - manifest_doi.each { doi_ref -> - temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" - } - return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + "* The pipeline\n" + temp_doi_ref + "\n" + "* The nf-core framework\n" + " https://doi.org/10.1038/s41587-020-0439-x\n\n" + "* Software dependencies\n" + " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" -} - // // Generate workflow version string // @@ -150,33 +135,6 @@ def paramsSummaryMultiqc(summary_params) { return yaml_file_text } -// -// nf-core logo -// -def nfCoreLogo(monochrome_logs=true) { - def colors = logColours(monochrome_logs) as Map - String.format( - """\n - ${dashedLine(monochrome_logs)} - ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} - ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} - ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} - ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} - ${colors.green}`._,._,\'${colors.reset} - ${colors.purple} ${workflow.manifest.name} ${getWorkflowVersion()}${colors.reset} - ${dashedLine(monochrome_logs)} - """.stripIndent() - ) -} - -// -// Return dashed line -// -def dashedLine(monochrome_logs=true) { - def colors = logColours(monochrome_logs) as Map - return "-${colors.dim}----------------------------------------------------${colors.reset}-" -} - // // ANSII colours used for terminal logging // @@ -341,7 +299,7 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi def email_html = html_template.toString() // Render the sendmail template - def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as MemoryUnit def smail_fields = [email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes()] def sf = new File("${workflow.projectDir}/assets/sendmail_template.txt") def sendmail_template = engine.createTemplate(sf).make(smail_fields) diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test index 1dc317f8..e43d208b 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test @@ -41,58 +41,6 @@ nextflow_function { } } - test("Test Function workflowCitation") { - - function "workflowCitation" - - then { - assertAll( - { assert function.success }, - { assert snapshot(function.result).match() } - ) - } - } - - test("Test Function nfCoreLogo") { - - function "nfCoreLogo" - - when { - function { - """ - input[0] = false - """ - } - } - - then { - assertAll( - { assert function.success }, - { assert snapshot(function.result).match() } - ) - } - } - - test("Test Function dashedLine") { - - function "dashedLine" - - when { - function { - """ - input[0] = false - """ - } - } - - then { - assertAll( - { assert function.success }, - { assert snapshot(function.result).match() } - ) - } - } - test("Test Function without logColours") { function "logColours" diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap index 1037232c..02c67014 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap @@ -17,26 +17,6 @@ }, "timestamp": "2024-02-28T12:02:59.729647" }, - "Test Function nfCoreLogo": { - "content": [ - "\n\n-\u001b[2m----------------------------------------------------\u001b[0m-\n \u001b[0;32m,--.\u001b[0;30m/\u001b[0;32m,-.\u001b[0m\n\u001b[0;34m ___ __ __ __ ___ \u001b[0;32m/,-._.--~'\u001b[0m\n\u001b[0;34m |\\ | |__ __ / ` / \\ |__) |__ \u001b[0;33m} {\u001b[0m\n\u001b[0;34m | \\| | \\__, \\__/ | \\ |___ \u001b[0;32m\\`-._,-`-,\u001b[0m\n \u001b[0;32m`._,._,'\u001b[0m\n\u001b[0;35m nextflow_workflow v9.9.9\u001b[0m\n-\u001b[2m----------------------------------------------------\u001b[0m-\n" - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-02-28T12:03:10.562934" - }, - "Test Function workflowCitation": { - "content": [ - "If you use nextflow_workflow for your analysis please cite:\n\n* The pipeline\n https://doi.org/10.5281/zenodo.5070524\n\n* The nf-core framework\n https://doi.org/10.1038/s41587-020-0439-x\n\n* Software dependencies\n https://github.com/nextflow_workflow/blob/master/CITATIONS.md" - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-02-28T12:03:07.019761" - }, "Test Function without logColours": { "content": [ { @@ -95,16 +75,6 @@ }, "timestamp": "2024-02-28T12:03:17.969323" }, - "Test Function dashedLine": { - "content": [ - "-\u001b[2m----------------------------------------------------\u001b[0m-" - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-02-28T12:03:14.366181" - }, "Test Function with logColours": { "content": [ { From a199bab1ea434375a3e673032c6ec954e5302cf1 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Sat, 30 Nov 2024 08:14:15 +0000 Subject: [PATCH 05/89] make bwameth cache lenient until the touch index folder issue is resolved --- conf/base.config | 1 - conf/modules/bwameth_align.config | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/base.config b/conf/base.config index 7658ae6a..e983d2ac 100644 --- a/conf/base.config +++ b/conf/base.config @@ -72,7 +72,6 @@ process { time = { 1.d * task.attempt } } withName: BWAMETH_ALIGN { - cache = 'lenient' time = { 6.d * task.attempt } } } diff --git a/conf/modules/bwameth_align.config b/conf/modules/bwameth_align.config index d660b174..420a6646 100644 --- a/conf/modules/bwameth_align.config +++ b/conf/modules/bwameth_align.config @@ -1,5 +1,6 @@ process { withName: BWAMETH_ALIGN { + cache = 'lenient' ext.args = '' publishDir = [ path: { "${params.outdir}/${params.aligner}/alignments" }, From f9079368b793a4b78bc69c074b8605c2da3e9f14 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Sat, 30 Nov 2024 08:16:26 +0000 Subject: [PATCH 06/89] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49527461..2272f49e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - 🔧 Install `fastq_align_dedup_bismark` subworkflow from nf-core/subworkflows [#453](https://github.com/nf-core/methylseq/pull/457) - 🔧 Install `fasta_index_bismark_bwameth` subworkflow from nf-core/subworkflows [#466](https://github.com/nf-core/methylseq/pull/468) -- 🔧 Install `fastq_align_dedup_bwameth` subworkflow from nf-core/subworkflows [#467](https://github.com/nf-core/methylseq/pull/) +- 🔧 Install `fastq_align_dedup_bwameth` subworkflow from nf-core/subworkflows [#467](https://github.com/nf-core/methylseq/pull/475) - 🔧 reorg individual configs to `conf/modules/` named configs [#459](https://github.com/nf-core/methylseq/pull/469) - 🔧 `run_preseq` param + skip preseq/lcextrap module by default [#458](https://github.com/nf-core/methylseq/pull/470) - 🔧 `run_qualimap` param + skip qualimap module by default [#367](https://github.com/nf-core/methylseq/pull/471) From 5bf2730a93c23c46d4572ecd51cfbd0266a0dddc Mon Sep 17 00:00:00 2001 From: Sateesh Date: Sat, 30 Nov 2024 08:36:56 +0000 Subject: [PATCH 07/89] update bwameth snapshots after install --- CHANGELOG.md | 2 +- tests/bwameth.nf.test.snap | 12 ++++++------ tests/bwameth_rrbs.nf.test.snap | 12 ++++++------ ..._save_reference_save_align_intermeds.nf.test.snap | 12 ++++++------ tests/bwameth_skip_deduplication.nf.test.snap | 12 ++++++------ tests/bwameth_skip_trimming.nf.test.snap | 12 ++++++------ 6 files changed, 31 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2272f49e..abea1580 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - 🔧 Install `fastq_align_dedup_bismark` subworkflow from nf-core/subworkflows [#453](https://github.com/nf-core/methylseq/pull/457) - 🔧 Install `fasta_index_bismark_bwameth` subworkflow from nf-core/subworkflows [#466](https://github.com/nf-core/methylseq/pull/468) -- 🔧 Install `fastq_align_dedup_bwameth` subworkflow from nf-core/subworkflows [#467](https://github.com/nf-core/methylseq/pull/475) +- 🔧 Install `fastq_align_dedup_bwameth` subworkflow from nf-core/subworkflows & introduce `use_gpu` param [#467](https://github.com/nf-core/methylseq/pull/475) - 🔧 reorg individual configs to `conf/modules/` named configs [#459](https://github.com/nf-core/methylseq/pull/469) - 🔧 `run_preseq` param + skip preseq/lcextrap module by default [#458](https://github.com/nf-core/methylseq/pull/470) - 🔧 `run_qualimap` param + skip qualimap module by default [#367](https://github.com/nf-core/methylseq/pull/471) diff --git a/tests/bwameth.nf.test.snap b/tests/bwameth.nf.test.snap index f2bbb9fb..067b917c 100644 --- a/tests/bwameth.nf.test.snap +++ b/tests/bwameth.nf.test.snap @@ -212,13 +212,13 @@ ], [ "Ecoli_10K_methylated.flagstat:md5,f520e87cc4050799a0dbed72909572ec", - "Ecoli_10K_methylated.stats:md5,2df1b94e099f2948ad990224f7cf07a4", + "Ecoli_10K_methylated.stats:md5,6067718d5b05d81bb94929cd80b8839d", "SRR389222_sub1.flagstat:md5,ad8159a7683b9c3057bf54ccb77a2ddc", - "SRR389222_sub1.stats:md5,0f660ccc767662d4bd8c5197f3d46a87", + "SRR389222_sub1.stats:md5,7df53a3d66857dd289911a13fed8497d", "SRR389222_sub2.flagstat:md5,e768b4f54014d62b096152c44f920bc1", - "SRR389222_sub2.stats:md5,9343b74f4d825bfb2a554be3d8bed432", + "SRR389222_sub2.stats:md5,b0426a921c56ed8a86ade65adf7ed6dd", "SRR389222_sub3.flagstat:md5,39516f7d2c77e6fef9543edf8cddbcd6", - "SRR389222_sub3.stats:md5,f41e135f71893e2a8b589c4a66cbfeec", + "SRR389222_sub3.stats:md5,7c6053d7f87cbaa8782fc41a53d932fe", "Ecoli_10K_methylated.markdup.sorted_CpG.bedGraph:md5,d3bce01078b6e7749f3906bf9faad5b5", "SRR389222_sub1.markdup.sorted_CpG.bedGraph:md5,0944ad0e3b05791bf6dfd06be7ca4af1", "SRR389222_sub2.markdup.sorted_CpG.bedGraph:md5,0a3c19af55d1625e91903ae25d1d2098", @@ -245,7 +245,7 @@ "multiqc_cutadapt.txt:md5,883891ac4901385102be053212068732", "multiqc_picard_dups.txt:md5,82eaab96189f22fd93db38bb516ba6a5", "multiqc_samtools_flagstat.txt:md5,a8f5b45b8b3412a9adc3dd0d35208c73", - "multiqc_samtools_stats.txt:md5,e24271a8111b06c1f545b6b1a077a4a7", + "multiqc_samtools_stats.txt:md5,bfcab6a9bd26872ecd2ec469d222c70b", "picard_deduplication.txt:md5,7dacf4d4e0d7d2d9a7205beecfa7125c", "picard_histogram.txt:md5,d41d8cd98f00b204e9800998ecf8427e", "picard_histogram_1.txt:md5,d41d8cd98f00b204e9800998ecf8427e", @@ -294,7 +294,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-11-28T10:50:52.356626917" + "timestamp": "2024-11-30T08:25:15.895637414" }, "Params: bwameth with bwameth-index": { "content": [ diff --git a/tests/bwameth_rrbs.nf.test.snap b/tests/bwameth_rrbs.nf.test.snap index 4472fb73..dba97ced 100644 --- a/tests/bwameth_rrbs.nf.test.snap +++ b/tests/bwameth_rrbs.nf.test.snap @@ -188,13 +188,13 @@ ], [ "Ecoli_10K_methylated.flagstat:md5,c6a9a8b50c5e1d239071a8b75bc63b4f", - "Ecoli_10K_methylated.stats:md5,d9690e16a7995c35de2397e9800a1fd0", + "Ecoli_10K_methylated.stats:md5,cf163b532094830a574ccb005bcd5131", "SRR389222_sub1.flagstat:md5,cccae088e2f8686b890d019b60f63065", - "SRR389222_sub1.stats:md5,138223a2070c2c68fd8867ad731b31b0", + "SRR389222_sub1.stats:md5,9ab407ef500cbf2f065892fe4ebc31bd", "SRR389222_sub2.flagstat:md5,403af1bed1e3e96f5e04d65e96728828", - "SRR389222_sub2.stats:md5,2fa170e542d799b47717984dc7c0a4a4", + "SRR389222_sub2.stats:md5,41544bc1bedf31c7e1fa97487eb9ca2d", "SRR389222_sub3.flagstat:md5,73a5cb246f6388d5d449775d6e35f5fa", - "SRR389222_sub3.stats:md5,89ccd73e9ed78f5102c5736f255e034a", + "SRR389222_sub3.stats:md5,69b8a554734f573ca931e6aa4ae8f4f3", "Ecoli_10K_methylated.deduplicated.sorted_CpG.bedGraph:md5,aa55074c144aa6eaa184480038c75932", "SRR389222_sub1.deduplicated.sorted_CpG.bedGraph:md5,9e9e6922a8d99881874d299bfd7af5de", "SRR389222_sub2.deduplicated.sorted_CpG.bedGraph:md5,d6c973d9fa7c6bc5f47141d14c18a105", @@ -220,7 +220,7 @@ "multiqc_citations.txt:md5,ad6c1e326b4276501a3b42eaad747de1", "multiqc_cutadapt.txt:md5,8752cce2e7320f49516e897c60487831", "multiqc_samtools_flagstat.txt:md5,4389e377eb1db8dde26b0e8db6589c2d", - "multiqc_samtools_stats.txt:md5,3843eed6fb6ec8d27729a8b901ada0cf", + "multiqc_samtools_stats.txt:md5,d55db7d43ccd18c327449e853455bcab", "samtools-flagstat-dp_Percentage_of_total.txt:md5,6b61188e70378467e23174bd12b1aaaa", "samtools-flagstat-dp_Read_counts.txt:md5,1491440de94dc58b28ca8f61d893a710", "samtools-stats-dp.txt:md5,deab74e7245766aab6b2284492959fed", @@ -249,6 +249,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-11-28T11:03:39.089301861" + "timestamp": "2024-11-30T08:28:38.388886798" } } \ No newline at end of file diff --git a/tests/bwameth_save_reference_save_align_intermeds.nf.test.snap b/tests/bwameth_save_reference_save_align_intermeds.nf.test.snap index 2de87ce1..e1a70aca 100644 --- a/tests/bwameth_save_reference_save_align_intermeds.nf.test.snap +++ b/tests/bwameth_save_reference_save_align_intermeds.nf.test.snap @@ -232,13 +232,13 @@ ], [ "Ecoli_10K_methylated.flagstat:md5,f520e87cc4050799a0dbed72909572ec", - "Ecoli_10K_methylated.stats:md5,2df1b94e099f2948ad990224f7cf07a4", + "Ecoli_10K_methylated.stats:md5,6067718d5b05d81bb94929cd80b8839d", "SRR389222_sub1.flagstat:md5,ad8159a7683b9c3057bf54ccb77a2ddc", - "SRR389222_sub1.stats:md5,0f660ccc767662d4bd8c5197f3d46a87", + "SRR389222_sub1.stats:md5,7df53a3d66857dd289911a13fed8497d", "SRR389222_sub2.flagstat:md5,e768b4f54014d62b096152c44f920bc1", - "SRR389222_sub2.stats:md5,9343b74f4d825bfb2a554be3d8bed432", + "SRR389222_sub2.stats:md5,b0426a921c56ed8a86ade65adf7ed6dd", "SRR389222_sub3.flagstat:md5,39516f7d2c77e6fef9543edf8cddbcd6", - "SRR389222_sub3.stats:md5,f41e135f71893e2a8b589c4a66cbfeec", + "SRR389222_sub3.stats:md5,7c6053d7f87cbaa8782fc41a53d932fe", "genome.fa.bwameth.c2t:md5,e51d48ed28fa0c26e2f9c9f13d09403b", "genome.fa.bwameth.c2t.amb:md5,010a242c6764efb30141868a45d698b3", "genome.fa.bwameth.c2t.ann:md5,09b4db3d87a2d4dac9e10e807f377110", @@ -271,7 +271,7 @@ "multiqc_cutadapt.txt:md5,883891ac4901385102be053212068732", "multiqc_picard_dups.txt:md5,82eaab96189f22fd93db38bb516ba6a5", "multiqc_samtools_flagstat.txt:md5,a8f5b45b8b3412a9adc3dd0d35208c73", - "multiqc_samtools_stats.txt:md5,e24271a8111b06c1f545b6b1a077a4a7", + "multiqc_samtools_stats.txt:md5,bfcab6a9bd26872ecd2ec469d222c70b", "picard_deduplication.txt:md5,7dacf4d4e0d7d2d9a7205beecfa7125c", "picard_histogram.txt:md5,d41d8cd98f00b204e9800998ecf8427e", "picard_histogram_1.txt:md5,d41d8cd98f00b204e9800998ecf8427e", @@ -352,6 +352,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-11-28T10:54:55.04039479" + "timestamp": "2024-11-30T08:30:20.392223072" } } \ No newline at end of file diff --git a/tests/bwameth_skip_deduplication.nf.test.snap b/tests/bwameth_skip_deduplication.nf.test.snap index 397d9be4..a2af2143 100644 --- a/tests/bwameth_skip_deduplication.nf.test.snap +++ b/tests/bwameth_skip_deduplication.nf.test.snap @@ -188,13 +188,13 @@ ], [ "Ecoli_10K_methylated.flagstat:md5,f520e87cc4050799a0dbed72909572ec", - "Ecoli_10K_methylated.stats:md5,2df1b94e099f2948ad990224f7cf07a4", + "Ecoli_10K_methylated.stats:md5,bed52531fe0a73a798d381dccbe2a248", "SRR389222_sub1.flagstat:md5,ad8159a7683b9c3057bf54ccb77a2ddc", - "SRR389222_sub1.stats:md5,0f660ccc767662d4bd8c5197f3d46a87", + "SRR389222_sub1.stats:md5,7e2b586e29251d4c53442882598b7299", "SRR389222_sub2.flagstat:md5,e768b4f54014d62b096152c44f920bc1", - "SRR389222_sub2.stats:md5,9343b74f4d825bfb2a554be3d8bed432", + "SRR389222_sub2.stats:md5,d49b348483aba5d7f80215458f8b830c", "SRR389222_sub3.flagstat:md5,39516f7d2c77e6fef9543edf8cddbcd6", - "SRR389222_sub3.stats:md5,f41e135f71893e2a8b589c4a66cbfeec", + "SRR389222_sub3.stats:md5,db87ee5d3eb2d84b846929edb51fbaba", "Ecoli_10K_methylated.sorted_CpG.bedGraph:md5,aa88f19284e63097e9abccac4d7c6669", "SRR389222_sub1.sorted_CpG.bedGraph:md5,4475a9462ce86a019015b69b82a9d5a2", "SRR389222_sub2.sorted_CpG.bedGraph:md5,18b8845d9d985d3c203ca6c9c40aab1b", @@ -220,7 +220,7 @@ "multiqc_citations.txt:md5,ad6c1e326b4276501a3b42eaad747de1", "multiqc_cutadapt.txt:md5,883891ac4901385102be053212068732", "multiqc_samtools_flagstat.txt:md5,a8f5b45b8b3412a9adc3dd0d35208c73", - "multiqc_samtools_stats.txt:md5,e24271a8111b06c1f545b6b1a077a4a7", + "multiqc_samtools_stats.txt:md5,bfcab6a9bd26872ecd2ec469d222c70b", "samtools-flagstat-dp_Percentage_of_total.txt:md5,ff5d65a480449ba98587c43f3a2bd33e", "samtools-flagstat-dp_Read_counts.txt:md5,6a5332e2ad59f7da00b5a9a1a25fd2f8", "samtools-stats-dp.txt:md5,52021b9a9455b39b39dd357bc2e8d346", @@ -249,6 +249,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-11-28T10:56:17.383748035" + "timestamp": "2024-11-30T08:31:33.308611147" } } \ No newline at end of file diff --git a/tests/bwameth_skip_trimming.nf.test.snap b/tests/bwameth_skip_trimming.nf.test.snap index e7fb27c8..d6a20a17 100644 --- a/tests/bwameth_skip_trimming.nf.test.snap +++ b/tests/bwameth_skip_trimming.nf.test.snap @@ -173,13 +173,13 @@ ], [ "Ecoli_10K_methylated.flagstat:md5,7c3bae0f9c700dde88785a7e94c6ae2c", - "Ecoli_10K_methylated.stats:md5,1eb750b964d4da8e9bb23ea28514873b", + "Ecoli_10K_methylated.stats:md5,aab233c65fcd8348af6f1a48e82e517f", "SRR389222_sub1.flagstat:md5,897d500a710a56a7098172167fa71108", - "SRR389222_sub1.stats:md5,6625430e604b48c41c3868e684083065", + "SRR389222_sub1.stats:md5,4ee1cbf4f178495ab51519b19555ba9d", "SRR389222_sub2.flagstat:md5,37d33d68b31d4550620dc17582a3a750", - "SRR389222_sub2.stats:md5,ce9216e71c0eb24371fb388429c59894", + "SRR389222_sub2.stats:md5,7d28cf8939637b827190b0507ff97ead", "SRR389222_sub3.flagstat:md5,e3e925a7b7356e93d14f5e9e102c55b7", - "SRR389222_sub3.stats:md5,4f5336d853dfada24d9c34a754a7f0a8", + "SRR389222_sub3.stats:md5,08059932de70d28e8f889729dd741cc1", "Ecoli_10K_methylated.markdup.sorted_CpG.bedGraph:md5,b43c4c8ff346ae865b2b5db996650942", "SRR389222_sub1.markdup.sorted_CpG.bedGraph:md5,0944ad0e3b05791bf6dfd06be7ca4af1", "SRR389222_sub2.markdup.sorted_CpG.bedGraph:md5,0a3c19af55d1625e91903ae25d1d2098", @@ -202,7 +202,7 @@ "multiqc_citations.txt:md5,ee1baba4adea57b43a9ae59ce7e1dd7f", "multiqc_picard_dups.txt:md5,2dd92451825bab5f1d74bd96933756e6", "multiqc_samtools_flagstat.txt:md5,707a91291107beffe14f9cc2d540f4a0", - "multiqc_samtools_stats.txt:md5,76db021090766f669ea1276dab863cfe", + "multiqc_samtools_stats.txt:md5,e96769e0e74bb5de5b00fef59756e02f", "picard_deduplication.txt:md5,b86a20261610bdf7a2e63ff43e232c75", "picard_histogram.txt:md5,d41d8cd98f00b204e9800998ecf8427e", "picard_histogram_1.txt:md5,d41d8cd98f00b204e9800998ecf8427e", @@ -251,6 +251,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-11-28T11:05:15.608887033" + "timestamp": "2024-11-30T08:32:50.058435618" } } \ No newline at end of file From 596a697d6ff2b3fcb22cc9accc926a37e983d1c9 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 05:58:07 +0000 Subject: [PATCH 08/89] CI: add gpu tag test job --- .github/workflows/ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cca6e659..bef73947 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,17 @@ jobs: --profile "+${{ matrix.profile }}" \ --filter ${{ matrix.filter }} + - name: Run GPU Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) + if: matrix.profile == 'docker' || matrix.profile == 'singularity' + run: | + nf-test test \ + --ci \ + --shard ${{ matrix.shard }}/${{ strategy.job-total }} \ + --changed-since HEAD^ \ + --profile "+${{ matrix.profile }}" \ + --filter ${{ matrix.filter }} \ + --tag "gpu" + - name: Output log on failure if: failure() run: | From e03286f95f8decb9e7ff3233b426426fbe8672fd Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 06:45:11 +0000 Subject: [PATCH 09/89] CI: add gpu tag test job --- .github/workflows/ci.yml | 129 ++++++++++++++++++++++++++++++++++----- 1 file changed, 114 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bef73947..9c79e341 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,8 +23,119 @@ concurrency: cancel-in-progress: true jobs: - test: - name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }})" + nf-test-changes: + name: nf-test-changes + runs-on: ubuntu-latest + outputs: + # Expose detected tags as 'modules' and 'workflows' output variables + paths: ${{ steps.list.outputs.components }} + modules: ${{ steps.outputs.outputs.modules }} + subworkflows: ${{ steps.outputs.outputs.subworkflows}} + # Prod for version bumping + steps: + - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner + run: | + ls -la ./ + rm -rf ./* || true + rm -rf ./.??* || true + ls -la ./ + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 + + - name: List nf-test files + id: list + run: | + COMPONENTS=$(nf-test list --tags | grep gpu) + echo "components=$COMPONENTS" >> $GITHUB_ENV + env: + components: ${{ steps.list.outputs.components }} + + nf-test-gpu: + runs-on: "gpu" + name: "GPU | ${{ matrix.profile }} | ${{ matrix.shard }}" + needs: nf-test-changes + if: ${{ needs.nf-test-changes.outputs.components != '[]' }} + strategy: + fail-fast: false + matrix: + NXF_VER: + - "24.04.2" + - "latest-everything" + shard: [1, 2] + filter: [pipeline, workflow] + profile: + - "docker" + - "singularity" + isMaster: + - ${{ github.base_ref == 'master' }} + # Exclude conda and singularity on dev + exclude: + - isMaster: false + profile: "conda" + - isMaster: false + profile: "singularity" + env: + NXF_ANSI_LOG: false + TOTAL_SHARDS: 2 + + steps: + - name: Check out pipeline code + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 + with: + fetch-depth: 0 + + - name: Set up Nextflow + uses: nf-core/setup-nextflow@v2 + with: + version: "${{ matrix.NXF_VER }}" + + - name: Install pdiff to see diff between nf-test snapshots + run: | + python -m pip install --upgrade pip + pip install pdiff + + - uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFT_VER }} + + - name: Set up Apptainer + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: matrix.profile == 'singularity' + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Clean up Disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + + - name: Run GPU Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) + run: | + nf-test test \ + --ci \ + --shard ${{ matrix.shard }}/${{ strategy.job-total }} \ + --changed-since HEAD^ \ + --profile "+${{ matrix.profile }}" \ + --filter ${{ matrix.filter }} \ + --tag "gpu" + + - name: Output log on failure + if: failure() + run: | + sudo apt install bat > /dev/null + batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log + + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + if: always() # always run even if the previous step fails + with: + report_paths: test.xml + + nf-test: + name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.profile }})" # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" runs-on: ubuntu-latest @@ -37,10 +148,9 @@ jobs: shard: [1, 2, 3, 4, 5] filter: [pipeline, workflow] profile: + - "conda" - "docker" - "singularity" - test_name: - - "test" isMaster: - ${{ github.base_ref == 'master' }} # Exclude conda and singularity on dev @@ -106,17 +216,6 @@ jobs: --profile "+${{ matrix.profile }}" \ --filter ${{ matrix.filter }} - - name: Run GPU Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) - if: matrix.profile == 'docker' || matrix.profile == 'singularity' - run: | - nf-test test \ - --ci \ - --shard ${{ matrix.shard }}/${{ strategy.job-total }} \ - --changed-since HEAD^ \ - --profile "+${{ matrix.profile }}" \ - --filter ${{ matrix.filter }} \ - --tag "gpu" - - name: Output log on failure if: failure() run: | From 31af1890255e260d9e78ac45ce15eb20107857a8 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 06:58:51 +0000 Subject: [PATCH 10/89] CI: add gpu tag test job --- .github/workflows/ci.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c79e341..906e81a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,11 +27,7 @@ jobs: name: nf-test-changes runs-on: ubuntu-latest outputs: - # Expose detected tags as 'modules' and 'workflows' output variables - paths: ${{ steps.list.outputs.components }} - modules: ${{ steps.outputs.outputs.modules }} - subworkflows: ${{ steps.outputs.outputs.subworkflows}} - # Prod for version bumping + paths: ${{ steps.list.outputs.paths }} steps: - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner run: | @@ -39,17 +35,27 @@ jobs: rm -rf ./* || true rm -rf ./.??* || true ls -la ./ - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Install pdiff to see diff between nf-test snapshots + run: | + python -m pip install --upgrade pip + pip install pdiff + + - uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFT_VER }} + - name: List nf-test files id: list run: | COMPONENTS=$(nf-test list --tags | grep gpu) - echo "components=$COMPONENTS" >> $GITHUB_ENV + echo "paths=$paths" >> $GITHUB_ENV env: - components: ${{ steps.list.outputs.components }} + paths: ${{ steps.list.outputs.paths }} nf-test-gpu: runs-on: "gpu" From af77e4a167b730097e8c7f9a60bbacf4bd3f3c34 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 07:01:32 +0000 Subject: [PATCH 11/89] CI: add gpu tag test job --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 906e81a3..56e84813 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,10 +53,14 @@ jobs: id: list run: | COMPONENTS=$(nf-test list --tags | grep gpu) - echo "paths=$paths" >> $GITHUB_ENV + echo "paths=$COMPONENTS" >> $GITHUB_ENV env: paths: ${{ steps.list.outputs.paths }} + - name: debug + run: | + echo ${{ steps.list.outputs.paths }} + nf-test-gpu: runs-on: "gpu" name: "GPU | ${{ matrix.profile }} | ${{ matrix.shard }}" From 0712cf0fb7f816c93d0dc57e89d7537281376301 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 07:05:11 +0000 Subject: [PATCH 12/89] CI: add gpu tag test job --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56e84813..892c5b7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,9 +53,7 @@ jobs: id: list run: | COMPONENTS=$(nf-test list --tags | grep gpu) - echo "paths=$COMPONENTS" >> $GITHUB_ENV - env: - paths: ${{ steps.list.outputs.paths }} + echo "paths=$COMPONENTS" >> $GITHUB_OUTPUT - name: debug run: | From 4990bf4722e6cdd07a5b3124e5ab8f8925c2e0c0 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 07:09:08 +0000 Subject: [PATCH 13/89] CI: add gpu tag test job --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 892c5b7b..4b63761b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: runs-on: "gpu" name: "GPU | ${{ matrix.profile }} | ${{ matrix.shard }}" needs: nf-test-changes - if: ${{ needs.nf-test-changes.outputs.components != '[]' }} + if: ${{ needs.nf-test-changes.outputs.paths != '[]' }} strategy: fail-fast: false matrix: From 9e6b1a979a436d24905cefa899540ab7acd74064 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 07:39:33 +0000 Subject: [PATCH 14/89] CI: add gpu tag test job --- .github/workflows/ci.yml | 68 +++++++++++++--------------------------- 1 file changed, 22 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b63761b..19985f14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,54 +23,18 @@ concurrency: cancel-in-progress: true jobs: - nf-test-changes: - name: nf-test-changes - runs-on: ubuntu-latest - outputs: - paths: ${{ steps.list.outputs.paths }} - steps: - - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner - run: | - ls -la ./ - rm -rf ./* || true - rm -rf ./.??* || true - ls -la ./ - - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install pdiff to see diff between nf-test snapshots - run: | - python -m pip install --upgrade pip - pip install pdiff - - - uses: nf-core/setup-nf-test@v1 - with: - version: ${{ env.NFT_VER }} - - - name: List nf-test files - id: list - run: | - COMPONENTS=$(nf-test list --tags | grep gpu) - echo "paths=$COMPONENTS" >> $GITHUB_OUTPUT - - - name: debug - run: | - echo ${{ steps.list.outputs.paths }} - nf-test-gpu: - runs-on: "gpu" - name: "GPU | ${{ matrix.profile }} | ${{ matrix.shard }}" - needs: nf-test-changes - if: ${{ needs.nf-test-changes.outputs.paths != '[]' }} + name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.profile }})" + # Only run on push if this is the nf-core dev branch (merged PRs) + if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" + runs-on: ubuntu-latest strategy: fail-fast: false matrix: NXF_VER: - "24.04.2" - "latest-everything" - shard: [1, 2] + shard: [1, 2, 3, 4, 5] filter: [pipeline, workflow] profile: - "docker" @@ -117,14 +81,23 @@ jobs: mkdir -p $NXF_SINGULARITY_CACHEDIR mkdir -p $NXF_SINGULARITY_LIBRARYDIR + - name: Set up Miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 + with: + miniconda-version: "latest" + auto-update-conda: true + conda-solver: libmamba + channels: conda-forge,bioconda + - name: Clean up Disk space uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - name: Run GPU Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) + - name: Run GPU Tests (Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }}) run: | nf-test test \ --ci \ - --shard ${{ matrix.shard }}/${{ strategy.job-total }} \ + --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ --changed-since HEAD^ \ --profile "+${{ matrix.profile }}" \ --filter ${{ matrix.filter }} \ @@ -156,7 +129,6 @@ jobs: shard: [1, 2, 3, 4, 5] filter: [pipeline, workflow] profile: - - "conda" - "docker" - "singularity" isMaster: @@ -167,6 +139,10 @@ jobs: profile: "conda" - isMaster: false profile: "singularity" + env: + NXF_ANSI_LOG: false + TOTAL_SHARDS: 5 + steps: - name: Check out pipeline code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 @@ -215,11 +191,11 @@ jobs: - name: Clean up Disk space uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) + - name: Run Tests (Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }}) run: | nf-test test \ --ci \ - --shard ${{ matrix.shard }}/${{ strategy.job-total }} \ + --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ --changed-since HEAD^ \ --profile "+${{ matrix.profile }}" \ --filter ${{ matrix.filter }} From 63c856f3835d64eaf1db88b598ea6b5c48400dd2 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 07:47:37 +0000 Subject: [PATCH 15/89] CI: add gpu tag test job --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19985f14..2d2322b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.profile }})" # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" - runs-on: ubuntu-latest + runs-on: "gpu" strategy: fail-fast: false matrix: From 10521aa92aa07a9352c165e121bf9a10e261b9a7 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 07:50:32 +0000 Subject: [PATCH 16/89] CI: add gpu tag test job --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d2322b9..aebeea6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ concurrency: jobs: nf-test-gpu: - name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.profile }})" + name: "Run GPU Tests (${{ matrix.NXF_VER }} | ${{ matrix.profile }})" # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" runs-on: "gpu" @@ -116,7 +116,7 @@ jobs: report_paths: test.xml nf-test: - name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.profile }})" + name: "Run CPU Tests (${{ matrix.NXF_VER }} | ${{ matrix.profile }})" # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" runs-on: ubuntu-latest @@ -125,7 +125,6 @@ jobs: matrix: NXF_VER: - "24.04.2" - - "latest-everything" shard: [1, 2, 3, 4, 5] filter: [pipeline, workflow] profile: From 4c0008042aabffc318f6767ee6ada456454fd970 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 07:52:17 +0000 Subject: [PATCH 17/89] CI: add gpu tag test job --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aebeea6b..e7be25c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,8 +33,7 @@ jobs: matrix: NXF_VER: - "24.04.2" - - "latest-everything" - shard: [1, 2, 3, 4, 5] + shard: [1, 2] filter: [pipeline, workflow] profile: - "docker" From 2d482c72837bd97baf9fe29dfc0e6abb20c0515e Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 08:03:50 +0000 Subject: [PATCH 18/89] CI: add gpu tag test job --- .github/workflows/ci.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7be25c7..eca7f5a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,8 +42,6 @@ jobs: - ${{ github.base_ref == 'master' }} # Exclude conda and singularity on dev exclude: - - isMaster: false - profile: "conda" - isMaster: false profile: "singularity" env: @@ -80,15 +78,6 @@ jobs: mkdir -p $NXF_SINGULARITY_CACHEDIR mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - name: Set up Miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 - with: - miniconda-version: "latest" - auto-update-conda: true - conda-solver: libmamba - channels: conda-forge,bioconda - - name: Clean up Disk space uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 From d571e3cf43efe092d7fa52c54a7010ef44c5e1f0 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 08:20:56 +0000 Subject: [PATCH 19/89] CI: add gpu tag test job --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eca7f5a5..40629809 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,6 +113,7 @@ jobs: matrix: NXF_VER: - "24.04.2" + - "latest-everything" shard: [1, 2, 3, 4, 5] filter: [pipeline, workflow] profile: From 231cbc319e9d7a4ed62ac1b140438c7f8b4981c2 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 08:30:14 +0000 Subject: [PATCH 20/89] add bwameth_gpu test --- tests/bwameth_gpu.nf.test | 63 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/bwameth_gpu.nf.test diff --git a/tests/bwameth_gpu.nf.test b/tests/bwameth_gpu.nf.test new file mode 100644 index 00000000..b6d2c183 --- /dev/null +++ b/tests/bwameth_gpu.nf.test @@ -0,0 +1,63 @@ +nextflow_pipeline { + + name "Test Workflow main.nf" + script "../main.nf" + config "./nextflow.config" + + tag "gpu" + + test("Params: bwameth") { + when { + params { + aligner = "bwameth" + use_gpu = true + outdir = "$outputDir" + } + } + + then { + // stable_name: All files + folders in ${params.outdir}/ with a stable name + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_path: All files in ${params.outdir}/ with stable content + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + // bam_files: All bam files + def bam_files = getAllFilesFromDir(params.outdir, include: ['**/*.bam']) + assertAll( + { assert workflow.success}, + { assert snapshot( + // Number of tasks + workflow.trace.succeeded().size(), + // pipeline versions.yml file for multiqc from which Nextflow version is removed because we tests pipelines on multiple Nextflow versions + removeNextflowVersion("$outputDir/pipeline_info/nf_core_pipeline_software_mqc_versions.yml"), + // All stable path name + stable_name, + // All files with stable contents + stable_path, + // All bam files + bam_files.collect{ file -> [ file.getName(), bam(file.toString()).getReadsMD5() ] } + ).match() } + ) + } + } + + test("Params: bwameth with bwameth-index") { + when { + params { + aligner = "bwameth" + bwameth_index = "https://github.com/nf-core/test-datasets/raw/methylseq/reference/Bwameth_Index.tar.gz" + outdir = "$outputDir" + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + // Number of tasks + workflow.trace.succeeded().size() + ).match() } + ) + } + } + +} From 0b2f0698bf290ede28b3e91bdddec3f9be0b31ea Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 08:36:08 +0000 Subject: [PATCH 21/89] add gpu profile to config --- nextflow.config | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nextflow.config b/nextflow.config index b671d1dd..2e2ce746 100644 --- a/nextflow.config +++ b/nextflow.config @@ -221,6 +221,17 @@ profiles { executor.cpus = 4 executor.memory = 8.GB } + docker_self_hosted { + docker.enabled = true + docker.fixOwnership = true + docker.runOptions = '--platform=linux/amd64' + } + gpu { + docker.runOptions = '-u $(id -u):$(id -g) --gpus all' + apptainer.runOptions = '--nv' + singularity.runOptions = '--nv' + use_gpu = true + } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } test_ci { params.pipelines_testdata_base_path = './test-datasets/' } From b26a4aa7fd97e748740e20daf82bd1eac7ae5e71 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 09:08:46 +0000 Subject: [PATCH 22/89] add bwameth gpu test + snap --- assets/samplesheet_gpu.csv | 4 + ...th_gpu.nf.test => bwameth_use_gpu.nf.test} | 9 +- tests/bwameth_use_gpu.nf.test.snap | 289 ++++++++++++++++++ tests/nextflow.config | 4 + 4 files changed, 302 insertions(+), 4 deletions(-) create mode 100644 assets/samplesheet_gpu.csv rename tests/{bwameth_gpu.nf.test => bwameth_use_gpu.nf.test} (90%) create mode 100644 tests/bwameth_use_gpu.nf.test.snap diff --git a/assets/samplesheet_gpu.csv b/assets/samplesheet_gpu.csv new file mode 100644 index 00000000..fb599e76 --- /dev/null +++ b/assets/samplesheet_gpu.csv @@ -0,0 +1,4 @@ +sample,fastq_1,fastq_2,genome +SRR389222_sub1,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz,, +SRR389222_sub2,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub2.fastq.gz,, +SRR389222_sub3,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub3.fastq.gz,, diff --git a/tests/bwameth_gpu.nf.test b/tests/bwameth_use_gpu.nf.test similarity index 90% rename from tests/bwameth_gpu.nf.test rename to tests/bwameth_use_gpu.nf.test index b6d2c183..2cdf7cd0 100644 --- a/tests/bwameth_gpu.nf.test +++ b/tests/bwameth_use_gpu.nf.test @@ -6,12 +6,13 @@ nextflow_pipeline { tag "gpu" - test("Params: bwameth") { + test("Params: bwameth | use_gpu") { when { params { - aligner = "bwameth" - use_gpu = true - outdir = "$outputDir" + input = "${projectDir}/assets/samplesheet_gpu.csv" + aligner = "bwameth" + use_gpu = true + outdir = "$outputDir" } } diff --git a/tests/bwameth_use_gpu.nf.test.snap b/tests/bwameth_use_gpu.nf.test.snap new file mode 100644 index 00000000..80416005 --- /dev/null +++ b/tests/bwameth_use_gpu.nf.test.snap @@ -0,0 +1,289 @@ +{ + "Params: bwameth | use_gpu": { + "content": [ + 36, + { + "BWAMETH_INDEX": { + "bwameth": "0.2.7" + }, + "FASTQC": { + "fastqc": "0.12.1" + }, + "GUNZIP": { + "gunzip": 1.1 + }, + "PARABRICKS_FQ2BAMMETH": { + "pbrun": "4.3.2-1" + }, + "TRIMGALORE": { + "trimgalore": "0.6.10", + "cutadapt": 4.9 + }, + "Workflow": { + "nf-core/methylseq": "v2.8.0dev" + } + }, + [ + "bwameth", + "bwameth/alignments", + "bwameth/alignments/samtools_stats", + "bwameth/alignments/samtools_stats/SRR389222_sub1.flagstat", + "bwameth/alignments/samtools_stats/SRR389222_sub1.stats", + "bwameth/alignments/samtools_stats/SRR389222_sub2.flagstat", + "bwameth/alignments/samtools_stats/SRR389222_sub2.stats", + "bwameth/alignments/samtools_stats/SRR389222_sub3.flagstat", + "bwameth/alignments/samtools_stats/SRR389222_sub3.stats", + "bwameth/deduplicated", + "bwameth/deduplicated/SRR389222_sub1.deduplicated.sorted.bam", + "bwameth/deduplicated/SRR389222_sub1.markdup.sorted.bam", + "bwameth/deduplicated/SRR389222_sub1.markdup.sorted.bam.bai", + "bwameth/deduplicated/SRR389222_sub2.deduplicated.sorted.bam", + "bwameth/deduplicated/SRR389222_sub2.markdup.sorted.bam", + "bwameth/deduplicated/SRR389222_sub2.markdup.sorted.bam.bai", + "bwameth/deduplicated/SRR389222_sub3.deduplicated.sorted.bam", + "bwameth/deduplicated/SRR389222_sub3.markdup.sorted.bam", + "bwameth/deduplicated/SRR389222_sub3.markdup.sorted.bam.bai", + "bwameth/deduplicated/picard_metrics", + "bwameth/deduplicated/picard_metrics/SRR389222_sub1.markdup.sorted.MarkDuplicates.metrics.txt", + "bwameth/deduplicated/picard_metrics/SRR389222_sub2.markdup.sorted.MarkDuplicates.metrics.txt", + "bwameth/deduplicated/picard_metrics/SRR389222_sub3.markdup.sorted.MarkDuplicates.metrics.txt", + "fastqc", + "fastqc/SRR389222_sub1_fastqc.html", + "fastqc/SRR389222_sub2_fastqc.html", + "fastqc/SRR389222_sub3_fastqc.html", + "fastqc/zips", + "fastqc/zips/SRR389222_sub1_fastqc.zip", + "fastqc/zips/SRR389222_sub2_fastqc.zip", + "fastqc/zips/SRR389222_sub3_fastqc.zip", + "methyldackel", + "methyldackel/SRR389222_sub1.markdup.sorted_CpG.bedGraph", + "methyldackel/SRR389222_sub2.markdup.sorted_CpG.bedGraph", + "methyldackel/SRR389222_sub3.markdup.sorted_CpG.bedGraph", + "methyldackel/mbias", + "methyldackel/mbias/SRR389222_sub1.mbias.txt", + "methyldackel/mbias/SRR389222_sub2.mbias.txt", + "methyldackel/mbias/SRR389222_sub3.mbias.txt", + "multiqc", + "multiqc/bwameth", + "multiqc/bwameth/multiqc_data", + "multiqc/bwameth/multiqc_data/cutadapt_filtered_reads_plot.txt", + "multiqc/bwameth/multiqc_data/cutadapt_trimmed_sequences_plot_3_Counts.txt", + "multiqc/bwameth/multiqc_data/cutadapt_trimmed_sequences_plot_3_Obs_Exp.txt", + "multiqc/bwameth/multiqc_data/fastqc-status-check-heatmap.txt", + "multiqc/bwameth/multiqc_data/fastqc_overrepresented_sequences_plot.txt", + "multiqc/bwameth/multiqc_data/fastqc_per_base_n_content_plot.txt", + "multiqc/bwameth/multiqc_data/fastqc_per_base_sequence_quality_plot.txt", + "multiqc/bwameth/multiqc_data/fastqc_per_sequence_gc_content_plot_Counts.txt", + "multiqc/bwameth/multiqc_data/fastqc_per_sequence_gc_content_plot_Percentages.txt", + "multiqc/bwameth/multiqc_data/fastqc_per_sequence_quality_scores_plot.txt", + "multiqc/bwameth/multiqc_data/fastqc_sequence_counts_plot.txt", + "multiqc/bwameth/multiqc_data/fastqc_sequence_duplication_levels_plot.txt", + "multiqc/bwameth/multiqc_data/fastqc_top_overrepresented_sequences_table.txt", + "multiqc/bwameth/multiqc_data/multiqc.log", + "multiqc/bwameth/multiqc_data/multiqc_citations.txt", + "multiqc/bwameth/multiqc_data/multiqc_cutadapt.txt", + "multiqc/bwameth/multiqc_data/multiqc_data.json", + "multiqc/bwameth/multiqc_data/multiqc_fastqc.txt", + "multiqc/bwameth/multiqc_data/multiqc_general_stats.txt", + "multiqc/bwameth/multiqc_data/multiqc_samtools_flagstat.txt", + "multiqc/bwameth/multiqc_data/multiqc_samtools_stats.txt", + "multiqc/bwameth/multiqc_data/multiqc_software_versions.txt", + "multiqc/bwameth/multiqc_data/multiqc_sources.txt", + "multiqc/bwameth/multiqc_data/picard_histogram.txt", + "multiqc/bwameth/multiqc_data/picard_histogram_1.txt", + "multiqc/bwameth/multiqc_data/picard_histogram_2.txt", + "multiqc/bwameth/multiqc_data/samtools-flagstat-dp_Percentage_of_total.txt", + "multiqc/bwameth/multiqc_data/samtools-flagstat-dp_Read_counts.txt", + "multiqc/bwameth/multiqc_data/samtools-stats-dp.txt", + "multiqc/bwameth/multiqc_data/samtools_alignment_plot.txt", + "multiqc/bwameth/multiqc_plots", + "multiqc/bwameth/multiqc_plots/pdf", + "multiqc/bwameth/multiqc_plots/pdf/cutadapt_filtered_reads_plot-cnt.pdf", + "multiqc/bwameth/multiqc_plots/pdf/cutadapt_filtered_reads_plot-pct.pdf", + "multiqc/bwameth/multiqc_plots/pdf/cutadapt_trimmed_sequences_plot_3_Counts.pdf", + "multiqc/bwameth/multiqc_plots/pdf/cutadapt_trimmed_sequences_plot_3_Obs_Exp.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc-status-check-heatmap.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_overrepresented_sequences_plot.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_per_base_n_content_plot.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_per_base_sequence_quality_plot.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_per_sequence_gc_content_plot_Counts.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_per_sequence_gc_content_plot_Percentages.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_per_sequence_quality_scores_plot.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_sequence_counts_plot-cnt.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_sequence_counts_plot-pct.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_sequence_duplication_levels_plot.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_top_overrepresented_sequences_table.pdf", + "multiqc/bwameth/multiqc_plots/pdf/general_stats_table.pdf", + "multiqc/bwameth/multiqc_plots/pdf/samtools-flagstat-dp_Percentage_of_total.pdf", + "multiqc/bwameth/multiqc_plots/pdf/samtools-flagstat-dp_Read_counts.pdf", + "multiqc/bwameth/multiqc_plots/pdf/samtools-stats-dp.pdf", + "multiqc/bwameth/multiqc_plots/pdf/samtools_alignment_plot-cnt.pdf", + "multiqc/bwameth/multiqc_plots/pdf/samtools_alignment_plot-pct.pdf", + "multiqc/bwameth/multiqc_plots/png", + "multiqc/bwameth/multiqc_plots/png/cutadapt_filtered_reads_plot-cnt.png", + "multiqc/bwameth/multiqc_plots/png/cutadapt_filtered_reads_plot-pct.png", + "multiqc/bwameth/multiqc_plots/png/cutadapt_trimmed_sequences_plot_3_Counts.png", + "multiqc/bwameth/multiqc_plots/png/cutadapt_trimmed_sequences_plot_3_Obs_Exp.png", + "multiqc/bwameth/multiqc_plots/png/fastqc-status-check-heatmap.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_overrepresented_sequences_plot.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_per_base_n_content_plot.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_per_base_sequence_quality_plot.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_per_sequence_gc_content_plot_Counts.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_per_sequence_gc_content_plot_Percentages.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_per_sequence_quality_scores_plot.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_sequence_counts_plot-cnt.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_sequence_counts_plot-pct.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_sequence_duplication_levels_plot.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_top_overrepresented_sequences_table.png", + "multiqc/bwameth/multiqc_plots/png/general_stats_table.png", + "multiqc/bwameth/multiqc_plots/png/samtools-flagstat-dp_Percentage_of_total.png", + "multiqc/bwameth/multiqc_plots/png/samtools-flagstat-dp_Read_counts.png", + "multiqc/bwameth/multiqc_plots/png/samtools-stats-dp.png", + "multiqc/bwameth/multiqc_plots/png/samtools_alignment_plot-cnt.png", + "multiqc/bwameth/multiqc_plots/png/samtools_alignment_plot-pct.png", + "multiqc/bwameth/multiqc_plots/svg", + "multiqc/bwameth/multiqc_plots/svg/cutadapt_filtered_reads_plot-cnt.svg", + "multiqc/bwameth/multiqc_plots/svg/cutadapt_filtered_reads_plot-pct.svg", + "multiqc/bwameth/multiqc_plots/svg/cutadapt_trimmed_sequences_plot_3_Counts.svg", + "multiqc/bwameth/multiqc_plots/svg/cutadapt_trimmed_sequences_plot_3_Obs_Exp.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc-status-check-heatmap.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_overrepresented_sequences_plot.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_per_base_n_content_plot.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_per_base_sequence_quality_plot.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_per_sequence_gc_content_plot_Counts.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_per_sequence_gc_content_plot_Percentages.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_per_sequence_quality_scores_plot.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_sequence_counts_plot-cnt.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_sequence_counts_plot-pct.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_sequence_duplication_levels_plot.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_top_overrepresented_sequences_table.svg", + "multiqc/bwameth/multiqc_plots/svg/general_stats_table.svg", + "multiqc/bwameth/multiqc_plots/svg/samtools-flagstat-dp_Percentage_of_total.svg", + "multiqc/bwameth/multiqc_plots/svg/samtools-flagstat-dp_Read_counts.svg", + "multiqc/bwameth/multiqc_plots/svg/samtools-stats-dp.svg", + "multiqc/bwameth/multiqc_plots/svg/samtools_alignment_plot-cnt.svg", + "multiqc/bwameth/multiqc_plots/svg/samtools_alignment_plot-pct.svg", + "multiqc/bwameth/multiqc_report.html", + "parabricks", + "parabricks/SRR389222_sub1.bam", + "parabricks/SRR389222_sub1.bam.bai", + "parabricks/SRR389222_sub2.bam", + "parabricks/SRR389222_sub2.bam.bai", + "parabricks/SRR389222_sub3.bam", + "parabricks/SRR389222_sub3.bam.bai", + "pipeline_info", + "pipeline_info/nf_core_pipeline_software_mqc_versions.yml", + "trimgalore", + "trimgalore/fastqc", + "trimgalore/fastqc/SRR389222_sub1_trimmed_fastqc.html", + "trimgalore/fastqc/SRR389222_sub2_trimmed_fastqc.html", + "trimgalore/fastqc/SRR389222_sub3_trimmed_fastqc.html", + "trimgalore/fastqc/zips", + "trimgalore/fastqc/zips/SRR389222_sub1_trimmed_fastqc.zip", + "trimgalore/fastqc/zips/SRR389222_sub2_trimmed_fastqc.zip", + "trimgalore/fastqc/zips/SRR389222_sub3_trimmed_fastqc.zip", + "trimgalore/logs", + "trimgalore/logs/SRR389222_sub1.fastq.gz_trimming_report.txt", + "trimgalore/logs/SRR389222_sub2.fastq.gz_trimming_report.txt", + "trimgalore/logs/SRR389222_sub3.fastq.gz_trimming_report.txt" + ], + [ + "SRR389222_sub1.flagstat:md5,ad8159a7683b9c3057bf54ccb77a2ddc", + "SRR389222_sub1.stats:md5,7df53a3d66857dd289911a13fed8497d", + "SRR389222_sub2.flagstat:md5,e768b4f54014d62b096152c44f920bc1", + "SRR389222_sub2.stats:md5,b0426a921c56ed8a86ade65adf7ed6dd", + "SRR389222_sub3.flagstat:md5,39516f7d2c77e6fef9543edf8cddbcd6", + "SRR389222_sub3.stats:md5,7c6053d7f87cbaa8782fc41a53d932fe", + "SRR389222_sub1.markdup.sorted_CpG.bedGraph:md5,0944ad0e3b05791bf6dfd06be7ca4af1", + "SRR389222_sub2.markdup.sorted_CpG.bedGraph:md5,0a3c19af55d1625e91903ae25d1d2098", + "SRR389222_sub3.markdup.sorted_CpG.bedGraph:md5,a91e6b039e9581a9d2ef6f6057918b39", + "SRR389222_sub1.mbias.txt:md5,fce04d733e066d0b933cedc602e2af81", + "SRR389222_sub2.mbias.txt:md5,fce04d733e066d0b933cedc602e2af81", + "SRR389222_sub3.mbias.txt:md5,fce04d733e066d0b933cedc602e2af81", + "cutadapt_filtered_reads_plot.txt:md5,e6192537579202ae2eae0583df8ce114", + "cutadapt_trimmed_sequences_plot_3_Counts.txt:md5,425a49f98170520efd358fa3c4af6ef5", + "cutadapt_trimmed_sequences_plot_3_Obs_Exp.txt:md5,b38beb1846401098b286a088fe63acee", + "fastqc-status-check-heatmap.txt:md5,7289b15cb0ebb19d56331b8f2e82b2b3", + "fastqc_overrepresented_sequences_plot.txt:md5,51888494be30068524e5e338b548318f", + "fastqc_per_base_n_content_plot.txt:md5,f786b052f9613e8dab0679ee6abdb089", + "fastqc_per_base_sequence_quality_plot.txt:md5,a59e07f36250f4ee1cac14f641d12b41", + "fastqc_per_sequence_gc_content_plot_Counts.txt:md5,60230197e11203641ee3a9242ca3cc1e", + "fastqc_per_sequence_gc_content_plot_Percentages.txt:md5,93afe16c96977906f458de742ecfa546", + "fastqc_per_sequence_quality_scores_plot.txt:md5,91318c1d6bb24eb198fb58d00932297b", + "fastqc_sequence_counts_plot.txt:md5,ff6b53f673a0fa22a6be7589292a2cc2", + "fastqc_sequence_duplication_levels_plot.txt:md5,b86ac1b21d5f9adb7875640ddef95b5a", + "fastqc_top_overrepresented_sequences_table.txt:md5,36b233534f727b98041df8c987e05b9e", + "multiqc_citations.txt:md5,ad6c1e326b4276501a3b42eaad747de1", + "multiqc_cutadapt.txt:md5,8562f9a4488d847dd6457a95b983b808", + "multiqc_samtools_flagstat.txt:md5,082795eb003adb27217832b56fbda0d3", + "multiqc_samtools_stats.txt:md5,ac3208aff0cc0866678c3746197558bb", + "picard_histogram.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "picard_histogram_1.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "picard_histogram_2.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "samtools-flagstat-dp_Percentage_of_total.txt:md5,921b0c969e7c154c0cd18b331e2a6c1c", + "samtools-flagstat-dp_Read_counts.txt:md5,c0764320e38e62b48d59f608094804b8", + "samtools-stats-dp.txt:md5,99418ae30d5855a3d8cd9cfdf21dd14d", + "samtools_alignment_plot.txt:md5,34b8fa8a88b6f5ef00153b07aabc028d", + "SRR389222_sub1.bam:md5,a5d5600e09697b1d9a31b0547ab7edcf", + "SRR389222_sub1.bam.bai:md5,c95cb45c3f0a9a82bf78a7914b38ebaf", + "SRR389222_sub2.bam:md5,bb04f6c07f75892530c7d960997ac9a7", + "SRR389222_sub2.bam.bai:md5,fbe5ef44841a8b6028bdb7c14ae67e59", + "SRR389222_sub3.bam:md5,2c1cca453526e1672e87bb07e30af30a", + "SRR389222_sub3.bam.bai:md5,d7d62a85aac95328011ba48c37f3cf1c" + ], + [ + [ + "SRR389222_sub1.deduplicated.sorted.bam", + "97e6767c5ddea54cb2a6498192a09c68" + ], + [ + "SRR389222_sub1.markdup.sorted.bam", + "97e6767c5ddea54cb2a6498192a09c68" + ], + [ + "SRR389222_sub2.deduplicated.sorted.bam", + "313ae71af4023059a295ed8a132fd9ad" + ], + [ + "SRR389222_sub2.markdup.sorted.bam", + "313ae71af4023059a295ed8a132fd9ad" + ], + [ + "SRR389222_sub3.deduplicated.sorted.bam", + "9eaa07df5ad58a78ab57ee55f948a044" + ], + [ + "SRR389222_sub3.markdup.sorted.bam", + "9eaa07df5ad58a78ab57ee55f948a044" + ], + [ + "SRR389222_sub1.bam", + "97e6767c5ddea54cb2a6498192a09c68" + ], + [ + "SRR389222_sub2.bam", + "313ae71af4023059a295ed8a132fd9ad" + ], + [ + "SRR389222_sub3.bam", + "9eaa07df5ad58a78ab57ee55f948a044" + ] + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-12-02T08:58:09.474824" + }, + "Params: bwameth with bwameth-index": { + "content": [ + 47 + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-12-02T08:59:21.276625" + } +} diff --git a/tests/nextflow.config b/tests/nextflow.config index 687ba08a..c1495884 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -28,6 +28,10 @@ process { withName: PRESEQ_LCEXTRAP { errorStrategy = 'ignore' } + + withName: 'PARABRICKS_FQ2BAMMETH' { + ext.args = '--low-memory' + } } // Impose same minimum Nextflow version as the pipeline for testing From 24e35a6ee2dd63e93032ebcd827d21868adc0726 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 09:12:01 +0000 Subject: [PATCH 23/89] update bwameth default test formatting --- tests/bwameth.nf.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bwameth.nf.test b/tests/bwameth.nf.test index 04283574..3268ea0a 100644 --- a/tests/bwameth.nf.test +++ b/tests/bwameth.nf.test @@ -7,8 +7,8 @@ nextflow_pipeline { test("Params: bwameth") { when { params { - aligner = "bwameth" - outdir = "$outputDir" + aligner = "bwameth" + outdir = "$outputDir" } } From 9b43b3ec08de3ea61dd9623605250044406d0c86 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 09:13:56 +0000 Subject: [PATCH 24/89] update bwameth use_gpu test with index name --- tests/bwameth_use_gpu.nf.test | 2 +- tests/bwameth_use_gpu.nf.test.snap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bwameth_use_gpu.nf.test b/tests/bwameth_use_gpu.nf.test index 2cdf7cd0..dd3be93e 100644 --- a/tests/bwameth_use_gpu.nf.test +++ b/tests/bwameth_use_gpu.nf.test @@ -41,7 +41,7 @@ nextflow_pipeline { } } - test("Params: bwameth with bwameth-index") { + test("Params: bwameth | use_gpu | with bwameth-index") { when { params { aligner = "bwameth" diff --git a/tests/bwameth_use_gpu.nf.test.snap b/tests/bwameth_use_gpu.nf.test.snap index 80416005..78c4f399 100644 --- a/tests/bwameth_use_gpu.nf.test.snap +++ b/tests/bwameth_use_gpu.nf.test.snap @@ -276,7 +276,7 @@ }, "timestamp": "2024-12-02T08:58:09.474824" }, - "Params: bwameth with bwameth-index": { + "Params: bwameth | use_gpu | with bwameth-index": { "content": [ 47 ], From 5ebc7e26e6908a5a123e24696305ac4b71b94ae0 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 09:14:57 +0000 Subject: [PATCH 25/89] minor formatting fix --- tests/bwameth.nf.test | 2 +- tests/bwameth_use_gpu.nf.test | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/bwameth.nf.test b/tests/bwameth.nf.test index 3268ea0a..3ed5c859 100644 --- a/tests/bwameth.nf.test +++ b/tests/bwameth.nf.test @@ -18,7 +18,7 @@ nextflow_pipeline { // stable_path: All files in ${params.outdir}/ with stable content def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') // bam_files: All bam files - def bam_files = getAllFilesFromDir(params.outdir, include: ['**/*.bam']) + def bam_files = getAllFilesFromDir(params.outdir, include: ['**/*.bam']) assertAll( { assert workflow.success}, { assert snapshot( diff --git a/tests/bwameth_use_gpu.nf.test b/tests/bwameth_use_gpu.nf.test index dd3be93e..ff35aaa5 100644 --- a/tests/bwameth_use_gpu.nf.test +++ b/tests/bwameth_use_gpu.nf.test @@ -44,9 +44,9 @@ nextflow_pipeline { test("Params: bwameth | use_gpu | with bwameth-index") { when { params { - aligner = "bwameth" + aligner = "bwameth" bwameth_index = "https://github.com/nf-core/test-datasets/raw/methylseq/reference/Bwameth_Index.tar.gz" - outdir = "$outputDir" + outdir = "$outputDir" } } From 210199e4ea1740cd6c81b42794764f4ae68fdde4 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 12:04:57 +0000 Subject: [PATCH 26/89] add fq2bammeth config + test + snap, ci: nf-test job filter by tag cpu --- conf/modules/bwameth_align.config | 2 +- conf/modules/parabricks_fq2bammeth.config | 12 +++++ .../fastq_align_dedup_bwameth.config | 1 + tests/bismark_emseq_clip_r1.nf.test | 1 + tests/bwameth.nf.test | 1 + tests/bwameth_use_gpu.nf.test | 7 ++- tests/bwameth_use_gpu.nf.test.snap | 49 +++++-------------- tests/nextflow.config | 4 -- 8 files changed, 31 insertions(+), 46 deletions(-) create mode 100644 conf/modules/parabricks_fq2bammeth.config diff --git a/conf/modules/bwameth_align.config b/conf/modules/bwameth_align.config index 420a6646..bc4ab758 100644 --- a/conf/modules/bwameth_align.config +++ b/conf/modules/bwameth_align.config @@ -1,6 +1,6 @@ process { withName: BWAMETH_ALIGN { - cache = 'lenient' + cache = 'lenient' // This is set because in the module command the index files are touched so as to have bwameth not complain ext.args = '' publishDir = [ path: { "${params.outdir}/${params.aligner}/alignments" }, diff --git a/conf/modules/parabricks_fq2bammeth.config b/conf/modules/parabricks_fq2bammeth.config new file mode 100644 index 00000000..71e2596a --- /dev/null +++ b/conf/modules/parabricks_fq2bammeth.config @@ -0,0 +1,12 @@ +process { + withName: PARABRICKS_FQ2BAMMETH { + cache = 'lenient' // This is set because in the module command the index files are touched so as to have bwameth not complain + ext.args = '--low-memory' + publishDir = [ + path: { "${params.outdir}/${params.aligner}/alignments" }, + pattern: "*.bam", + mode: params.publish_dir_mode, + enabled: params.save_align_intermeds + ] + } +} diff --git a/conf/subworkflows/fastq_align_dedup_bwameth.config b/conf/subworkflows/fastq_align_dedup_bwameth.config index c68bf509..fec290d8 100644 --- a/conf/subworkflows/fastq_align_dedup_bwameth.config +++ b/conf/subworkflows/fastq_align_dedup_bwameth.config @@ -1,5 +1,6 @@ includeConfig "../modules/bwameth_align.config" +includeConfig "../modules/parabricks_fq2bammeth.config" includeConfig "../modules/samtools_sort.config" includeConfig "../modules/samtools_flagstat.config" includeConfig "../modules/samtools_stats.config" diff --git a/tests/bismark_emseq_clip_r1.nf.test b/tests/bismark_emseq_clip_r1.nf.test index c1bc57a0..0f2bc92a 100644 --- a/tests/bismark_emseq_clip_r1.nf.test +++ b/tests/bismark_emseq_clip_r1.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bismark | em_seq | clip_r1") { when { diff --git a/tests/bwameth.nf.test b/tests/bwameth.nf.test index 3ed5c859..373f2a99 100644 --- a/tests/bwameth.nf.test +++ b/tests/bwameth.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bwameth") { when { diff --git a/tests/bwameth_use_gpu.nf.test b/tests/bwameth_use_gpu.nf.test index ff35aaa5..d3f50457 100644 --- a/tests/bwameth_use_gpu.nf.test +++ b/tests/bwameth_use_gpu.nf.test @@ -3,7 +3,6 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" - tag "gpu" test("Params: bwameth | use_gpu") { @@ -41,12 +40,12 @@ nextflow_pipeline { } } - test("Params: bwameth | use_gpu | with bwameth-index") { + test("Params: bwameth | use_gpu |with bwameth-index") { when { params { - aligner = "bwameth" + aligner = "bwameth" bwameth_index = "https://github.com/nf-core/test-datasets/raw/methylseq/reference/Bwameth_Index.tar.gz" - outdir = "$outputDir" + outdir = "$outputDir" } } diff --git a/tests/bwameth_use_gpu.nf.test.snap b/tests/bwameth_use_gpu.nf.test.snap index 78c4f399..cf54e1e8 100644 --- a/tests/bwameth_use_gpu.nf.test.snap +++ b/tests/bwameth_use_gpu.nf.test.snap @@ -1,4 +1,14 @@ { + "Params: bwameth | use_gpu |with bwameth-index": { + "content": [ + 47 + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-12-02T11:56:58.163433" + }, "Params: bwameth | use_gpu": { "content": [ 36, @@ -164,13 +174,6 @@ "multiqc/bwameth/multiqc_plots/svg/samtools_alignment_plot-cnt.svg", "multiqc/bwameth/multiqc_plots/svg/samtools_alignment_plot-pct.svg", "multiqc/bwameth/multiqc_report.html", - "parabricks", - "parabricks/SRR389222_sub1.bam", - "parabricks/SRR389222_sub1.bam.bai", - "parabricks/SRR389222_sub2.bam", - "parabricks/SRR389222_sub2.bam.bai", - "parabricks/SRR389222_sub3.bam", - "parabricks/SRR389222_sub3.bam.bai", "pipeline_info", "pipeline_info/nf_core_pipeline_software_mqc_versions.yml", "trimgalore", @@ -223,13 +226,7 @@ "samtools-flagstat-dp_Percentage_of_total.txt:md5,921b0c969e7c154c0cd18b331e2a6c1c", "samtools-flagstat-dp_Read_counts.txt:md5,c0764320e38e62b48d59f608094804b8", "samtools-stats-dp.txt:md5,99418ae30d5855a3d8cd9cfdf21dd14d", - "samtools_alignment_plot.txt:md5,34b8fa8a88b6f5ef00153b07aabc028d", - "SRR389222_sub1.bam:md5,a5d5600e09697b1d9a31b0547ab7edcf", - "SRR389222_sub1.bam.bai:md5,c95cb45c3f0a9a82bf78a7914b38ebaf", - "SRR389222_sub2.bam:md5,bb04f6c07f75892530c7d960997ac9a7", - "SRR389222_sub2.bam.bai:md5,fbe5ef44841a8b6028bdb7c14ae67e59", - "SRR389222_sub3.bam:md5,2c1cca453526e1672e87bb07e30af30a", - "SRR389222_sub3.bam.bai:md5,d7d62a85aac95328011ba48c37f3cf1c" + "samtools_alignment_plot.txt:md5,34b8fa8a88b6f5ef00153b07aabc028d" ], [ [ @@ -255,18 +252,6 @@ [ "SRR389222_sub3.markdup.sorted.bam", "9eaa07df5ad58a78ab57ee55f948a044" - ], - [ - "SRR389222_sub1.bam", - "97e6767c5ddea54cb2a6498192a09c68" - ], - [ - "SRR389222_sub2.bam", - "313ae71af4023059a295ed8a132fd9ad" - ], - [ - "SRR389222_sub3.bam", - "9eaa07df5ad58a78ab57ee55f948a044" ] ] ], @@ -274,16 +259,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.0" }, - "timestamp": "2024-12-02T08:58:09.474824" - }, - "Params: bwameth | use_gpu | with bwameth-index": { - "content": [ - 47 - ], - "meta": { - "nf-test": "0.9.2", - "nextflow": "24.10.0" - }, - "timestamp": "2024-12-02T08:59:21.276625" + "timestamp": "2024-12-02T11:55:47.061799" } } diff --git a/tests/nextflow.config b/tests/nextflow.config index c1495884..687ba08a 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -28,10 +28,6 @@ process { withName: PRESEQ_LCEXTRAP { errorStrategy = 'ignore' } - - withName: 'PARABRICKS_FQ2BAMMETH' { - ext.args = '--low-memory' - } } // Impose same minimum Nextflow version as the pipeline for testing From c4d81d43580343e5099af5ab71dde9fd04cdd555 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 12:05:20 +0000 Subject: [PATCH 27/89] ci: nf-test job filter by tag cpu --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40629809..7e761f7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,7 +186,8 @@ jobs: --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ --changed-since HEAD^ \ --profile "+${{ matrix.profile }}" \ - --filter ${{ matrix.filter }} + --filter ${{ matrix.filter }} \ + --tag "cpu" - name: Output log on failure if: failure() From 67c8708b33cdd6be5d2cb16ed5a5ec20f1af02f9 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 12:07:56 +0000 Subject: [PATCH 28/89] add cpu/gpu tags to tests --- tests/bismark_hisat.nf.test | 1 + tests/bismark_hisat_rrbs.nf.test | 1 + tests/bismark_hisat_save_reference_save_align_intermeds.nf.test | 1 + tests/bismark_nomeseq.nf.test | 1 + tests/bismark_rrbs.nf.test | 1 + tests/bismark_run_preseq.nf.test | 1 + tests/bismark_run_qualimap.nf.test | 1 + tests/bismark_save_reference_save_align_intermeds.nf.test | 1 + tests/bismark_skip_deduplication.nf.test | 1 + tests/bismark_skip_multiqc.nf.test | 1 + tests/bismark_skip_trimming.nf.test | 1 + tests/bwameth_rrbs.nf.test | 1 + tests/bwameth_save_reference_save_align_intermeds.nf.test | 1 + tests/bwameth_skip_deduplication.nf.test | 1 + tests/bwameth_skip_trimming.nf.test | 1 + tests/default.nf.test | 1 + 16 files changed, 16 insertions(+) diff --git a/tests/bismark_hisat.nf.test b/tests/bismark_hisat.nf.test index 8ac37b1e..e6eb541f 100644 --- a/tests/bismark_hisat.nf.test +++ b/tests/bismark_hisat.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bismark_hisat") { when { diff --git a/tests/bismark_hisat_rrbs.nf.test b/tests/bismark_hisat_rrbs.nf.test index b8715634..b3299798 100644 --- a/tests/bismark_hisat_rrbs.nf.test +++ b/tests/bismark_hisat_rrbs.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bismark_hisat | rrbs") { when { diff --git a/tests/bismark_hisat_save_reference_save_align_intermeds.nf.test b/tests/bismark_hisat_save_reference_save_align_intermeds.nf.test index 2f2b0a0c..b42aa6cf 100644 --- a/tests/bismark_hisat_save_reference_save_align_intermeds.nf.test +++ b/tests/bismark_hisat_save_reference_save_align_intermeds.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bismark_hisat | save_reference | save_align_intermeds") { when { diff --git a/tests/bismark_nomeseq.nf.test b/tests/bismark_nomeseq.nf.test index 4cd98ef6..cd2f81e6 100644 --- a/tests/bismark_nomeseq.nf.test +++ b/tests/bismark_nomeseq.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bismark | nomeseq") { when { diff --git a/tests/bismark_rrbs.nf.test b/tests/bismark_rrbs.nf.test index 185a9998..65b9393c 100644 --- a/tests/bismark_rrbs.nf.test +++ b/tests/bismark_rrbs.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bismark | rrbs") { when { diff --git a/tests/bismark_run_preseq.nf.test b/tests/bismark_run_preseq.nf.test index 229e6ea5..eae52753 100644 --- a/tests/bismark_run_preseq.nf.test +++ b/tests/bismark_run_preseq.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bismark | run_preseq") { when { diff --git a/tests/bismark_run_qualimap.nf.test b/tests/bismark_run_qualimap.nf.test index a224a13b..f8c309b7 100644 --- a/tests/bismark_run_qualimap.nf.test +++ b/tests/bismark_run_qualimap.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bismark | run_qualimap") { when { diff --git a/tests/bismark_save_reference_save_align_intermeds.nf.test b/tests/bismark_save_reference_save_align_intermeds.nf.test index cc5fc7a0..5238c116 100644 --- a/tests/bismark_save_reference_save_align_intermeds.nf.test +++ b/tests/bismark_save_reference_save_align_intermeds.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bismark | save_reference | save_align_intermeds") { when { diff --git a/tests/bismark_skip_deduplication.nf.test b/tests/bismark_skip_deduplication.nf.test index 055b77e6..3b6066a6 100644 --- a/tests/bismark_skip_deduplication.nf.test +++ b/tests/bismark_skip_deduplication.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bismark | skip_deduplication") { when { diff --git a/tests/bismark_skip_multiqc.nf.test b/tests/bismark_skip_multiqc.nf.test index 99cbe91f..3af45a91 100644 --- a/tests/bismark_skip_multiqc.nf.test +++ b/tests/bismark_skip_multiqc.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bismark | skip_multiqc") { when { diff --git a/tests/bismark_skip_trimming.nf.test b/tests/bismark_skip_trimming.nf.test index 28b415c6..db115373 100644 --- a/tests/bismark_skip_trimming.nf.test +++ b/tests/bismark_skip_trimming.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bismark | skip_trimming") { when { diff --git a/tests/bwameth_rrbs.nf.test b/tests/bwameth_rrbs.nf.test index cc0fba43..4ad081b4 100644 --- a/tests/bwameth_rrbs.nf.test +++ b/tests/bwameth_rrbs.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bwameth | rrbs") { when { diff --git a/tests/bwameth_save_reference_save_align_intermeds.nf.test b/tests/bwameth_save_reference_save_align_intermeds.nf.test index 2ce41cd3..6a924304 100644 --- a/tests/bwameth_save_reference_save_align_intermeds.nf.test +++ b/tests/bwameth_save_reference_save_align_intermeds.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bwameth | save_reference | save_align_intermeds") { when { diff --git a/tests/bwameth_skip_deduplication.nf.test b/tests/bwameth_skip_deduplication.nf.test index 0b208bf4..daa22c57 100644 --- a/tests/bwameth_skip_deduplication.nf.test +++ b/tests/bwameth_skip_deduplication.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bwameth | skip_deduplication") { when { diff --git a/tests/bwameth_skip_trimming.nf.test b/tests/bwameth_skip_trimming.nf.test index 3b01599d..92c185d0 100644 --- a/tests/bwameth_skip_trimming.nf.test +++ b/tests/bwameth_skip_trimming.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: bwameth | skip_trimming") { when { diff --git a/tests/default.nf.test b/tests/default.nf.test index 6a210a07..7108fe98 100644 --- a/tests/default.nf.test +++ b/tests/default.nf.test @@ -3,6 +3,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "../main.nf" config "./nextflow.config" + tag "cpu" test("Params: default") { when { From d6917c309f5038fe6c37d654d675acbcca0db857 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 12:10:26 +0000 Subject: [PATCH 29/89] touch cpu and gpu test for debug --- tests/bismark_hisat.nf.test | 4 ++-- tests/bwameth_use_gpu.nf.test | 20 -------------------- tests/bwameth_use_gpu.nf.test.snap | 10 ---------- 3 files changed, 2 insertions(+), 32 deletions(-) diff --git a/tests/bismark_hisat.nf.test b/tests/bismark_hisat.nf.test index e6eb541f..42bce0ae 100644 --- a/tests/bismark_hisat.nf.test +++ b/tests/bismark_hisat.nf.test @@ -8,8 +8,8 @@ nextflow_pipeline { test("Params: bismark_hisat") { when { params { - aligner = "bismark_hisat" - outdir = "$outputDir" + aligner = "bismark_hisat" + outdir = "$outputDir" } } diff --git a/tests/bwameth_use_gpu.nf.test b/tests/bwameth_use_gpu.nf.test index d3f50457..34eced52 100644 --- a/tests/bwameth_use_gpu.nf.test +++ b/tests/bwameth_use_gpu.nf.test @@ -40,24 +40,4 @@ nextflow_pipeline { } } - test("Params: bwameth | use_gpu |with bwameth-index") { - when { - params { - aligner = "bwameth" - bwameth_index = "https://github.com/nf-core/test-datasets/raw/methylseq/reference/Bwameth_Index.tar.gz" - outdir = "$outputDir" - } - } - - then { - assertAll( - { assert workflow.success }, - { assert snapshot( - // Number of tasks - workflow.trace.succeeded().size() - ).match() } - ) - } - } - } diff --git a/tests/bwameth_use_gpu.nf.test.snap b/tests/bwameth_use_gpu.nf.test.snap index cf54e1e8..306d1f6b 100644 --- a/tests/bwameth_use_gpu.nf.test.snap +++ b/tests/bwameth_use_gpu.nf.test.snap @@ -1,14 +1,4 @@ { - "Params: bwameth | use_gpu |with bwameth-index": { - "content": [ - 47 - ], - "meta": { - "nf-test": "0.9.2", - "nextflow": "24.10.0" - }, - "timestamp": "2024-12-02T11:56:58.163433" - }, "Params: bwameth | use_gpu": { "content": [ 36, From 754065d7d39de26548f76e2fa01f58608ac175ed Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 12:47:16 +0000 Subject: [PATCH 30/89] add bwameth run_qualimap test --- tests/.nftignore | 2 +- tests/bismark_run_qualimap.nf.test.snap | 8 +- tests/bwameth_run_qualimap.nf.test | 42 ++ tests/bwameth_run_qualimap.nf.test.snap | 586 ++++++++++++++++++++++++ 4 files changed, 631 insertions(+), 7 deletions(-) create mode 100644 tests/bwameth_run_qualimap.nf.test create mode 100644 tests/bwameth_run_qualimap.nf.test.snap diff --git a/tests/.nftignore b/tests/.nftignore index 7c0558cf..5c3f0a21 100644 --- a/tests/.nftignore +++ b/tests/.nftignore @@ -14,7 +14,7 @@ multiqc/{bismark,bismark_hisat,bwameth}/multiqc_data/multiqc_sources.txt multiqc/{bismark,bismark_hisat,bwameth}/multiqc_report.html multiqc/{bismark,bismark_hisat,bwameth}/multiqc_plots/{pdf,png,svg}/*.{pdf,png,svg} pipeline_info/*.{html,json,txt,yml} -qualimap/**/qualimapReport.html +{bismark,bismark_hisat,bwameth}/qualimap/bamqc/*/qualimapReport.html qualimap/**/css/* qualimap/**/images_qualimapReport/* qualimap/**/raw_data_qualimapReport/* diff --git a/tests/bismark_run_qualimap.nf.test.snap b/tests/bismark_run_qualimap.nf.test.snap index ba30b070..78214c46 100644 --- a/tests/bismark_run_qualimap.nf.test.snap +++ b/tests/bismark_run_qualimap.nf.test.snap @@ -556,7 +556,6 @@ "genome_mapping_quality_histogram.png:md5,74776b1e7e8c8876af1cf89d9dbb5f31", "genome_reads_content_per_read_position.png:md5,c6920dc357411880132c207c7d0aa68d", "genome_uniq_read_starts_histogram.png:md5,8f7549a63eee3d3f022699aaa5bb858d", - "qualimapReport.html:md5,bbc015c0d1f3c41a08c82fb7a6839039", "coverage_across_reference.txt:md5,2bb9eb828acfd71ace8f1e6a2ca124b4", "coverage_histogram.txt:md5,6ee761660a6072e5b6b75b7fe8d2c69e", "duplication_rate_histogram.txt:md5,d5f25ad3572b98880b6904dcf4b3ed25", @@ -601,7 +600,6 @@ "genome_mapping_quality_histogram.png:md5,7ba0db9375d09fd49946b9bca4ba92b4", "genome_reads_content_per_read_position.png:md5,1a6a7717ea948d2518808f72f58ffdef", "genome_uniq_read_starts_histogram.png:md5,395413ab1d0e10af72b116fd8cb12685", - "qualimapReport.html:md5,1fe626f663d66a7ce0c6e1e0207d829d", "coverage_across_reference.txt:md5,081ddc3be3349b0c57d9676c03d84803", "coverage_histogram.txt:md5,47b75803a6925e883e254bb91d6220d2", "duplication_rate_histogram.txt:md5,3d06b34e40d1fdeb55c4ad19dc6ddad1", @@ -643,7 +641,6 @@ "genome_mapping_quality_histogram.png:md5,017451f46bc875ac58e7906bf7979956", "genome_reads_content_per_read_position.png:md5,fd94ec5bfe5ca4adfa6ebdeaebe25157", "genome_uniq_read_starts_histogram.png:md5,2f2555ef88c3bda05315469e684f5b19", - "qualimapReport.html:md5,af640b29bd28f80bdc0ab6fe7d9c97f4", "coverage_across_reference.txt:md5,a76b3da0ebb2f8bd6c36ec1902cf13f2", "coverage_histogram.txt:md5,ca854c05e9619233b7d95d44051cab80", "duplication_rate_histogram.txt:md5,530bb05c648d610227e121e1e74f41f8", @@ -685,7 +682,6 @@ "genome_mapping_quality_histogram.png:md5,a22ee10fcb9e34a84f607b089855fb92", "genome_reads_content_per_read_position.png:md5,9fa406f353148448f6b9ac11880bc6de", "genome_uniq_read_starts_histogram.png:md5,cf2fe3e30e11e7d6eb2db679cd12dc03", - "qualimapReport.html:md5,c8114fbb2a63da68111f0d4cbb8a1973", "coverage_across_reference.txt:md5,fb9c989fc6c15d0a8fd0a58b8d3c20ce", "coverage_histogram.txt:md5,8a3f45461442657433914c6a5ecb1301", "duplication_rate_histogram.txt:md5,ec9a705fd3a300b126d556afa14c326b", @@ -752,6 +748,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-11-28T09:47:26.992492832" + "timestamp": "2024-12-02T12:34:53.964190449" } -} +} \ No newline at end of file diff --git a/tests/bwameth_run_qualimap.nf.test b/tests/bwameth_run_qualimap.nf.test new file mode 100644 index 00000000..5f671e42 --- /dev/null +++ b/tests/bwameth_run_qualimap.nf.test @@ -0,0 +1,42 @@ +nextflow_pipeline { + + name "Test Workflow main.nf" + script "../main.nf" + config "./nextflow.config" + tag "cpu" + + test("Params: bwameth | run_qualimap") { + when { + params { + aligner = "bwameth" + outdir = "$outputDir" + run_qualimap = true + } + } + + then { + // stable_name: All files + folders in ${params.outdir}/ with a stable name + def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_path: All files in ${params.outdir}/ with stable content + def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + // bam_files: All bam files + def bam_files = getAllFilesFromDir(params.outdir, include: ['**/*.bam']) + assertAll( + { assert workflow.success}, + { assert snapshot( + // Number of tasks + workflow.trace.succeeded().size(), + // pipeline versions.yml file for multiqc from which Nextflow version is removed because we tests pipelines on multiple Nextflow versions + removeNextflowVersion("$outputDir/pipeline_info/nf_core_pipeline_software_mqc_versions.yml"), + // All stable path name + stable_name, + // All files with stable contents + stable_path, + // All bam files + bam_files.collect{ file -> [ file.getName(), bam(file.toString()).getReadsMD5() ] } + ).match() } + ) + } + } + +} diff --git a/tests/bwameth_run_qualimap.nf.test.snap b/tests/bwameth_run_qualimap.nf.test.snap new file mode 100644 index 00000000..cc1376a2 --- /dev/null +++ b/tests/bwameth_run_qualimap.nf.test.snap @@ -0,0 +1,586 @@ +{ + "Params: bwameth | run_qualimap": { + "content": [ + 51, + { + "BWAMETH_ALIGN": { + "bwameth": "0.2.7" + }, + "BWAMETH_INDEX": { + "bwameth": "0.2.7" + }, + "FASTQC": { + "fastqc": "0.12.1" + }, + "GUNZIP": { + "gunzip": 1.1 + }, + "QUALIMAP_BAMQC": { + "qualimap": 2.3 + }, + "TRIMGALORE": { + "trimgalore": "0.6.10", + "cutadapt": 4.9 + }, + "Workflow": { + "nf-core/methylseq": "v2.8.0dev" + } + }, + [ + "bwameth", + "bwameth/alignments", + "bwameth/alignments/samtools_stats", + "bwameth/alignments/samtools_stats/Ecoli_10K_methylated.flagstat", + "bwameth/alignments/samtools_stats/Ecoli_10K_methylated.stats", + "bwameth/alignments/samtools_stats/SRR389222_sub1.flagstat", + "bwameth/alignments/samtools_stats/SRR389222_sub1.stats", + "bwameth/alignments/samtools_stats/SRR389222_sub2.flagstat", + "bwameth/alignments/samtools_stats/SRR389222_sub2.stats", + "bwameth/alignments/samtools_stats/SRR389222_sub3.flagstat", + "bwameth/alignments/samtools_stats/SRR389222_sub3.stats", + "bwameth/deduplicated", + "bwameth/deduplicated/Ecoli_10K_methylated.deduplicated.sorted.bam", + "bwameth/deduplicated/Ecoli_10K_methylated.markdup.sorted.bam", + "bwameth/deduplicated/Ecoli_10K_methylated.markdup.sorted.bam.bai", + "bwameth/deduplicated/SRR389222_sub1.deduplicated.sorted.bam", + "bwameth/deduplicated/SRR389222_sub1.markdup.sorted.bam", + "bwameth/deduplicated/SRR389222_sub1.markdup.sorted.bam.bai", + "bwameth/deduplicated/SRR389222_sub2.deduplicated.sorted.bam", + "bwameth/deduplicated/SRR389222_sub2.markdup.sorted.bam", + "bwameth/deduplicated/SRR389222_sub2.markdup.sorted.bam.bai", + "bwameth/deduplicated/SRR389222_sub3.deduplicated.sorted.bam", + "bwameth/deduplicated/SRR389222_sub3.markdup.sorted.bam", + "bwameth/deduplicated/SRR389222_sub3.markdup.sorted.bam.bai", + "bwameth/deduplicated/picard_metrics", + "bwameth/deduplicated/picard_metrics/Ecoli_10K_methylated.markdup.sorted.MarkDuplicates.metrics.txt", + "bwameth/deduplicated/picard_metrics/SRR389222_sub1.markdup.sorted.MarkDuplicates.metrics.txt", + "bwameth/deduplicated/picard_metrics/SRR389222_sub2.markdup.sorted.MarkDuplicates.metrics.txt", + "bwameth/deduplicated/picard_metrics/SRR389222_sub3.markdup.sorted.MarkDuplicates.metrics.txt", + "bwameth/qualimap", + "bwameth/qualimap/bamqc", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/agogo.css", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/ajax-loader.gif", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/basic.css", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/bgfooter.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/bgtop.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/comment-bright.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/comment-close.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/comment.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/doctools.js", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/down-pressed.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/down.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/file.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/jquery.js", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/minus.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/plus.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/pygments.css", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/qualimap_logo_small.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/report.css", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/searchtools.js", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/underscore.js", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/up-pressed.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/up.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/css/websupport.js", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/genome_results.txt", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport/genome_coverage_0to50_histogram.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport/genome_coverage_across_reference.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport/genome_coverage_histogram.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport/genome_coverage_quotes.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport/genome_gc_content_per_window.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport/genome_homopolymer_indels.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport/genome_insert_size_across_reference.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport/genome_insert_size_histogram.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport/genome_mapping_quality_across_reference.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport/genome_mapping_quality_histogram.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport/genome_reads_clipping_profile.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport/genome_reads_content_per_read_position.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/images_qualimapReport/genome_uniq_read_starts_histogram.png", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/qualimapReport.html", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/raw_data_qualimapReport", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/raw_data_qualimapReport/coverage_across_reference.txt", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/raw_data_qualimapReport/coverage_histogram.txt", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/raw_data_qualimapReport/duplication_rate_histogram.txt", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/raw_data_qualimapReport/genome_fraction_coverage.txt", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/raw_data_qualimapReport/homopolymer_indels.txt", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/raw_data_qualimapReport/insert_size_across_reference.txt", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/raw_data_qualimapReport/insert_size_histogram.txt", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/raw_data_qualimapReport/mapped_reads_clipping_profile.txt", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/raw_data_qualimapReport/mapped_reads_gc-content_distribution.txt", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/raw_data_qualimapReport/mapped_reads_nucleotide_content.txt", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/raw_data_qualimapReport/mapping_quality_across_reference.txt", + "bwameth/qualimap/bamqc/Ecoli_10K_methylated/raw_data_qualimapReport/mapping_quality_histogram.txt", + "bwameth/qualimap/bamqc/SRR389222_sub1", + "bwameth/qualimap/bamqc/SRR389222_sub1/css", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/agogo.css", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/ajax-loader.gif", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/basic.css", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/bgfooter.png", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/bgtop.png", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/comment-bright.png", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/comment-close.png", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/comment.png", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/doctools.js", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/down-pressed.png", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/down.png", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/file.png", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/jquery.js", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/minus.png", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/plus.png", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/pygments.css", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/qualimap_logo_small.png", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/report.css", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/searchtools.js", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/underscore.js", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/up-pressed.png", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/up.png", + "bwameth/qualimap/bamqc/SRR389222_sub1/css/websupport.js", + "bwameth/qualimap/bamqc/SRR389222_sub1/genome_results.txt", + "bwameth/qualimap/bamqc/SRR389222_sub1/qualimapReport.html", + "bwameth/qualimap/bamqc/SRR389222_sub1/raw_data_qualimapReport", + "bwameth/qualimap/bamqc/SRR389222_sub2", + "bwameth/qualimap/bamqc/SRR389222_sub2/css", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/agogo.css", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/ajax-loader.gif", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/basic.css", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/bgfooter.png", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/bgtop.png", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/comment-bright.png", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/comment-close.png", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/comment.png", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/doctools.js", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/down-pressed.png", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/down.png", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/file.png", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/jquery.js", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/minus.png", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/plus.png", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/pygments.css", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/qualimap_logo_small.png", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/report.css", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/searchtools.js", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/underscore.js", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/up-pressed.png", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/up.png", + "bwameth/qualimap/bamqc/SRR389222_sub2/css/websupport.js", + "bwameth/qualimap/bamqc/SRR389222_sub2/genome_results.txt", + "bwameth/qualimap/bamqc/SRR389222_sub2/qualimapReport.html", + "bwameth/qualimap/bamqc/SRR389222_sub2/raw_data_qualimapReport", + "bwameth/qualimap/bamqc/SRR389222_sub3", + "bwameth/qualimap/bamqc/SRR389222_sub3/css", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/agogo.css", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/ajax-loader.gif", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/basic.css", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/bgfooter.png", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/bgtop.png", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/comment-bright.png", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/comment-close.png", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/comment.png", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/doctools.js", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/down-pressed.png", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/down.png", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/file.png", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/jquery.js", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/minus.png", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/plus.png", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/pygments.css", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/qualimap_logo_small.png", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/report.css", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/searchtools.js", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/underscore.js", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/up-pressed.png", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/up.png", + "bwameth/qualimap/bamqc/SRR389222_sub3/css/websupport.js", + "bwameth/qualimap/bamqc/SRR389222_sub3/genome_results.txt", + "bwameth/qualimap/bamqc/SRR389222_sub3/qualimapReport.html", + "bwameth/qualimap/bamqc/SRR389222_sub3/raw_data_qualimapReport", + "bwameth/qualimap/bamqc/versions.yml", + "fastqc", + "fastqc/Ecoli_10K_methylated_1_fastqc.html", + "fastqc/Ecoli_10K_methylated_2_fastqc.html", + "fastqc/SRR389222_sub1_fastqc.html", + "fastqc/SRR389222_sub2_fastqc.html", + "fastqc/SRR389222_sub3_fastqc.html", + "fastqc/zips", + "fastqc/zips/Ecoli_10K_methylated_1_fastqc.zip", + "fastqc/zips/Ecoli_10K_methylated_2_fastqc.zip", + "fastqc/zips/SRR389222_sub1_fastqc.zip", + "fastqc/zips/SRR389222_sub2_fastqc.zip", + "fastqc/zips/SRR389222_sub3_fastqc.zip", + "methyldackel", + "methyldackel/Ecoli_10K_methylated.markdup.sorted_CpG.bedGraph", + "methyldackel/SRR389222_sub1.markdup.sorted_CpG.bedGraph", + "methyldackel/SRR389222_sub2.markdup.sorted_CpG.bedGraph", + "methyldackel/SRR389222_sub3.markdup.sorted_CpG.bedGraph", + "methyldackel/mbias", + "methyldackel/mbias/Ecoli_10K_methylated.mbias.txt", + "methyldackel/mbias/SRR389222_sub1.mbias.txt", + "methyldackel/mbias/SRR389222_sub2.mbias.txt", + "methyldackel/mbias/SRR389222_sub3.mbias.txt", + "multiqc", + "multiqc/bwameth", + "multiqc/bwameth/multiqc_data", + "multiqc/bwameth/multiqc_data/cutadapt_filtered_reads_plot.txt", + "multiqc/bwameth/multiqc_data/cutadapt_trimmed_sequences_plot_3_Counts.txt", + "multiqc/bwameth/multiqc_data/cutadapt_trimmed_sequences_plot_3_Obs_Exp.txt", + "multiqc/bwameth/multiqc_data/fastqc-status-check-heatmap.txt", + "multiqc/bwameth/multiqc_data/fastqc_adapter_content_plot.txt", + "multiqc/bwameth/multiqc_data/fastqc_overrepresented_sequences_plot.txt", + "multiqc/bwameth/multiqc_data/fastqc_per_base_n_content_plot.txt", + "multiqc/bwameth/multiqc_data/fastqc_per_base_sequence_quality_plot.txt", + "multiqc/bwameth/multiqc_data/fastqc_per_sequence_gc_content_plot_Counts.txt", + "multiqc/bwameth/multiqc_data/fastqc_per_sequence_gc_content_plot_Percentages.txt", + "multiqc/bwameth/multiqc_data/fastqc_per_sequence_quality_scores_plot.txt", + "multiqc/bwameth/multiqc_data/fastqc_sequence_counts_plot.txt", + "multiqc/bwameth/multiqc_data/fastqc_sequence_duplication_levels_plot.txt", + "multiqc/bwameth/multiqc_data/fastqc_top_overrepresented_sequences_table.txt", + "multiqc/bwameth/multiqc_data/multiqc.log", + "multiqc/bwameth/multiqc_data/multiqc_citations.txt", + "multiqc/bwameth/multiqc_data/multiqc_cutadapt.txt", + "multiqc/bwameth/multiqc_data/multiqc_data.json", + "multiqc/bwameth/multiqc_data/multiqc_fastqc.txt", + "multiqc/bwameth/multiqc_data/multiqc_general_stats.txt", + "multiqc/bwameth/multiqc_data/multiqc_picard_dups.txt", + "multiqc/bwameth/multiqc_data/multiqc_qualimap_bamqc_genome_results.txt", + "multiqc/bwameth/multiqc_data/multiqc_samtools_flagstat.txt", + "multiqc/bwameth/multiqc_data/multiqc_samtools_stats.txt", + "multiqc/bwameth/multiqc_data/multiqc_software_versions.txt", + "multiqc/bwameth/multiqc_data/multiqc_sources.txt", + "multiqc/bwameth/multiqc_data/picard_deduplication.txt", + "multiqc/bwameth/multiqc_data/picard_histogram.txt", + "multiqc/bwameth/multiqc_data/picard_histogram_1.txt", + "multiqc/bwameth/multiqc_data/picard_histogram_2.txt", + "multiqc/bwameth/multiqc_data/qualimap_coverage_histogram.txt", + "multiqc/bwameth/multiqc_data/qualimap_gc_content.txt", + "multiqc/bwameth/multiqc_data/qualimap_genome_fraction.txt", + "multiqc/bwameth/multiqc_data/qualimap_insert_size.txt", + "multiqc/bwameth/multiqc_data/samtools-flagstat-dp_Percentage_of_total.txt", + "multiqc/bwameth/multiqc_data/samtools-flagstat-dp_Read_counts.txt", + "multiqc/bwameth/multiqc_data/samtools-stats-dp.txt", + "multiqc/bwameth/multiqc_data/samtools_alignment_plot.txt", + "multiqc/bwameth/multiqc_plots", + "multiqc/bwameth/multiqc_plots/pdf", + "multiqc/bwameth/multiqc_plots/pdf/cutadapt_filtered_reads_plot-cnt.pdf", + "multiqc/bwameth/multiqc_plots/pdf/cutadapt_filtered_reads_plot-pct.pdf", + "multiqc/bwameth/multiqc_plots/pdf/cutadapt_trimmed_sequences_plot_3_Counts.pdf", + "multiqc/bwameth/multiqc_plots/pdf/cutadapt_trimmed_sequences_plot_3_Obs_Exp.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc-status-check-heatmap.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_adapter_content_plot.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_overrepresented_sequences_plot.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_per_base_n_content_plot.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_per_base_sequence_quality_plot.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_per_sequence_gc_content_plot_Counts.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_per_sequence_gc_content_plot_Percentages.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_per_sequence_quality_scores_plot.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_sequence_counts_plot-cnt.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_sequence_counts_plot-pct.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_sequence_duplication_levels_plot.pdf", + "multiqc/bwameth/multiqc_plots/pdf/fastqc_top_overrepresented_sequences_table.pdf", + "multiqc/bwameth/multiqc_plots/pdf/general_stats_table.pdf", + "multiqc/bwameth/multiqc_plots/pdf/picard_deduplication-cnt.pdf", + "multiqc/bwameth/multiqc_plots/pdf/picard_deduplication-pct.pdf", + "multiqc/bwameth/multiqc_plots/pdf/qualimap_coverage_histogram.pdf", + "multiqc/bwameth/multiqc_plots/pdf/qualimap_gc_content.pdf", + "multiqc/bwameth/multiqc_plots/pdf/qualimap_genome_fraction.pdf", + "multiqc/bwameth/multiqc_plots/pdf/qualimap_insert_size.pdf", + "multiqc/bwameth/multiqc_plots/pdf/samtools-flagstat-dp_Percentage_of_total.pdf", + "multiqc/bwameth/multiqc_plots/pdf/samtools-flagstat-dp_Read_counts.pdf", + "multiqc/bwameth/multiqc_plots/pdf/samtools-stats-dp.pdf", + "multiqc/bwameth/multiqc_plots/pdf/samtools_alignment_plot-cnt.pdf", + "multiqc/bwameth/multiqc_plots/pdf/samtools_alignment_plot-pct.pdf", + "multiqc/bwameth/multiqc_plots/png", + "multiqc/bwameth/multiqc_plots/png/cutadapt_filtered_reads_plot-cnt.png", + "multiqc/bwameth/multiqc_plots/png/cutadapt_filtered_reads_plot-pct.png", + "multiqc/bwameth/multiqc_plots/png/cutadapt_trimmed_sequences_plot_3_Counts.png", + "multiqc/bwameth/multiqc_plots/png/cutadapt_trimmed_sequences_plot_3_Obs_Exp.png", + "multiqc/bwameth/multiqc_plots/png/fastqc-status-check-heatmap.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_adapter_content_plot.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_overrepresented_sequences_plot.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_per_base_n_content_plot.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_per_base_sequence_quality_plot.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_per_sequence_gc_content_plot_Counts.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_per_sequence_gc_content_plot_Percentages.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_per_sequence_quality_scores_plot.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_sequence_counts_plot-cnt.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_sequence_counts_plot-pct.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_sequence_duplication_levels_plot.png", + "multiqc/bwameth/multiqc_plots/png/fastqc_top_overrepresented_sequences_table.png", + "multiqc/bwameth/multiqc_plots/png/general_stats_table.png", + "multiqc/bwameth/multiqc_plots/png/picard_deduplication-cnt.png", + "multiqc/bwameth/multiqc_plots/png/picard_deduplication-pct.png", + "multiqc/bwameth/multiqc_plots/png/qualimap_coverage_histogram.png", + "multiqc/bwameth/multiqc_plots/png/qualimap_gc_content.png", + "multiqc/bwameth/multiqc_plots/png/qualimap_genome_fraction.png", + "multiqc/bwameth/multiqc_plots/png/qualimap_insert_size.png", + "multiqc/bwameth/multiqc_plots/png/samtools-flagstat-dp_Percentage_of_total.png", + "multiqc/bwameth/multiqc_plots/png/samtools-flagstat-dp_Read_counts.png", + "multiqc/bwameth/multiqc_plots/png/samtools-stats-dp.png", + "multiqc/bwameth/multiqc_plots/png/samtools_alignment_plot-cnt.png", + "multiqc/bwameth/multiqc_plots/png/samtools_alignment_plot-pct.png", + "multiqc/bwameth/multiqc_plots/svg", + "multiqc/bwameth/multiqc_plots/svg/cutadapt_filtered_reads_plot-cnt.svg", + "multiqc/bwameth/multiqc_plots/svg/cutadapt_filtered_reads_plot-pct.svg", + "multiqc/bwameth/multiqc_plots/svg/cutadapt_trimmed_sequences_plot_3_Counts.svg", + "multiqc/bwameth/multiqc_plots/svg/cutadapt_trimmed_sequences_plot_3_Obs_Exp.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc-status-check-heatmap.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_adapter_content_plot.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_overrepresented_sequences_plot.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_per_base_n_content_plot.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_per_base_sequence_quality_plot.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_per_sequence_gc_content_plot_Counts.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_per_sequence_gc_content_plot_Percentages.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_per_sequence_quality_scores_plot.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_sequence_counts_plot-cnt.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_sequence_counts_plot-pct.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_sequence_duplication_levels_plot.svg", + "multiqc/bwameth/multiqc_plots/svg/fastqc_top_overrepresented_sequences_table.svg", + "multiqc/bwameth/multiqc_plots/svg/general_stats_table.svg", + "multiqc/bwameth/multiqc_plots/svg/picard_deduplication-cnt.svg", + "multiqc/bwameth/multiqc_plots/svg/picard_deduplication-pct.svg", + "multiqc/bwameth/multiqc_plots/svg/qualimap_coverage_histogram.svg", + "multiqc/bwameth/multiqc_plots/svg/qualimap_gc_content.svg", + "multiqc/bwameth/multiqc_plots/svg/qualimap_genome_fraction.svg", + "multiqc/bwameth/multiqc_plots/svg/qualimap_insert_size.svg", + "multiqc/bwameth/multiqc_plots/svg/samtools-flagstat-dp_Percentage_of_total.svg", + "multiqc/bwameth/multiqc_plots/svg/samtools-flagstat-dp_Read_counts.svg", + "multiqc/bwameth/multiqc_plots/svg/samtools-stats-dp.svg", + "multiqc/bwameth/multiqc_plots/svg/samtools_alignment_plot-cnt.svg", + "multiqc/bwameth/multiqc_plots/svg/samtools_alignment_plot-pct.svg", + "multiqc/bwameth/multiqc_report.html", + "pipeline_info", + "pipeline_info/nf_core_pipeline_software_mqc_versions.yml", + "trimgalore", + "trimgalore/fastqc", + "trimgalore/fastqc/Ecoli_10K_methylated_1_val_1_fastqc.html", + "trimgalore/fastqc/Ecoli_10K_methylated_2_val_2_fastqc.html", + "trimgalore/fastqc/SRR389222_sub1_trimmed_fastqc.html", + "trimgalore/fastqc/SRR389222_sub2_trimmed_fastqc.html", + "trimgalore/fastqc/SRR389222_sub3_trimmed_fastqc.html", + "trimgalore/fastqc/zips", + "trimgalore/fastqc/zips/Ecoli_10K_methylated_1_val_1_fastqc.zip", + "trimgalore/fastqc/zips/Ecoli_10K_methylated_2_val_2_fastqc.zip", + "trimgalore/fastqc/zips/SRR389222_sub1_trimmed_fastqc.zip", + "trimgalore/fastqc/zips/SRR389222_sub2_trimmed_fastqc.zip", + "trimgalore/fastqc/zips/SRR389222_sub3_trimmed_fastqc.zip", + "trimgalore/logs", + "trimgalore/logs/Ecoli_10K_methylated_1.fastq.gz_trimming_report.txt", + "trimgalore/logs/Ecoli_10K_methylated_2.fastq.gz_trimming_report.txt", + "trimgalore/logs/SRR389222_sub1.fastq.gz_trimming_report.txt", + "trimgalore/logs/SRR389222_sub2.fastq.gz_trimming_report.txt", + "trimgalore/logs/SRR389222_sub3.fastq.gz_trimming_report.txt" + ], + [ + "Ecoli_10K_methylated.flagstat:md5,f520e87cc4050799a0dbed72909572ec", + "Ecoli_10K_methylated.stats:md5,6067718d5b05d81bb94929cd80b8839d", + "SRR389222_sub1.flagstat:md5,ad8159a7683b9c3057bf54ccb77a2ddc", + "SRR389222_sub1.stats:md5,7df53a3d66857dd289911a13fed8497d", + "SRR389222_sub2.flagstat:md5,e768b4f54014d62b096152c44f920bc1", + "SRR389222_sub2.stats:md5,b0426a921c56ed8a86ade65adf7ed6dd", + "SRR389222_sub3.flagstat:md5,39516f7d2c77e6fef9543edf8cddbcd6", + "SRR389222_sub3.stats:md5,7c6053d7f87cbaa8782fc41a53d932fe", + "agogo.css:md5,bd757b1a7ce6fdc0288ba148680f4583", + "ajax-loader.gif:md5,ae6667053ad118020b8e68ccf307b519", + "basic.css:md5,25b2823342c0604924a2870eeb4e7e94", + "bgfooter.png:md5,ed01bb040346e4623cc87de331ddb4e1", + "bgtop.png:md5,626a50532dc6e5adbdd968746ef318e8", + "comment-bright.png:md5,0c850bb4920b581bf5e5dba5fa493a64", + "comment-close.png:md5,2635dda49c823e8122d4d11ed385f33d", + "comment.png:md5,882e40f3d6a16c6ed35659b105251525", + "doctools.js:md5,5ff571aa60e63f69c1890283e240ff8d", + "down-pressed.png:md5,ebe8979581eda700fb234a73c661a4b9", + "down.png:md5,f6f3c819cc7ca27d7fd3347e5e7ffe0f", + "file.png:md5,6587e59c55e626744eb6fc11129d99a7", + "jquery.js:md5,10092eee563dec2dca82b77d2cf5a1ae", + "minus.png:md5,8d572395aa95c89584a09813ada4dfa1", + "plus.png:md5,0125e6faa04e2cf0141a2d599d3bb220", + "pygments.css:md5,d625a0adb949f181bd0d3f1432b0fa7f", + "qualimap_logo_small.png:md5,7526f145a97be4682fd59e27dda4f71b", + "report.css:md5,7a5f09eaf7c176f966f4e8854168b812", + "searchtools.js:md5,d550841adeedc8ed47c40ee607620937", + "underscore.js:md5,db5ba047a66617d4cd3e8c5099cc51db", + "up-pressed.png:md5,8ea9bd109342f87fee97943b479c6f7e", + "up.png:md5,ecc373278454cc8ecc12d6ca69e55b36", + "websupport.js:md5,9e61e1e8a7433c56bd7e5a615affcf85", + "genome_results.txt:md5,1ee2e29ac41c2cd7e6a0f00741e8d16e", + "genome_coverage_0to50_histogram.png:md5,2c5cd211837b2d4fe01ddad354d5211e", + "genome_coverage_across_reference.png:md5,eb361fdf3a6b96f91c6fea5083cff81f", + "genome_coverage_histogram.png:md5,4edfa52e16b71c63b80a802cb16a2f04", + "genome_coverage_quotes.png:md5,0297a2d5a33e5c0cd6d40961d33139ac", + "genome_gc_content_per_window.png:md5,aa7ea66cb94cd582db83e744203a137f", + "genome_homopolymer_indels.png:md5,e9406a58d8df6b1b930636b07f2a6767", + "genome_insert_size_across_reference.png:md5,3900fed8a1775605adcdcb9acd499b80", + "genome_insert_size_histogram.png:md5,33957d27ec5b7180cc8447532cbfef57", + "genome_mapping_quality_across_reference.png:md5,4d663703c8d8b967ce2372e457e6bba9", + "genome_mapping_quality_histogram.png:md5,4c06a9a29fba3dc84a2b37e736ada597", + "genome_reads_clipping_profile.png:md5,31c0938199fce4207269e12d158f63dc", + "genome_reads_content_per_read_position.png:md5,e655bfeb69a21f69b5dbdb181dd1586a", + "genome_uniq_read_starts_histogram.png:md5,f45743d65d23c9a53902480b018e828c", + "coverage_across_reference.txt:md5,c4c7f930ad577d15de1358226acab252", + "coverage_histogram.txt:md5,cddf42d662856425649c196c731f048b", + "duplication_rate_histogram.txt:md5,764cb7ceebd9f6c40d68fac844d8645e", + "genome_fraction_coverage.txt:md5,53bbcec7b73cabbacfee1519a047bcbf", + "homopolymer_indels.txt:md5,99b50d128297f11553837958d07e0ec5", + "insert_size_across_reference.txt:md5,d55ce0a480c4651cda5f204bb5596cc2", + "insert_size_histogram.txt:md5,6cfc87a2fe207fe1b5e00a1dca58baa8", + "mapped_reads_clipping_profile.txt:md5,f6ba286879f9d7fa3cd44653e32d7e3f", + "mapped_reads_gc-content_distribution.txt:md5,5ccc04e2cfe72baf465834aaeec5f92c", + "mapped_reads_nucleotide_content.txt:md5,87d4e3b86ac646498e986e8b5e05226e", + "mapping_quality_across_reference.txt:md5,c483c7c5c75cc05ab696743934ae006b", + "mapping_quality_histogram.txt:md5,97b8ae36235e0dabc17f2b0697e9d078", + "agogo.css:md5,bd757b1a7ce6fdc0288ba148680f4583", + "ajax-loader.gif:md5,ae6667053ad118020b8e68ccf307b519", + "basic.css:md5,25b2823342c0604924a2870eeb4e7e94", + "bgfooter.png:md5,ed01bb040346e4623cc87de331ddb4e1", + "bgtop.png:md5,626a50532dc6e5adbdd968746ef318e8", + "comment-bright.png:md5,0c850bb4920b581bf5e5dba5fa493a64", + "comment-close.png:md5,2635dda49c823e8122d4d11ed385f33d", + "comment.png:md5,882e40f3d6a16c6ed35659b105251525", + "doctools.js:md5,5ff571aa60e63f69c1890283e240ff8d", + "down-pressed.png:md5,ebe8979581eda700fb234a73c661a4b9", + "down.png:md5,f6f3c819cc7ca27d7fd3347e5e7ffe0f", + "file.png:md5,6587e59c55e626744eb6fc11129d99a7", + "jquery.js:md5,10092eee563dec2dca82b77d2cf5a1ae", + "minus.png:md5,8d572395aa95c89584a09813ada4dfa1", + "plus.png:md5,0125e6faa04e2cf0141a2d599d3bb220", + "pygments.css:md5,d625a0adb949f181bd0d3f1432b0fa7f", + "qualimap_logo_small.png:md5,7526f145a97be4682fd59e27dda4f71b", + "report.css:md5,7a5f09eaf7c176f966f4e8854168b812", + "searchtools.js:md5,d550841adeedc8ed47c40ee607620937", + "underscore.js:md5,db5ba047a66617d4cd3e8c5099cc51db", + "up-pressed.png:md5,8ea9bd109342f87fee97943b479c6f7e", + "up.png:md5,ecc373278454cc8ecc12d6ca69e55b36", + "websupport.js:md5,9e61e1e8a7433c56bd7e5a615affcf85", + "genome_results.txt:md5,2c755b15ae5ad4a1d29395320a6264ca", + "agogo.css:md5,bd757b1a7ce6fdc0288ba148680f4583", + "ajax-loader.gif:md5,ae6667053ad118020b8e68ccf307b519", + "basic.css:md5,25b2823342c0604924a2870eeb4e7e94", + "bgfooter.png:md5,ed01bb040346e4623cc87de331ddb4e1", + "bgtop.png:md5,626a50532dc6e5adbdd968746ef318e8", + "comment-bright.png:md5,0c850bb4920b581bf5e5dba5fa493a64", + "comment-close.png:md5,2635dda49c823e8122d4d11ed385f33d", + "comment.png:md5,882e40f3d6a16c6ed35659b105251525", + "doctools.js:md5,5ff571aa60e63f69c1890283e240ff8d", + "down-pressed.png:md5,ebe8979581eda700fb234a73c661a4b9", + "down.png:md5,f6f3c819cc7ca27d7fd3347e5e7ffe0f", + "file.png:md5,6587e59c55e626744eb6fc11129d99a7", + "jquery.js:md5,10092eee563dec2dca82b77d2cf5a1ae", + "minus.png:md5,8d572395aa95c89584a09813ada4dfa1", + "plus.png:md5,0125e6faa04e2cf0141a2d599d3bb220", + "pygments.css:md5,d625a0adb949f181bd0d3f1432b0fa7f", + "qualimap_logo_small.png:md5,7526f145a97be4682fd59e27dda4f71b", + "report.css:md5,7a5f09eaf7c176f966f4e8854168b812", + "searchtools.js:md5,d550841adeedc8ed47c40ee607620937", + "underscore.js:md5,db5ba047a66617d4cd3e8c5099cc51db", + "up-pressed.png:md5,8ea9bd109342f87fee97943b479c6f7e", + "up.png:md5,ecc373278454cc8ecc12d6ca69e55b36", + "websupport.js:md5,9e61e1e8a7433c56bd7e5a615affcf85", + "genome_results.txt:md5,2294badeccd2e8870fd6c60d115687b0", + "agogo.css:md5,bd757b1a7ce6fdc0288ba148680f4583", + "ajax-loader.gif:md5,ae6667053ad118020b8e68ccf307b519", + "basic.css:md5,25b2823342c0604924a2870eeb4e7e94", + "bgfooter.png:md5,ed01bb040346e4623cc87de331ddb4e1", + "bgtop.png:md5,626a50532dc6e5adbdd968746ef318e8", + "comment-bright.png:md5,0c850bb4920b581bf5e5dba5fa493a64", + "comment-close.png:md5,2635dda49c823e8122d4d11ed385f33d", + "comment.png:md5,882e40f3d6a16c6ed35659b105251525", + "doctools.js:md5,5ff571aa60e63f69c1890283e240ff8d", + "down-pressed.png:md5,ebe8979581eda700fb234a73c661a4b9", + "down.png:md5,f6f3c819cc7ca27d7fd3347e5e7ffe0f", + "file.png:md5,6587e59c55e626744eb6fc11129d99a7", + "jquery.js:md5,10092eee563dec2dca82b77d2cf5a1ae", + "minus.png:md5,8d572395aa95c89584a09813ada4dfa1", + "plus.png:md5,0125e6faa04e2cf0141a2d599d3bb220", + "pygments.css:md5,d625a0adb949f181bd0d3f1432b0fa7f", + "qualimap_logo_small.png:md5,7526f145a97be4682fd59e27dda4f71b", + "report.css:md5,7a5f09eaf7c176f966f4e8854168b812", + "searchtools.js:md5,d550841adeedc8ed47c40ee607620937", + "underscore.js:md5,db5ba047a66617d4cd3e8c5099cc51db", + "up-pressed.png:md5,8ea9bd109342f87fee97943b479c6f7e", + "up.png:md5,ecc373278454cc8ecc12d6ca69e55b36", + "websupport.js:md5,9e61e1e8a7433c56bd7e5a615affcf85", + "genome_results.txt:md5,4da1144933d094185d02f7482c120804", + "versions.yml:md5,bb8bc6f11e6ca441cdb653f077b870c2", + "Ecoli_10K_methylated.markdup.sorted_CpG.bedGraph:md5,d3bce01078b6e7749f3906bf9faad5b5", + "SRR389222_sub1.markdup.sorted_CpG.bedGraph:md5,0944ad0e3b05791bf6dfd06be7ca4af1", + "SRR389222_sub2.markdup.sorted_CpG.bedGraph:md5,0a3c19af55d1625e91903ae25d1d2098", + "SRR389222_sub3.markdup.sorted_CpG.bedGraph:md5,a91e6b039e9581a9d2ef6f6057918b39", + "Ecoli_10K_methylated.mbias.txt:md5,3a448a2a6364b2de3129365a271c0534", + "SRR389222_sub1.mbias.txt:md5,fce04d733e066d0b933cedc602e2af81", + "SRR389222_sub2.mbias.txt:md5,fce04d733e066d0b933cedc602e2af81", + "SRR389222_sub3.mbias.txt:md5,fce04d733e066d0b933cedc602e2af81", + "cutadapt_filtered_reads_plot.txt:md5,ff1b6d4011a5be5410b9404f7ff56c56", + "cutadapt_trimmed_sequences_plot_3_Counts.txt:md5,57d07c19b0c30dc88df96c30bedcdc33", + "cutadapt_trimmed_sequences_plot_3_Obs_Exp.txt:md5,1e5ca4193f8290c5c812334d03615b41", + "fastqc-status-check-heatmap.txt:md5,218c2a7efbb9c02f652fa78fc739c866", + "fastqc_adapter_content_plot.txt:md5,48d8bd5ece984a74972ce0a0e814dd07", + "fastqc_overrepresented_sequences_plot.txt:md5,9c4b7064950b5b2c486d556984f21e9e", + "fastqc_per_base_n_content_plot.txt:md5,019d5cd4f61a31f4e7016140ca8ee36e", + "fastqc_per_base_sequence_quality_plot.txt:md5,e4c7d522114eab9bc9101e7ee9d83c8f", + "fastqc_per_sequence_gc_content_plot_Counts.txt:md5,302659bee8a02b6a38b24fe25f8eaa87", + "fastqc_per_sequence_gc_content_plot_Percentages.txt:md5,c75b396eb5dda8682fb1652a9b2e92a3", + "fastqc_per_sequence_quality_scores_plot.txt:md5,548c593f4a8be2d8627fc3bd1c280611", + "fastqc_sequence_counts_plot.txt:md5,58eeeb06bc9dcc1764c7daf0847d6bbf", + "fastqc_sequence_duplication_levels_plot.txt:md5,9e5712db9aa39b5754ea1e0c49cfff15", + "fastqc_top_overrepresented_sequences_table.txt:md5,6dfc5334a09c05392ea17369aa742d01", + "multiqc_citations.txt:md5,581df5b266851d0244e7cfb213351ffb", + "multiqc_cutadapt.txt:md5,883891ac4901385102be053212068732", + "multiqc_picard_dups.txt:md5,82eaab96189f22fd93db38bb516ba6a5", + "multiqc_samtools_flagstat.txt:md5,a8f5b45b8b3412a9adc3dd0d35208c73", + "multiqc_samtools_stats.txt:md5,bfcab6a9bd26872ecd2ec469d222c70b", + "picard_deduplication.txt:md5,7dacf4d4e0d7d2d9a7205beecfa7125c", + "picard_histogram.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "picard_histogram_1.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "picard_histogram_2.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "qualimap_coverage_histogram.txt:md5,c7c932e4d84b735bd434c1aaa713b6e6", + "qualimap_gc_content.txt:md5,893d1acba7a6675ff9e2eff2985fa4e3", + "qualimap_genome_fraction.txt:md5,ef5b6010b52a03f6e58356db7128d0dc", + "qualimap_insert_size.txt:md5,f86cc9fe7a12af8f62f26312660e9225", + "samtools-flagstat-dp_Percentage_of_total.txt:md5,ff5d65a480449ba98587c43f3a2bd33e", + "samtools-flagstat-dp_Read_counts.txt:md5,6a5332e2ad59f7da00b5a9a1a25fd2f8", + "samtools-stats-dp.txt:md5,52021b9a9455b39b39dd357bc2e8d346", + "samtools_alignment_plot.txt:md5,39f42e039f96cc2b17248b44ccaf425b" + ], + [ + [ + "Ecoli_10K_methylated.deduplicated.sorted.bam", + "a2921f761f431ede522016f83f75c12d" + ], + [ + "Ecoli_10K_methylated.markdup.sorted.bam", + "a2921f761f431ede522016f83f75c12d" + ], + [ + "SRR389222_sub1.deduplicated.sorted.bam", + "e50275d6d44138f19d65ee479444d5ce" + ], + [ + "SRR389222_sub1.markdup.sorted.bam", + "e50275d6d44138f19d65ee479444d5ce" + ], + [ + "SRR389222_sub2.deduplicated.sorted.bam", + "d365b8b2662b2abffb51dd67a1f463b4" + ], + [ + "SRR389222_sub2.markdup.sorted.bam", + "d365b8b2662b2abffb51dd67a1f463b4" + ], + [ + "SRR389222_sub3.deduplicated.sorted.bam", + "a9fd2900b9d759f8f4f62cbbb2744fde" + ], + [ + "SRR389222_sub3.markdup.sorted.bam", + "a9fd2900b9d759f8f4f62cbbb2744fde" + ] + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.2" + }, + "timestamp": "2024-12-02T12:46:20.537987965" + } +} \ No newline at end of file From 4ff6c9e5c6d706a3c3ee6bb2bb5b33c2580d74b2 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 2 Dec 2024 13:05:18 +0000 Subject: [PATCH 31/89] add setup python step --- .github/workflows/ci.yml | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e761f7c..5814bd97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,14 +59,16 @@ jobs: with: version: "${{ matrix.NXF_VER }}" - - name: Install pdiff to see diff between nf-test snapshots - run: | - python -m pip install --upgrade pip - pip install pdiff + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: "3.11" - - uses: nf-core/setup-nf-test@v1 + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 with: - version: ${{ env.NFT_VER }} + version: "${{ env.NFT_VER }}" + install-pdiff: true - name: Set up Apptainer if: matrix.profile == 'singularity' @@ -78,6 +80,17 @@ jobs: mkdir -p $NXF_SINGULARITY_CACHEDIR mkdir -p $NXF_SINGULARITY_LIBRARYDIR + # Set up secrets + - name: Set up nextflow secrets + if: env.SENTIEON_ENCRYPTION_KEY != '' && env.SENTIEON_LICENSE_MESSAGE != '' + shell: bash + run: | + python -m pip install cryptography + nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") + + # TODO Skip failing conda tests and document their failures + # https://github.com/nf-core/modules/issues/7017 + - name: Clean up Disk space uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 @@ -95,7 +108,7 @@ jobs: if: failure() run: | sudo apt install bat > /dev/null - batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log + batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/**/nextflow.log - name: Publish Test Report uses: mikepenz/action-junit-report@v3 From 416238733710ce15d060d8dc0d43a6ab5d8e0abd Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 03:06:46 +0000 Subject: [PATCH 32/89] restrict num shards for cpu/gpu to 2 --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5814bd97..30c145ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: - "singularity" isMaster: - ${{ github.base_ref == 'master' }} - # Exclude conda and singularity on dev + # Exclude singularity on dev exclude: - isMaster: false profile: "singularity" @@ -127,9 +127,10 @@ jobs: NXF_VER: - "24.04.2" - "latest-everything" - shard: [1, 2, 3, 4, 5] + shard: [1, 2] filter: [pipeline, workflow] profile: + - "conda" - "docker" - "singularity" isMaster: @@ -142,7 +143,7 @@ jobs: profile: "singularity" env: NXF_ANSI_LOG: false - TOTAL_SHARDS: 5 + TOTAL_SHARDS: 2 steps: - name: Check out pipeline code From b2fd76f4a6880cb88431c908ba4c13792c0b0165 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 09:41:11 +0000 Subject: [PATCH 33/89] add nf-test-shard and nf-test actions --- .github/actions/nf-test-shard/action.yml | 46 +++++ .github/actions/nf-test/action.yml | 101 +++++++++++ .github/workflows/ci.yml | 216 ----------------------- .github/workflows/nf-test-gpu.yml | 93 ++++++++++ .github/workflows/nf-test.yml | 96 ++++++++++ 5 files changed, 336 insertions(+), 216 deletions(-) create mode 100644 .github/actions/nf-test-shard/action.yml create mode 100644 .github/actions/nf-test/action.yml delete mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/nf-test-gpu.yml create mode 100644 .github/workflows/nf-test.yml diff --git a/.github/actions/nf-test-shard/action.yml b/.github/actions/nf-test-shard/action.yml new file mode 100644 index 00000000..bf8ef4ff --- /dev/null +++ b/.github/actions/nf-test-shard/action.yml @@ -0,0 +1,46 @@ +name: "Get number of shards" +description: "Get the number of nf-test shards for the current CI job" +outputs: + shards: + description: "Array of shard numbers" + value: ${{ steps.shards.outputs.shards }} + total_shards: + description: "Total number of shards" + value: ${{ steps.shards.outputs.total_shards }} + tags: + description: "Tags to test (`[,...]`)" + value: ${{ steps.shards.outputs.tags }} + +runs: + using: "composite" + steps: + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFT_VER }} + + - name: Get number of shards + id: shards + shell: bash + run: | + nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter process --follow-dependencies) + echo "nftest_output: $nftest_output" + if echo "$nftest_output" | grep -q 'Found 0 related test(s)'; then + shards="[]" + total_shards=0 + tags="[]" + else + number_of_shards=$(echo "$nftest_output" | grep -o 'Found [0-9]* related test' | tail -1 | awk '{print $2}') + shards=$(seq 1 "$number_of_shards" | jq -R . | jq -s .) + tags=$(echo "$nftest_output" | grep -o 'tag [^ ]*' | tail -1 | awk '{print $2}') + total_shards="$number_of_shards" + fi + echo "shards=${shards}" >> $GITHUB_OUTPUT + echo "total_shards=${total_shards}" >> $GITHUB_OUTPUT + echo "tags=${tags}" >> $GITHUB_OUTPUT + + # Debugging output + echo "nftest_output: $nftest_output" + echo "shards: $shards" + echo "total_shards: $total_shards" + echo "tags: $tags" diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml new file mode 100644 index 00000000..665cb393 --- /dev/null +++ b/.github/actions/nf-test/action.yml @@ -0,0 +1,101 @@ +name: "nf-test Action" +description: "Runs nf-test with common setup steps" +inputs: + profile: + description: "Profile to use" + required: true + shards: + description: "Shard number for this CI job" + required: true + total_shards: + description: "Total number of test shards(NOT the total number of matrix jobs)" + required: true + filters: + description: "Filter test cases by specified types (e.g., module, pipeline, workflow or function)" + required: true + +runs: + using: "composite" + steps: + - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v2 + with: + version: "${{ env.NXF_VERSION }}" + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: "3.11" + + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + with: + version: "${{ env.NFT_VER }}" + install-pdiff: true + + - name: Setup apptainer + if: contains(inputs.profile, 'singularity') + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: contains(inputs.profile, 'singularity') + shell: bash + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Conda setup + if: ${{inputs.profile == 'conda'}} + uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3 + with: + auto-update-conda: true + conda-solver: libmamba + conda-remove-defaults: true + + # Set up secrets + - name: Set up nextflow secrets + if: env.SENTIEON_ENCRYPTION_KEY != '' && env.SENTIEON_LICENSE_MESSAGE != '' + shell: bash + run: | + python -m pip install cryptography + nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") + + # TODO Skip failing conda tests and document their failures + # https://github.com/nf-core/modules/issues/7017 + - name: Clean up Disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + + - name: Run nf-test + shell: bash + env: + SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + run: | + NFT_WORKDIR=~ \ + nf-test test \ + --ci \ + --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \ + --changed-since HEAD^ \ + --profile=${{ inputs.profile }} \ + --filter ${{ inputs.filter }} \ + --tag ${{ inputs.tags }} + --tap=test.tap \ + --verbose \ + + # TODO If no test.tap, then make one to spoof? + - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 + if: ${{ inputs.path != '' }} + with: + path: >- + test.tap + + - name: Clean up + if: always() + shell: bash + run: | + sudo rm -rf /home/ubuntu/tests/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 30c145ce..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,216 +0,0 @@ -name: nf-core CI -# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors -on: - push: - branches: - - dev - pull_request: - release: - types: [published] - workflow_dispatch: - -env: - NXF_ANSI_LOG: false - NFT_VER: "0.9.2" - NFT_WORKDIR: "~" - NFT_DIFF: "pdiff" - NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" - NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity - NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity - -concurrency: - group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" - cancel-in-progress: true - -jobs: - nf-test-gpu: - name: "Run GPU Tests (${{ matrix.NXF_VER }} | ${{ matrix.profile }})" - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" - runs-on: "gpu" - strategy: - fail-fast: false - matrix: - NXF_VER: - - "24.04.2" - shard: [1, 2] - filter: [pipeline, workflow] - profile: - - "docker" - - "singularity" - isMaster: - - ${{ github.base_ref == 'master' }} - # Exclude singularity on dev - exclude: - - isMaster: false - profile: "singularity" - env: - NXF_ANSI_LOG: false - TOTAL_SHARDS: 2 - - steps: - - name: Check out pipeline code - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - with: - fetch-depth: 0 - - - name: Set up Nextflow - uses: nf-core/setup-nextflow@v2 - with: - version: "${{ matrix.NXF_VER }}" - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - name: Install nf-test - uses: nf-core/setup-nf-test@v1 - with: - version: "${{ env.NFT_VER }}" - install-pdiff: true - - - name: Set up Apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - # Set up secrets - - name: Set up nextflow secrets - if: env.SENTIEON_ENCRYPTION_KEY != '' && env.SENTIEON_LICENSE_MESSAGE != '' - shell: bash - run: | - python -m pip install cryptography - nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") - - # TODO Skip failing conda tests and document their failures - # https://github.com/nf-core/modules/issues/7017 - - - name: Clean up Disk space - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - - name: Run GPU Tests (Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }}) - run: | - nf-test test \ - --ci \ - --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ - --changed-since HEAD^ \ - --profile "+${{ matrix.profile }}" \ - --filter ${{ matrix.filter }} \ - --tag "gpu" - - - name: Output log on failure - if: failure() - run: | - sudo apt install bat > /dev/null - batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/**/nextflow.log - - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - if: always() # always run even if the previous step fails - with: - report_paths: test.xml - - nf-test: - name: "Run CPU Tests (${{ matrix.NXF_VER }} | ${{ matrix.profile }})" - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - NXF_VER: - - "24.04.2" - - "latest-everything" - shard: [1, 2] - filter: [pipeline, workflow] - profile: - - "conda" - - "docker" - - "singularity" - isMaster: - - ${{ github.base_ref == 'master' }} - # Exclude conda and singularity on dev - exclude: - - isMaster: false - profile: "conda" - - isMaster: false - profile: "singularity" - env: - NXF_ANSI_LOG: false - TOTAL_SHARDS: 2 - - steps: - - name: Check out pipeline code - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - with: - fetch-depth: 0 - - - name: Set up Nextflow - uses: nf-core/setup-nextflow@v2 - with: - version: "${{ matrix.NXF_VER }}" - - - name: Install pdiff to see diff between nf-test snapshots - run: | - python -m pip install --upgrade pip - pip install pdiff - - - uses: nf-core/setup-nf-test@v1 - with: - version: ${{ env.NFT_VER }} - - - name: Set up Apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - - name: Set up Miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 - with: - miniconda-version: "latest" - auto-update-conda: true - conda-solver: libmamba - channels: conda-forge,bioconda - - - name: Set up Conda - if: matrix.profile == 'conda' - run: | - echo $(realpath $CONDA)/condabin >> $GITHUB_PATH - echo $(realpath python) >> $GITHUB_PATH - - - name: Clean up Disk space - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - - name: Run Tests (Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }}) - run: | - nf-test test \ - --ci \ - --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ - --changed-since HEAD^ \ - --profile "+${{ matrix.profile }}" \ - --filter ${{ matrix.filter }} \ - --tag "cpu" - - - name: Output log on failure - if: failure() - run: | - sudo apt install bat > /dev/null - batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log - - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - if: always() # always run even if the previous step fails - with: - report_paths: test.xml diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml new file mode 100644 index 00000000..3ee2819c --- /dev/null +++ b/.github/workflows/nf-test-gpu.yml @@ -0,0 +1,93 @@ +name: Run GPU nf-tests +on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: string + default: "gpu" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver + NFT_VER: "0.9.2" + NXF_ANSI_LOG: false + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver + NXF_VER: "24.10.2" + +jobs: + nf-test-gpu: + runs-on: "gpu" + name: "GPU Test | ${{ matrix.profile }} | ${{ matrix.shard }}" + # Only run on push if this is the nf-core dev branch (merged PRs) + if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" + strategy: + fail-fast: false + matrix: + NXF_VER: + - "24.04.2" + shard: [1, 2] + filter: [pipeline] + profile: [docker, singularity] + env: + NXF_ANSI_LOG: false + TOTAL_SHARDS: 2 + + steps: + - name: Check out pipeline code + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 + with: + fetch-depth: 0 + + - name: Get number of shards + id: get-shards + uses: ./.github/actions/nf-test-shard + env: + NFT_VER: ${{ env.NFT_VER }} + + - name: Run nf-test Action + uses: ./.github/actions/nf-test + env: + SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} + SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} + SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + with: + profile: ${{ matrix.profile }},gpu + shards: ${{ steps.get-shards.outputs.shards }} + total_shards: ${{ steps.get-shards.outputs.total_shards }} + filters: pipeline + + confirm-pass: + runs-on: ubuntu-latest + needs: [nf-test-gpu] + if: always() + steps: + - name: All tests ok + if: ${{ success() || !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml new file mode 100644 index 00000000..e6db3072 --- /dev/null +++ b/.github/workflows/nf-test.yml @@ -0,0 +1,96 @@ +name: Run nf-tests +on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver + NFT_VER: "0.9.2" + NXF_ANSI_LOG: false + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver + NXF_VER: "24.10.2" + +jobs: + nf-test: + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + name: "Test | ${{ matrix.profile }} | ${{ matrix.shard }}" + # Only run on push if this is the nf-core dev branch (merged PRs) + if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" + strategy: + fail-fast: false + matrix: + NXF_VER: + - "24.04.2" + shard: [1, 2] + filter: [pipeline] + profile: [conda, docker_self_hosted, singularity] + env: + NXF_ANSI_LOG: false + TOTAL_SHARDS: 2 + + steps: + - name: Check out pipeline code + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 + with: + fetch-depth: 0 + + - name: Get number of shards + id: get-shards + uses: ./.github/actions/nf-test-shard + env: + NFT_VER: ${{ env.NFT_VER }} + + - name: Run nf-test Action + uses: ./.github/actions/nf-test + env: + SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} + SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} + SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + with: + profile: ${{ matrix.profile }} + shards: ${{ steps.get-shards.outputs.shards }} + total_shards: ${{ steps.get-shards.outputs.total_shards }} + filters: pipeline + + confirm-pass: + runs-on: ubuntu-latest + needs: [nf-test] + if: always() + steps: + - name: All tests ok + if: ${{ success() || !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" From 89cfb48e7bebe8a8ec4f66258ad6467bc5d23fbf Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 09:45:18 +0000 Subject: [PATCH 34/89] add ci triggers --- .github/workflows/nf-test-gpu.yml | 9 +++------ .github/workflows/nf-test.yml | 9 +++------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 3ee2819c..86cb14ad 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -2,13 +2,10 @@ name: Run GPU nf-tests on: push: branches: - # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging - - "renovate/**" # branches Renovate creates + - dev pull_request: - branches: [master] - merge_group: - types: [checks_requested] - branches: [master] + release: + types: [published] workflow_dispatch: inputs: runners: diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index e6db3072..ff7d4e99 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -2,13 +2,10 @@ name: Run nf-tests on: push: branches: - # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging - - "renovate/**" # branches Renovate creates + - dev pull_request: - branches: [master] - merge_group: - types: [checks_requested] - branches: [master] + release: + types: [published] workflow_dispatch: inputs: runners: From 36deedcf5800b4f221500baba01984049a0ff1f8 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 09:56:12 +0000 Subject: [PATCH 35/89] lint: ignore ci.yml --- .nf-core.yml | 1 + tests/bwameth.nf.test | 4 ++-- tests/bwameth_use_gpu.nf.test | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.nf-core.yml b/.nf-core.yml index 52a9a319..1c5c8f9d 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -3,6 +3,7 @@ lint: files_exist: - lib/nfcore_external_java_deps.jar - conf/modules.config + - .github/workflows/ci.yml modules_config: false nf_core_version: 3.0.2 repository_type: pipeline diff --git a/tests/bwameth.nf.test b/tests/bwameth.nf.test index 373f2a99..49bd959b 100644 --- a/tests/bwameth.nf.test +++ b/tests/bwameth.nf.test @@ -41,9 +41,9 @@ nextflow_pipeline { test("Params: bwameth with bwameth-index") { when { params { - aligner = "bwameth" + aligner = "bwameth" bwameth_index = "https://github.com/nf-core/test-datasets/raw/methylseq/reference/Bwameth_Index.tar.gz" - outdir = "$outputDir" + outdir = "$outputDir" } } diff --git a/tests/bwameth_use_gpu.nf.test b/tests/bwameth_use_gpu.nf.test index 34eced52..8abbdc4d 100644 --- a/tests/bwameth_use_gpu.nf.test +++ b/tests/bwameth_use_gpu.nf.test @@ -9,7 +9,7 @@ nextflow_pipeline { when { params { input = "${projectDir}/assets/samplesheet_gpu.csv" - aligner = "bwameth" + aligner = "bwameth" // uses parabricks/fq2bammeth use_gpu = true outdir = "$outputDir" } From c7a32b41b3b14980a423ad0263b9f08e34bdd9df Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 10:00:31 +0000 Subject: [PATCH 36/89] update nf-test-shard action --- .github/actions/nf-test-shard/action.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/actions/nf-test-shard/action.yml b/.github/actions/nf-test-shard/action.yml index bf8ef4ff..e41c67e1 100644 --- a/.github/actions/nf-test-shard/action.yml +++ b/.github/actions/nf-test-shard/action.yml @@ -23,24 +23,23 @@ runs: id: shards shell: bash run: | - nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter process --follow-dependencies) + nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter pipeline) + echo "nftest_output: $nftest_output" + if echo "$nftest_output" | grep -q 'Found 0 related test(s)'; then shards="[]" total_shards=0 - tags="[]" else number_of_shards=$(echo "$nftest_output" | grep -o 'Found [0-9]* related test' | tail -1 | awk '{print $2}') shards=$(seq 1 "$number_of_shards" | jq -R . | jq -s .) - tags=$(echo "$nftest_output" | grep -o 'tag [^ ]*' | tail -1 | awk '{print $2}') total_shards="$number_of_shards" fi + echo "shards=${shards}" >> $GITHUB_OUTPUT echo "total_shards=${total_shards}" >> $GITHUB_OUTPUT - echo "tags=${tags}" >> $GITHUB_OUTPUT # Debugging output echo "nftest_output: $nftest_output" echo "shards: $shards" echo "total_shards: $total_shards" - echo "tags: $tags" From a9f4ee08bc72ca15f19b9b137ca22c62d6deefb7 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 10:13:44 +0000 Subject: [PATCH 37/89] update nf-test-shard action --- .github/actions/nf-test-shard/action.yml | 7 ------- .github/workflows/nf-test-gpu.yml | 2 +- .github/workflows/nf-test.yml | 2 +- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/actions/nf-test-shard/action.yml b/.github/actions/nf-test-shard/action.yml index e41c67e1..7d2f8f25 100644 --- a/.github/actions/nf-test-shard/action.yml +++ b/.github/actions/nf-test-shard/action.yml @@ -7,9 +7,6 @@ outputs: total_shards: description: "Total number of shards" value: ${{ steps.shards.outputs.total_shards }} - tags: - description: "Tags to test (`[,...]`)" - value: ${{ steps.shards.outputs.tags }} runs: using: "composite" @@ -24,9 +21,7 @@ runs: shell: bash run: | nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter pipeline) - echo "nftest_output: $nftest_output" - if echo "$nftest_output" | grep -q 'Found 0 related test(s)'; then shards="[]" total_shards=0 @@ -35,10 +30,8 @@ runs: shards=$(seq 1 "$number_of_shards" | jq -R . | jq -s .) total_shards="$number_of_shards" fi - echo "shards=${shards}" >> $GITHUB_OUTPUT echo "total_shards=${total_shards}" >> $GITHUB_OUTPUT - # Debugging output echo "nftest_output: $nftest_output" echo "shards: $shards" diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 86cb14ad..43e8c1e6 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -69,7 +69,7 @@ jobs: profile: ${{ matrix.profile }},gpu shards: ${{ steps.get-shards.outputs.shards }} total_shards: ${{ steps.get-shards.outputs.total_shards }} - filters: pipeline + filters: ${{ matrix.filter }} confirm-pass: runs-on: ubuntu-latest diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index ff7d4e99..fa761320 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -72,7 +72,7 @@ jobs: profile: ${{ matrix.profile }} shards: ${{ steps.get-shards.outputs.shards }} total_shards: ${{ steps.get-shards.outputs.total_shards }} - filters: pipeline + filters: ${{ matrix.filter }} confirm-pass: runs-on: ubuntu-latest From 3c81b89db8c90a1bf660a88fce9faef383d99d63 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 10:35:26 +0000 Subject: [PATCH 38/89] add tags input for nf-test action --- .github/actions/nf-test/action.yml | 5 ++++- .github/workflows/nf-test-gpu.yml | 1 + .github/workflows/nf-test.yml | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index 665cb393..eb03567d 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -13,6 +13,9 @@ inputs: filters: description: "Filter test cases by specified types (e.g., module, pipeline, workflow or function)" required: true + tags: + description: "Test tags to be run" + required: true runs: using: "composite" @@ -83,9 +86,9 @@ runs: --changed-since HEAD^ \ --profile=${{ inputs.profile }} \ --filter ${{ inputs.filter }} \ - --tag ${{ inputs.tags }} --tap=test.tap \ --verbose \ + ${{ inputs.tags }} # TODO If no test.tap, then make one to spoof? - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 43e8c1e6..2fef06fc 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -70,6 +70,7 @@ jobs: shards: ${{ steps.get-shards.outputs.shards }} total_shards: ${{ steps.get-shards.outputs.total_shards }} filters: ${{ matrix.filter }} + tags: "--tag gpu" confirm-pass: runs-on: ubuntu-latest diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index fa761320..256331ba 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -73,6 +73,7 @@ jobs: shards: ${{ steps.get-shards.outputs.shards }} total_shards: ${{ steps.get-shards.outputs.total_shards }} filters: ${{ matrix.filter }} + tags: confirm-pass: runs-on: ubuntu-latest From b2785485d4f7bf722ef92c05a178bdb7349b26a4 Mon Sep 17 00:00:00 2001 From: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:06:05 +0530 Subject: [PATCH 39/89] Update .github/actions/nf-test-shard/action.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- .github/actions/nf-test-shard/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/nf-test-shard/action.yml b/.github/actions/nf-test-shard/action.yml index 7d2f8f25..e12b0af4 100644 --- a/.github/actions/nf-test-shard/action.yml +++ b/.github/actions/nf-test-shard/action.yml @@ -27,7 +27,7 @@ runs: total_shards=0 else number_of_shards=$(echo "$nftest_output" | grep -o 'Found [0-9]* related test' | tail -1 | awk '{print $2}') - shards=$(seq 1 "$number_of_shards" | jq -R . | jq -s .) + shards=$(seq 1 "$number_of_shards" | jq -R . | jq -c -s .) total_shards="$number_of_shards" fi echo "shards=${shards}" >> $GITHUB_OUTPUT From be8dbddfa80aecdf8cc1304273efab4a857dee8b Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 11:15:24 +0000 Subject: [PATCH 40/89] install pdiff --- .github/actions/nf-test-shard/action.yml | 1 + .github/workflows/nf-test-gpu.yml | 23 +++++++++++++++-------- .github/workflows/nf-test.yml | 23 +++++++++++++++-------- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/.github/actions/nf-test-shard/action.yml b/.github/actions/nf-test-shard/action.yml index e12b0af4..1b61e377 100644 --- a/.github/actions/nf-test-shard/action.yml +++ b/.github/actions/nf-test-shard/action.yml @@ -15,6 +15,7 @@ runs: uses: nf-core/setup-nf-test@v1 with: version: ${{ env.NFT_VER }} + install-pdiff: true - name: Get number of shards id: shards diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 2fef06fc..d6ee1e15 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -29,6 +29,19 @@ env: NXF_VER: "24.10.2" jobs: + get-shards: + runs-on: ubuntu-latest + name: "Get shards and total number of shards" + # Only run on push if this is the nf-core dev branch (merged PRs) + if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" + strategy: + fail-fast: true + + steps: + - uses: ./.github/actions/nf-test-shard + env: + NFT_VER: ${{ env.NFT_VER }} + nf-test-gpu: runs-on: "gpu" name: "GPU Test | ${{ matrix.profile }} | ${{ matrix.shard }}" @@ -52,12 +65,6 @@ jobs: with: fetch-depth: 0 - - name: Get number of shards - id: get-shards - uses: ./.github/actions/nf-test-shard - env: - NFT_VER: ${{ env.NFT_VER }} - - name: Run nf-test Action uses: ./.github/actions/nf-test env: @@ -67,8 +74,8 @@ jobs: SENTIEON_AUTH_MECH: "GitHub Actions - token" with: profile: ${{ matrix.profile }},gpu - shards: ${{ steps.get-shards.outputs.shards }} - total_shards: ${{ steps.get-shards.outputs.total_shards }} + shards: ${{ job.get-shards.outputs.shards }} + total_shards: ${{ job.get-shards.outputs.total_shards }} filters: ${{ matrix.filter }} tags: "--tag gpu" diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 256331ba..e3f55932 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -32,6 +32,19 @@ env: NXF_VER: "24.10.2" jobs: + get-shards: + runs-on: ubuntu-latest + name: "Get shards and total number of shards" + # Only run on push if this is the nf-core dev branch (merged PRs) + if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" + strategy: + fail-fast: true + + steps: + - uses: ./.github/actions/nf-test-shard + env: + NFT_VER: ${{ env.NFT_VER }} + nf-test: runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} name: "Test | ${{ matrix.profile }} | ${{ matrix.shard }}" @@ -55,12 +68,6 @@ jobs: with: fetch-depth: 0 - - name: Get number of shards - id: get-shards - uses: ./.github/actions/nf-test-shard - env: - NFT_VER: ${{ env.NFT_VER }} - - name: Run nf-test Action uses: ./.github/actions/nf-test env: @@ -70,8 +77,8 @@ jobs: SENTIEON_AUTH_MECH: "GitHub Actions - token" with: profile: ${{ matrix.profile }} - shards: ${{ steps.get-shards.outputs.shards }} - total_shards: ${{ steps.get-shards.outputs.total_shards }} + shards: ${{ job.get-shards.outputs.shards }} + total_shards: ${{ job.get-shards.outputs.total_shards }} filters: ${{ matrix.filter }} tags: From d004b5b07941c6966922343a9cc7ccaf16fb7596 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 11:21:28 +0000 Subject: [PATCH 41/89] nf-test-shard output checkout --- .github/workflows/nf-test-gpu.yml | 3 +++ .github/workflows/nf-test.yml | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index d6ee1e15..745212f6 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -34,6 +34,9 @@ jobs: name: "Get shards and total number of shards" # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" + outputs: + shards: ${{ steps.list.outputs.shards }} + total_shards: ${{ steps.outputs.outputs.total_shards }} strategy: fail-fast: true diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index e3f55932..377dbcf5 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -37,11 +37,15 @@ jobs: name: "Get shards and total number of shards" # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" + outputs: + shards: ${{ steps.list.outputs.shards }} + total_shards: ${{ steps.outputs.outputs.total_shards }} strategy: fail-fast: true steps: - - uses: ./.github/actions/nf-test-shard + - name: list + uses: ./.github/actions/nf-test-shard env: NFT_VER: ${{ env.NFT_VER }} From 7233ca29f30da7746f8007200f49670785168010 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 13:30:32 +0000 Subject: [PATCH 42/89] nf-test-shard output checkout --- .github/workflows/nf-test-gpu.yml | 17 ++++++++++++----- .github/workflows/nf-test.yml | 16 +++++++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 745212f6..97a939a6 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -35,13 +35,20 @@ jobs: # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" outputs: - shards: ${{ steps.list.outputs.shards }} - total_shards: ${{ steps.outputs.outputs.total_shards }} + shards: ${{ steps.list-shards.outputs.shards }} + total_shards: ${{ steps.list-shards.outputs.total_shards }} strategy: fail-fast: true steps: - - uses: ./.github/actions/nf-test-shard + - name: Check out pipeline code + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 + with: + fetch-depth: 0 + + - name: run nf-test-shard action + id: list-shards + uses: ./.github/actions/nf-test-shard env: NFT_VER: ${{ env.NFT_VER }} @@ -77,8 +84,8 @@ jobs: SENTIEON_AUTH_MECH: "GitHub Actions - token" with: profile: ${{ matrix.profile }},gpu - shards: ${{ job.get-shards.outputs.shards }} - total_shards: ${{ job.get-shards.outputs.total_shards }} + shards: ${{ needs.get-shards.outputs.shards }} + total_shards: ${{ needs.get-shards.outputs.total_shards }} filters: ${{ matrix.filter }} tags: "--tag gpu" diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 377dbcf5..6d9a6c4a 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -38,13 +38,19 @@ jobs: # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" outputs: - shards: ${{ steps.list.outputs.shards }} - total_shards: ${{ steps.outputs.outputs.total_shards }} + shards: ${{ steps.list-shards.outputs.shards }} + total_shards: ${{ steps.list-shards.outputs.total_shards }} strategy: fail-fast: true steps: - - name: list + - name: Check out pipeline code + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 + with: + fetch-depth: 0 + + - name: run nf-test-shard action + id: list-shards uses: ./.github/actions/nf-test-shard env: NFT_VER: ${{ env.NFT_VER }} @@ -81,8 +87,8 @@ jobs: SENTIEON_AUTH_MECH: "GitHub Actions - token" with: profile: ${{ matrix.profile }} - shards: ${{ job.get-shards.outputs.shards }} - total_shards: ${{ job.get-shards.outputs.total_shards }} + shards: ${{ needs.get-shards.outputs.shards }} + total_shards: ${{ needs.get-shards.outputs.total_shards }} filters: ${{ matrix.filter }} tags: From 4febb476bcec1b48a48e16869002a178484c5268 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 13:58:53 +0000 Subject: [PATCH 43/89] fix job outputs --- .github/actions/nf-test-shard/action.yml | 2 +- .github/actions/nf-test/action.yml | 15 +------------ .github/workflows/nf-test-gpu.yml | 27 ++++++++++------------- .github/workflows/nf-test.yml | 28 ++++++++++-------------- 4 files changed, 26 insertions(+), 46 deletions(-) diff --git a/.github/actions/nf-test-shard/action.yml b/.github/actions/nf-test-shard/action.yml index 1b61e377..a8054075 100644 --- a/.github/actions/nf-test-shard/action.yml +++ b/.github/actions/nf-test-shard/action.yml @@ -1,7 +1,7 @@ name: "Get number of shards" description: "Get the number of nf-test shards for the current CI job" outputs: - shards: + shard: description: "Array of shard numbers" value: ${{ steps.shards.outputs.shards }} total_shards: diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index eb03567d..3675eac4 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -4,7 +4,7 @@ inputs: profile: description: "Profile to use" required: true - shards: + shard: description: "Shard number for this CI job" required: true total_shards: @@ -60,24 +60,11 @@ runs: conda-solver: libmamba conda-remove-defaults: true - # Set up secrets - - name: Set up nextflow secrets - if: env.SENTIEON_ENCRYPTION_KEY != '' && env.SENTIEON_LICENSE_MESSAGE != '' - shell: bash - run: | - python -m pip install cryptography - nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") - - # TODO Skip failing conda tests and document their failures - # https://github.com/nf-core/modules/issues/7017 - name: Clean up Disk space uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - name: Run nf-test shell: bash - env: - SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }} - SENTIEON_AUTH_MECH: "GitHub Actions - token" run: | NFT_WORKDIR=~ \ nf-test test \ diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 97a939a6..0e8ab625 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -31,15 +31,10 @@ env: jobs: get-shards: runs-on: ubuntu-latest - name: "Get shards and total number of shards" - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" + name: "Get Shards" outputs: - shards: ${{ steps.list-shards.outputs.shards }} - total_shards: ${{ steps.list-shards.outputs.total_shards }} - strategy: - fail-fast: true - + shard: ${{ steps.set-shards.outputs.shards }} + total_shards: ${{ steps.set-shards.outputs.total_shards }} steps: - name: Check out pipeline code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 @@ -47,14 +42,21 @@ jobs: fetch-depth: 0 - name: run nf-test-shard action - id: list-shards + id: set-shards uses: ./.github/actions/nf-test-shard env: NFT_VER: ${{ env.NFT_VER }} + - name: Set outputs + id: set-outputs + run: | + echo "shard=${{ steps.set-shards.outputs.shard }}" >> $GITHUB_ENV + echo "total_shards=${{ steps.set-shards.outputs.total_shards }}" >> $GITHUB_ENV + nf-test-gpu: runs-on: "gpu" name: "GPU Test | ${{ matrix.profile }} | ${{ matrix.shard }}" + needs: [get-shards] # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" strategy: @@ -77,14 +79,9 @@ jobs: - name: Run nf-test Action uses: ./.github/actions/nf-test - env: - SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} - SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} - SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} - SENTIEON_AUTH_MECH: "GitHub Actions - token" with: profile: ${{ matrix.profile }},gpu - shards: ${{ needs.get-shards.outputs.shards }} + shard: ${{ needs.get-shards.outputs.shard }} total_shards: ${{ needs.get-shards.outputs.total_shards }} filters: ${{ matrix.filter }} tags: "--tag gpu" diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 6d9a6c4a..4def396b 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -34,15 +34,10 @@ env: jobs: get-shards: runs-on: ubuntu-latest - name: "Get shards and total number of shards" - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" + name: "Get Shards" outputs: - shards: ${{ steps.list-shards.outputs.shards }} - total_shards: ${{ steps.list-shards.outputs.total_shards }} - strategy: - fail-fast: true - + shard: ${{ steps.set-shards.outputs.shards }} + total_shards: ${{ steps.set-shards.outputs.total_shards }} steps: - name: Check out pipeline code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 @@ -50,14 +45,21 @@ jobs: fetch-depth: 0 - name: run nf-test-shard action - id: list-shards + id: set-shards uses: ./.github/actions/nf-test-shard env: NFT_VER: ${{ env.NFT_VER }} + - name: Set outputs + id: set-outputs + run: | + echo "shard=${{ steps.set-shards.outputs.shard }}" >> $GITHUB_ENV + echo "total_shards=${{ steps.set-shards.outputs.total_shards }}" >> $GITHUB_ENV + nf-test: runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} name: "Test | ${{ matrix.profile }} | ${{ matrix.shard }}" + needs: [get-shards] # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" strategy: @@ -80,17 +82,11 @@ jobs: - name: Run nf-test Action uses: ./.github/actions/nf-test - env: - SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} - SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} - SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} - SENTIEON_AUTH_MECH: "GitHub Actions - token" with: profile: ${{ matrix.profile }} - shards: ${{ needs.get-shards.outputs.shards }} + shard: ${{ needs.get-shards.outputs.shard }} total_shards: ${{ needs.get-shards.outputs.total_shards }} filters: ${{ matrix.filter }} - tags: confirm-pass: runs-on: ubuntu-latest From 70093142e4a436356de370a9642a4628e0286dd3 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 14:07:15 +0000 Subject: [PATCH 44/89] fix job outputs --- .github/actions/nf-test-shard/action.yml | 2 +- .github/actions/nf-test/action.yml | 2 +- .github/workflows/nf-test-gpu.yml | 4 ++-- .github/workflows/nf-test.yml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/nf-test-shard/action.yml b/.github/actions/nf-test-shard/action.yml index a8054075..0f6f2cdb 100644 --- a/.github/actions/nf-test-shard/action.yml +++ b/.github/actions/nf-test-shard/action.yml @@ -31,7 +31,7 @@ runs: shards=$(seq 1 "$number_of_shards" | jq -R . | jq -c -s .) total_shards="$number_of_shards" fi - echo "shards=${shards}" >> $GITHUB_OUTPUT + echo "shard=${shard}" >> $GITHUB_OUTPUT echo "total_shards=${total_shards}" >> $GITHUB_OUTPUT # Debugging output echo "nftest_output: $nftest_output" diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index 3675eac4..a911cf60 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -72,7 +72,7 @@ runs: --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \ --changed-since HEAD^ \ --profile=${{ inputs.profile }} \ - --filter ${{ inputs.filter }} \ + --filter ${{ inputs.filters }} \ --tap=test.tap \ --verbose \ ${{ inputs.tags }} diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 0e8ab625..22c07308 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -65,7 +65,7 @@ jobs: NXF_VER: - "24.04.2" shard: [1, 2] - filter: [pipeline] + filters: [pipeline] profile: [docker, singularity] env: NXF_ANSI_LOG: false @@ -83,7 +83,7 @@ jobs: profile: ${{ matrix.profile }},gpu shard: ${{ needs.get-shards.outputs.shard }} total_shards: ${{ needs.get-shards.outputs.total_shards }} - filters: ${{ matrix.filter }} + filters: ${{ matrix.filters }} tags: "--tag gpu" confirm-pass: diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 4def396b..5d74986f 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -68,7 +68,7 @@ jobs: NXF_VER: - "24.04.2" shard: [1, 2] - filter: [pipeline] + filters: [pipeline] profile: [conda, docker_self_hosted, singularity] env: NXF_ANSI_LOG: false @@ -86,7 +86,7 @@ jobs: profile: ${{ matrix.profile }} shard: ${{ needs.get-shards.outputs.shard }} total_shards: ${{ needs.get-shards.outputs.total_shards }} - filters: ${{ matrix.filter }} + filters: ${{ matrix.filters }} confirm-pass: runs-on: ubuntu-latest From 804e70e68cecf92c16296dfcbf196fa2f54ca9cf Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 14:23:37 +0000 Subject: [PATCH 45/89] fix nf-test-shard action --- .github/actions/nf-test-shard/action.yml | 33 +++++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/actions/nf-test-shard/action.yml b/.github/actions/nf-test-shard/action.yml index 0f6f2cdb..af3ca823 100644 --- a/.github/actions/nf-test-shard/action.yml +++ b/.github/actions/nf-test-shard/action.yml @@ -21,19 +21,32 @@ runs: id: shards shell: bash run: | + # Run nf-test to get the number of related tests nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter pipeline) - echo "nftest_output: $nftest_output" + echo "nf-test dry-run output: $nftest_output" + + # Default values for shard and total_shards + shard="[]" + total_shards=0 + + # Check if there are related tests if echo "$nftest_output" | grep -q 'Found 0 related test(s)'; then - shards="[]" - total_shards=0 + echo "No related tests found." else + # Extract the number of related tests number_of_shards=$(echo "$nftest_output" | grep -o 'Found [0-9]* related test' | tail -1 | awk '{print $2}') - shards=$(seq 1 "$number_of_shards" | jq -R . | jq -c -s .) - total_shards="$number_of_shards" + if [[ -n "$number_of_shards" && "$number_of_shards" -gt 0 ]]; then + shard=$(seq 1 "$number_of_shards" | jq -R . | jq -c -s .) + total_shards="$number_of_shards" + else + echo "Unexpected output format. Falling back to default values." + fi fi - echo "shard=${shard}" >> $GITHUB_OUTPUT - echo "total_shards=${total_shards}" >> $GITHUB_OUTPUT + + # Write to GitHub Actions outputs + echo "shard=$shard" >> $GITHUB_OUTPUT + echo "total_shards=$total_shards" >> $GITHUB_OUTPUT + # Debugging output - echo "nftest_output: $nftest_output" - echo "shards: $shards" - echo "total_shards: $total_shards" + echo "Final shard array: $shard" + echo "Total number of shards: $total_shards" From 6a58c888a0df47e120dd043f5e8538da668bef88 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 14:27:39 +0000 Subject: [PATCH 46/89] fix nf-test-shard action --- .github/actions/nf-test-shard/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/nf-test-shard/action.yml b/.github/actions/nf-test-shard/action.yml index af3ca823..79e27cd9 100644 --- a/.github/actions/nf-test-shard/action.yml +++ b/.github/actions/nf-test-shard/action.yml @@ -3,7 +3,7 @@ description: "Get the number of nf-test shards for the current CI job" outputs: shard: description: "Array of shard numbers" - value: ${{ steps.shards.outputs.shards }} + value: ${{ steps.shards.outputs.shard }} total_shards: description: "Total number of shards" value: ${{ steps.shards.outputs.total_shards }} From 258449be9e62b8268b615779ebc8c1c767cea95e Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 14:31:41 +0000 Subject: [PATCH 47/89] fix nf-test-shard action --- .github/workflows/nf-test-gpu.yml | 2 +- .github/workflows/nf-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 22c07308..5c5915ab 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -33,7 +33,7 @@ jobs: runs-on: ubuntu-latest name: "Get Shards" outputs: - shard: ${{ steps.set-shards.outputs.shards }} + shard: ${{ steps.set-shards.outputs.shard }} total_shards: ${{ steps.set-shards.outputs.total_shards }} steps: - name: Check out pipeline code diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 5d74986f..59759c1a 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-latest name: "Get Shards" outputs: - shard: ${{ steps.set-shards.outputs.shards }} + shard: ${{ steps.set-shards.outputs.shard }} total_shards: ${{ steps.set-shards.outputs.total_shards }} steps: - name: Check out pipeline code From f6454d18cdd6010b2d350c513ea4def1e0b5e631 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 14:42:11 +0000 Subject: [PATCH 48/89] fix nf-test-shard action --- .github/workflows/nf-test-gpu.yml | 5 +---- .github/workflows/nf-test.yml | 7 ++----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 5c5915ab..e9fa43a6 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -64,12 +64,9 @@ jobs: matrix: NXF_VER: - "24.04.2" - shard: [1, 2] + shard: ${{ needs.get-shards.outputs.shard }} filters: [pipeline] profile: [docker, singularity] - env: - NXF_ANSI_LOG: false - TOTAL_SHARDS: 2 steps: - name: Check out pipeline code diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 59759c1a..53314dae 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -67,12 +67,9 @@ jobs: matrix: NXF_VER: - "24.04.2" - shard: [1, 2] + shard: ${{ needs.get-shards.outputs.shard }} filters: [pipeline] profile: [conda, docker_self_hosted, singularity] - env: - NXF_ANSI_LOG: false - TOTAL_SHARDS: 2 steps: - name: Check out pipeline code @@ -84,7 +81,7 @@ jobs: uses: ./.github/actions/nf-test with: profile: ${{ matrix.profile }} - shard: ${{ needs.get-shards.outputs.shard }} + shard: ${{ matrix.shard }} total_shards: ${{ needs.get-shards.outputs.total_shards }} filters: ${{ matrix.filters }} From 001b63f554f08b51ec971a68b38cc98f49bf96cd Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 14:49:56 +0000 Subject: [PATCH 49/89] get-shards matrix json parse --- .github/workflows/nf-test-gpu.yml | 2 +- .github/workflows/nf-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index e9fa43a6..c079b514 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -78,7 +78,7 @@ jobs: uses: ./.github/actions/nf-test with: profile: ${{ matrix.profile }},gpu - shard: ${{ needs.get-shards.outputs.shard }} + shard: ${{ fromJson(needs.get-shards.outputs.shard) }} total_shards: ${{ needs.get-shards.outputs.total_shards }} filters: ${{ matrix.filters }} tags: "--tag gpu" diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 53314dae..e1c866c5 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -67,7 +67,7 @@ jobs: matrix: NXF_VER: - "24.04.2" - shard: ${{ needs.get-shards.outputs.shard }} + shard: ${{ fromJson(needs.get-shards.outputs.shard) }} filters: [pipeline] profile: [conda, docker_self_hosted, singularity] From 118b817d63746575c9643d6316d1cb52e3d9c4d8 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 14:53:56 +0000 Subject: [PATCH 50/89] get-shards matrix json parse --- .github/workflows/nf-test-gpu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index c079b514..8d5d11cd 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -64,7 +64,7 @@ jobs: matrix: NXF_VER: - "24.04.2" - shard: ${{ needs.get-shards.outputs.shard }} + shard: ${{ fromJson(needs.get-shards.outputs.shard) }} filters: [pipeline] profile: [docker, singularity] @@ -78,7 +78,7 @@ jobs: uses: ./.github/actions/nf-test with: profile: ${{ matrix.profile }},gpu - shard: ${{ fromJson(needs.get-shards.outputs.shard) }} + shard: ${{ matrix.shard }} total_shards: ${{ needs.get-shards.outputs.total_shards }} filters: ${{ matrix.filters }} tags: "--tag gpu" From e2f76793c6029dd5bf5ad61d8138b47f692472c0 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 14:58:35 +0000 Subject: [PATCH 51/89] get-shards matrix json parse --- .github/workflows/nf-test-gpu.yml | 2 +- .github/workflows/nf-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 8d5d11cd..4afb8773 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -79,7 +79,7 @@ jobs: with: profile: ${{ matrix.profile }},gpu shard: ${{ matrix.shard }} - total_shards: ${{ needs.get-shards.outputs.total_shards }} + total_shards: 2 filters: ${{ matrix.filters }} tags: "--tag gpu" diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index e1c866c5..55ea68ee 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -82,7 +82,7 @@ jobs: with: profile: ${{ matrix.profile }} shard: ${{ matrix.shard }} - total_shards: ${{ needs.get-shards.outputs.total_shards }} + total_shards: 2 filters: ${{ matrix.filters }} confirm-pass: From d6a1aabb95fecabf8a1f28932b19631008318c78 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 3 Dec 2024 15:03:48 +0000 Subject: [PATCH 52/89] fix sharding --- .github/workflows/nf-test-gpu.yml | 2 +- .github/workflows/nf-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 4afb8773..8d5d11cd 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -79,7 +79,7 @@ jobs: with: profile: ${{ matrix.profile }},gpu shard: ${{ matrix.shard }} - total_shards: 2 + total_shards: ${{ needs.get-shards.outputs.total_shards }} filters: ${{ matrix.filters }} tags: "--tag gpu" diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 55ea68ee..e1c866c5 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -82,7 +82,7 @@ jobs: with: profile: ${{ matrix.profile }} shard: ${{ matrix.shard }} - total_shards: 2 + total_shards: ${{ needs.get-shards.outputs.total_shards }} filters: ${{ matrix.filters }} confirm-pass: From 46637d439e706d8f868b5db61f8a9bbb2efa981c Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Wed, 4 Dec 2024 10:42:16 +1300 Subject: [PATCH 53/89] Now filtering nf-test dry-run by --tag, picking up Executed instead of related and capped shards with max_shards --- .github/actions/nf-test-shard/action.yml | 18 +++++++++++++----- .github/actions/nf-test/action.yml | 4 ++-- .github/workflows/nf-test-gpu.yml | 5 ++++- .github/workflows/nf-test.yml | 4 ++++ 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/actions/nf-test-shard/action.yml b/.github/actions/nf-test-shard/action.yml index 79e27cd9..52837a5a 100644 --- a/.github/actions/nf-test-shard/action.yml +++ b/.github/actions/nf-test-shard/action.yml @@ -1,5 +1,12 @@ name: "Get number of shards" description: "Get the number of nf-test shards for the current CI job" +inputs: + tags: + description: "Tags to pass as argument for nf-test --tag parameter" + required: true + max_shards: + description: "Maximum number of shards allowed" + required: true outputs: shard: description: "Array of shard numbers" @@ -22,7 +29,7 @@ runs: shell: bash run: | # Run nf-test to get the number of related tests - nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter pipeline) + nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter pipeline --tag ${{ inputs.tags }}) echo "nf-test dry-run output: $nftest_output" # Default values for shard and total_shards @@ -30,14 +37,15 @@ runs: total_shards=0 # Check if there are related tests - if echo "$nftest_output" | grep -q 'Found 0 related test(s)'; then + if echo "$nftest_output" | grep -q 'Nothing to do'; then echo "No related tests found." else # Extract the number of related tests - number_of_shards=$(echo "$nftest_output" | grep -o 'Found [0-9]* related test' | tail -1 | awk '{print $2}') + number_of_shards=$(echo "$nftest_output" | sed -n 's|.*Executed \([0-9]*\) tests.*|\1|p') if [[ -n "$number_of_shards" && "$number_of_shards" -gt 0 ]]; then - shard=$(seq 1 "$number_of_shards" | jq -R . | jq -c -s .) - total_shards="$number_of_shards" + shards_to_run=$(( $number_of_shards < ${{ inputs.max_shards }} ? $number_of_shards : ${{ inputs.max_shards }} )) + shard=$(seq 1 "$shards_to_run" | jq -R . | jq -c -s .) + total_shards="$shards_to_run" else echo "Unexpected output format. Falling back to default values." fi diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index a911cf60..84131a13 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -14,7 +14,7 @@ inputs: description: "Filter test cases by specified types (e.g., module, pipeline, workflow or function)" required: true tags: - description: "Test tags to be run" + description: "Tags to pass as argument for nf-test --tag parameter" required: true runs: @@ -75,7 +75,7 @@ runs: --filter ${{ inputs.filters }} \ --tap=test.tap \ --verbose \ - ${{ inputs.tags }} + --tag ${{ inputs.tags }} # TODO If no test.tap, then make one to spoof? - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 8d5d11cd..00c435bd 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -46,6 +46,9 @@ jobs: uses: ./.github/actions/nf-test-shard env: NFT_VER: ${{ env.NFT_VER }} + with: + tags: "cpu" + max_shards: 2 - name: Set outputs id: set-outputs @@ -81,7 +84,7 @@ jobs: shard: ${{ matrix.shard }} total_shards: ${{ needs.get-shards.outputs.total_shards }} filters: ${{ matrix.filters }} - tags: "--tag gpu" + tags: "gpu" confirm-pass: runs-on: ubuntu-latest diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index e1c866c5..309f3a3a 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -49,6 +49,9 @@ jobs: uses: ./.github/actions/nf-test-shard env: NFT_VER: ${{ env.NFT_VER }} + with: + tags: "cpu" + max_shards: 5 - name: Set outputs id: set-outputs @@ -84,6 +87,7 @@ jobs: shard: ${{ matrix.shard }} total_shards: ${{ needs.get-shards.outputs.total_shards }} filters: ${{ matrix.filters }} + tags: "cpu" confirm-pass: runs-on: ubuntu-latest From c3c1592ef140738daea589cfa2d7343cdd3a28ca Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Wed, 4 Dec 2024 11:19:08 +1300 Subject: [PATCH 54/89] Fixed gpu typo, removed docker_self_hosted and bumped default resources --- .github/workflows/nf-test-gpu.yml | 2 +- .github/workflows/nf-test.yml | 2 +- nextflow.config | 5 ----- tests/nextflow.config | 10 +++++----- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 00c435bd..6f551a72 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -47,7 +47,7 @@ jobs: env: NFT_VER: ${{ env.NFT_VER }} with: - tags: "cpu" + tags: "gpu" max_shards: 2 - name: Set outputs diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 309f3a3a..0ef2825e 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -72,7 +72,7 @@ jobs: - "24.04.2" shard: ${{ fromJson(needs.get-shards.outputs.shard) }} filters: [pipeline] - profile: [conda, docker_self_hosted, singularity] + profile: [conda, docker, singularity] steps: - name: Check out pipeline code diff --git a/nextflow.config b/nextflow.config index 2e2ce746..153740e3 100644 --- a/nextflow.config +++ b/nextflow.config @@ -221,11 +221,6 @@ profiles { executor.cpus = 4 executor.memory = 8.GB } - docker_self_hosted { - docker.enabled = true - docker.fixOwnership = true - docker.runOptions = '--platform=linux/amd64' - } gpu { docker.runOptions = '-u $(id -u):$(id -g) --gpus all' apptainer.runOptions = '--nv' diff --git a/tests/nextflow.config b/tests/nextflow.config index 687ba08a..75fed85a 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -19,11 +19,11 @@ params { // Impose sensible resource limits for testing process { - withName: '.*' { - cpus = 2 - memory = 3.GB - time = 2.h - } + resourceLimits = [ + cpus: 4, + memory: '15.GB', + time: '2.h' + ] withName: PRESEQ_LCEXTRAP { errorStrategy = 'ignore' From 60947abac782641023811a55f9d7864eae2e9bc6 Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Wed, 4 Dec 2024 11:24:42 +1300 Subject: [PATCH 55/89] Reduced resources --- tests/nextflow.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/nextflow.config b/tests/nextflow.config index 75fed85a..8bc36ab6 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -20,8 +20,8 @@ params { // Impose sensible resource limits for testing process { resourceLimits = [ - cpus: 4, - memory: '15.GB', + cpus: 2, + memory: '3.GB', time: '2.h' ] From 7edb069c8a6a7a5ef51a4e1247099d4f7110539b Mon Sep 17 00:00:00 2001 From: Sateesh Date: Wed, 4 Dec 2024 03:50:23 +0000 Subject: [PATCH 56/89] add back docker_self_hosted profile + remove disk cleanup step --- .github/actions/nf-test/action.yml | 3 --- .github/workflows/nf-test-gpu.yml | 2 +- .github/workflows/nf-test.yml | 2 +- tests/nextflow.config | 8 ++++++++ 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index 84131a13..2745a4d5 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -60,9 +60,6 @@ runs: conda-solver: libmamba conda-remove-defaults: true - - name: Clean up Disk space - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - name: Run nf-test shell: bash run: | diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 6f551a72..e483576e 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -69,7 +69,7 @@ jobs: - "24.04.2" shard: ${{ fromJson(needs.get-shards.outputs.shard) }} filters: [pipeline] - profile: [docker, singularity] + profile: [docker_self_hosted, singularity] steps: - name: Check out pipeline code diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 0ef2825e..309f3a3a 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -72,7 +72,7 @@ jobs: - "24.04.2" shard: ${{ fromJson(needs.get-shards.outputs.shard) }} filters: [pipeline] - profile: [conda, docker, singularity] + profile: [conda, docker_self_hosted, singularity] steps: - name: Check out pipeline code diff --git a/tests/nextflow.config b/tests/nextflow.config index 8bc36ab6..dd4096ec 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -30,6 +30,14 @@ process { } } +profiles { + docker_self_hosted { + docker.enabled = true + docker.fixOwnership = true + docker.runOptions = '--platform=linux/amd64' + } +} + // Impose same minimum Nextflow version as the pipeline for testing manifest { nextflowVersion = '!>=24.04.2' From c381d9cb935f337b6f4fb1a9aac109e5d33bf663 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Wed, 4 Dec 2024 04:56:59 +0000 Subject: [PATCH 57/89] test latest-everything also for cpu tests --- .github/workflows/nf-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 309f3a3a..c5253281 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -70,6 +70,7 @@ jobs: matrix: NXF_VER: - "24.04.2" + - "latest-everything" shard: ${{ fromJson(needs.get-shards.outputs.shard) }} filters: [pipeline] profile: [conda, docker_self_hosted, singularity] From 4f78c3f27eebea61f7a7a57568134cc77481e2d3 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Wed, 4 Dec 2024 05:21:33 +0000 Subject: [PATCH 58/89] update bwameth cpu tests snaps --- tests/bwameth.nf.test.snap | 10 +++++----- tests/bwameth_rrbs.nf.test.snap | 10 +++++----- tests/bwameth_run_qualimap.nf.test.snap | 10 +++++----- ...th_save_reference_save_align_intermeds.nf.test.snap | 10 +++++----- tests/bwameth_skip_deduplication.nf.test.snap | 10 +++++----- tests/bwameth_skip_trimming.nf.test.snap | 10 +++++----- tests/nextflow.config | 4 ---- 7 files changed, 30 insertions(+), 34 deletions(-) diff --git a/tests/bwameth.nf.test.snap b/tests/bwameth.nf.test.snap index 067b917c..73919fc3 100644 --- a/tests/bwameth.nf.test.snap +++ b/tests/bwameth.nf.test.snap @@ -212,13 +212,13 @@ ], [ "Ecoli_10K_methylated.flagstat:md5,f520e87cc4050799a0dbed72909572ec", - "Ecoli_10K_methylated.stats:md5,6067718d5b05d81bb94929cd80b8839d", + "Ecoli_10K_methylated.stats:md5,be3e2bea0d6867aab35d06bfbdae0aa2", "SRR389222_sub1.flagstat:md5,ad8159a7683b9c3057bf54ccb77a2ddc", - "SRR389222_sub1.stats:md5,7df53a3d66857dd289911a13fed8497d", + "SRR389222_sub1.stats:md5,21e116cc6374c4a2d55d2c383cd4281e", "SRR389222_sub2.flagstat:md5,e768b4f54014d62b096152c44f920bc1", - "SRR389222_sub2.stats:md5,b0426a921c56ed8a86ade65adf7ed6dd", + "SRR389222_sub2.stats:md5,75cb0f1e953843150e3b217ec3f271da", "SRR389222_sub3.flagstat:md5,39516f7d2c77e6fef9543edf8cddbcd6", - "SRR389222_sub3.stats:md5,7c6053d7f87cbaa8782fc41a53d932fe", + "SRR389222_sub3.stats:md5,6876e424094ecdf302bd2097f8e21b1c", "Ecoli_10K_methylated.markdup.sorted_CpG.bedGraph:md5,d3bce01078b6e7749f3906bf9faad5b5", "SRR389222_sub1.markdup.sorted_CpG.bedGraph:md5,0944ad0e3b05791bf6dfd06be7ca4af1", "SRR389222_sub2.markdup.sorted_CpG.bedGraph:md5,0a3c19af55d1625e91903ae25d1d2098", @@ -294,7 +294,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-11-30T08:25:15.895637414" + "timestamp": "2024-12-04T04:32:19.286428584" }, "Params: bwameth with bwameth-index": { "content": [ diff --git a/tests/bwameth_rrbs.nf.test.snap b/tests/bwameth_rrbs.nf.test.snap index dba97ced..a3e8af2c 100644 --- a/tests/bwameth_rrbs.nf.test.snap +++ b/tests/bwameth_rrbs.nf.test.snap @@ -188,13 +188,13 @@ ], [ "Ecoli_10K_methylated.flagstat:md5,c6a9a8b50c5e1d239071a8b75bc63b4f", - "Ecoli_10K_methylated.stats:md5,cf163b532094830a574ccb005bcd5131", + "Ecoli_10K_methylated.stats:md5,b0f5b95cf23935f12ccfcba960cc350f", "SRR389222_sub1.flagstat:md5,cccae088e2f8686b890d019b60f63065", - "SRR389222_sub1.stats:md5,9ab407ef500cbf2f065892fe4ebc31bd", + "SRR389222_sub1.stats:md5,3ca5f1e6e587768302b3e1c54ea42121", "SRR389222_sub2.flagstat:md5,403af1bed1e3e96f5e04d65e96728828", - "SRR389222_sub2.stats:md5,41544bc1bedf31c7e1fa97487eb9ca2d", + "SRR389222_sub2.stats:md5,4b3589d2d54ec6fdc7fc4726723c698d", "SRR389222_sub3.flagstat:md5,73a5cb246f6388d5d449775d6e35f5fa", - "SRR389222_sub3.stats:md5,69b8a554734f573ca931e6aa4ae8f4f3", + "SRR389222_sub3.stats:md5,249976d67f53ea30f09f51d98b811f50", "Ecoli_10K_methylated.deduplicated.sorted_CpG.bedGraph:md5,aa55074c144aa6eaa184480038c75932", "SRR389222_sub1.deduplicated.sorted_CpG.bedGraph:md5,9e9e6922a8d99881874d299bfd7af5de", "SRR389222_sub2.deduplicated.sorted_CpG.bedGraph:md5,d6c973d9fa7c6bc5f47141d14c18a105", @@ -249,6 +249,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-11-30T08:28:38.388886798" + "timestamp": "2024-12-04T04:36:29.396758993" } } \ No newline at end of file diff --git a/tests/bwameth_run_qualimap.nf.test.snap b/tests/bwameth_run_qualimap.nf.test.snap index cc1376a2..1c83d052 100644 --- a/tests/bwameth_run_qualimap.nf.test.snap +++ b/tests/bwameth_run_qualimap.nf.test.snap @@ -373,13 +373,13 @@ ], [ "Ecoli_10K_methylated.flagstat:md5,f520e87cc4050799a0dbed72909572ec", - "Ecoli_10K_methylated.stats:md5,6067718d5b05d81bb94929cd80b8839d", + "Ecoli_10K_methylated.stats:md5,be3e2bea0d6867aab35d06bfbdae0aa2", "SRR389222_sub1.flagstat:md5,ad8159a7683b9c3057bf54ccb77a2ddc", - "SRR389222_sub1.stats:md5,7df53a3d66857dd289911a13fed8497d", + "SRR389222_sub1.stats:md5,21e116cc6374c4a2d55d2c383cd4281e", "SRR389222_sub2.flagstat:md5,e768b4f54014d62b096152c44f920bc1", - "SRR389222_sub2.stats:md5,b0426a921c56ed8a86ade65adf7ed6dd", + "SRR389222_sub2.stats:md5,75cb0f1e953843150e3b217ec3f271da", "SRR389222_sub3.flagstat:md5,39516f7d2c77e6fef9543edf8cddbcd6", - "SRR389222_sub3.stats:md5,7c6053d7f87cbaa8782fc41a53d932fe", + "SRR389222_sub3.stats:md5,6876e424094ecdf302bd2097f8e21b1c", "agogo.css:md5,bd757b1a7ce6fdc0288ba148680f4583", "ajax-loader.gif:md5,ae6667053ad118020b8e68ccf307b519", "basic.css:md5,25b2823342c0604924a2870eeb4e7e94", @@ -581,6 +581,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-12-02T12:46:20.537987965" + "timestamp": "2024-12-04T04:39:07.483507526" } } \ No newline at end of file diff --git a/tests/bwameth_save_reference_save_align_intermeds.nf.test.snap b/tests/bwameth_save_reference_save_align_intermeds.nf.test.snap index e1a70aca..abb2ba40 100644 --- a/tests/bwameth_save_reference_save_align_intermeds.nf.test.snap +++ b/tests/bwameth_save_reference_save_align_intermeds.nf.test.snap @@ -232,13 +232,13 @@ ], [ "Ecoli_10K_methylated.flagstat:md5,f520e87cc4050799a0dbed72909572ec", - "Ecoli_10K_methylated.stats:md5,6067718d5b05d81bb94929cd80b8839d", + "Ecoli_10K_methylated.stats:md5,be3e2bea0d6867aab35d06bfbdae0aa2", "SRR389222_sub1.flagstat:md5,ad8159a7683b9c3057bf54ccb77a2ddc", - "SRR389222_sub1.stats:md5,7df53a3d66857dd289911a13fed8497d", + "SRR389222_sub1.stats:md5,21e116cc6374c4a2d55d2c383cd4281e", "SRR389222_sub2.flagstat:md5,e768b4f54014d62b096152c44f920bc1", - "SRR389222_sub2.stats:md5,b0426a921c56ed8a86ade65adf7ed6dd", + "SRR389222_sub2.stats:md5,75cb0f1e953843150e3b217ec3f271da", "SRR389222_sub3.flagstat:md5,39516f7d2c77e6fef9543edf8cddbcd6", - "SRR389222_sub3.stats:md5,7c6053d7f87cbaa8782fc41a53d932fe", + "SRR389222_sub3.stats:md5,6876e424094ecdf302bd2097f8e21b1c", "genome.fa.bwameth.c2t:md5,e51d48ed28fa0c26e2f9c9f13d09403b", "genome.fa.bwameth.c2t.amb:md5,010a242c6764efb30141868a45d698b3", "genome.fa.bwameth.c2t.ann:md5,09b4db3d87a2d4dac9e10e807f377110", @@ -352,6 +352,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-11-30T08:30:20.392223072" + "timestamp": "2024-12-04T04:41:04.030358868" } } \ No newline at end of file diff --git a/tests/bwameth_skip_deduplication.nf.test.snap b/tests/bwameth_skip_deduplication.nf.test.snap index a2af2143..78b0dba3 100644 --- a/tests/bwameth_skip_deduplication.nf.test.snap +++ b/tests/bwameth_skip_deduplication.nf.test.snap @@ -188,13 +188,13 @@ ], [ "Ecoli_10K_methylated.flagstat:md5,f520e87cc4050799a0dbed72909572ec", - "Ecoli_10K_methylated.stats:md5,bed52531fe0a73a798d381dccbe2a248", + "Ecoli_10K_methylated.stats:md5,48d2b9fa62581706d67b027138cff41e", "SRR389222_sub1.flagstat:md5,ad8159a7683b9c3057bf54ccb77a2ddc", - "SRR389222_sub1.stats:md5,7e2b586e29251d4c53442882598b7299", + "SRR389222_sub1.stats:md5,a578019df62d8ae1e4976f20ac1c7993", "SRR389222_sub2.flagstat:md5,e768b4f54014d62b096152c44f920bc1", - "SRR389222_sub2.stats:md5,d49b348483aba5d7f80215458f8b830c", + "SRR389222_sub2.stats:md5,b60c66a26f2c346cbb85fb59c8de392b", "SRR389222_sub3.flagstat:md5,39516f7d2c77e6fef9543edf8cddbcd6", - "SRR389222_sub3.stats:md5,db87ee5d3eb2d84b846929edb51fbaba", + "SRR389222_sub3.stats:md5,cf3e88bd028b048c6f6a416eacef6a1b", "Ecoli_10K_methylated.sorted_CpG.bedGraph:md5,aa88f19284e63097e9abccac4d7c6669", "SRR389222_sub1.sorted_CpG.bedGraph:md5,4475a9462ce86a019015b69b82a9d5a2", "SRR389222_sub2.sorted_CpG.bedGraph:md5,18b8845d9d985d3c203ca6c9c40aab1b", @@ -249,6 +249,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-11-30T08:31:33.308611147" + "timestamp": "2024-12-04T04:42:38.088671477" } } \ No newline at end of file diff --git a/tests/bwameth_skip_trimming.nf.test.snap b/tests/bwameth_skip_trimming.nf.test.snap index d6a20a17..17f52c4a 100644 --- a/tests/bwameth_skip_trimming.nf.test.snap +++ b/tests/bwameth_skip_trimming.nf.test.snap @@ -173,13 +173,13 @@ ], [ "Ecoli_10K_methylated.flagstat:md5,7c3bae0f9c700dde88785a7e94c6ae2c", - "Ecoli_10K_methylated.stats:md5,aab233c65fcd8348af6f1a48e82e517f", + "Ecoli_10K_methylated.stats:md5,920027e64f96196193a9c585431f3edb", "SRR389222_sub1.flagstat:md5,897d500a710a56a7098172167fa71108", - "SRR389222_sub1.stats:md5,4ee1cbf4f178495ab51519b19555ba9d", + "SRR389222_sub1.stats:md5,ca9d535c6181d1f1b60172eb81f03061", "SRR389222_sub2.flagstat:md5,37d33d68b31d4550620dc17582a3a750", - "SRR389222_sub2.stats:md5,7d28cf8939637b827190b0507ff97ead", + "SRR389222_sub2.stats:md5,d38aa6d840e3af9d3c010cf788f8388b", "SRR389222_sub3.flagstat:md5,e3e925a7b7356e93d14f5e9e102c55b7", - "SRR389222_sub3.stats:md5,08059932de70d28e8f889729dd741cc1", + "SRR389222_sub3.stats:md5,a29f5a125fd90c5ba3d2e1100f33883c", "Ecoli_10K_methylated.markdup.sorted_CpG.bedGraph:md5,b43c4c8ff346ae865b2b5db996650942", "SRR389222_sub1.markdup.sorted_CpG.bedGraph:md5,0944ad0e3b05791bf6dfd06be7ca4af1", "SRR389222_sub2.markdup.sorted_CpG.bedGraph:md5,0a3c19af55d1625e91903ae25d1d2098", @@ -251,6 +251,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-11-30T08:32:50.058435618" + "timestamp": "2024-12-04T04:44:20.47452156" } } \ No newline at end of file diff --git a/tests/nextflow.config b/tests/nextflow.config index dd4096ec..5b09a39e 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -24,10 +24,6 @@ process { memory: '3.GB', time: '2.h' ] - - withName: PRESEQ_LCEXTRAP { - errorStrategy = 'ignore' - } } profiles { From 0066282ee55a04f1032c500aef95025af858bf18 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Thu, 5 Dec 2024 05:52:21 +0000 Subject: [PATCH 59/89] update bwameth gpu test snap --- tests/bwameth_use_gpu.nf.test.snap | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/bwameth_use_gpu.nf.test.snap b/tests/bwameth_use_gpu.nf.test.snap index 306d1f6b..10ddeb8e 100644 --- a/tests/bwameth_use_gpu.nf.test.snap +++ b/tests/bwameth_use_gpu.nf.test.snap @@ -182,11 +182,11 @@ ], [ "SRR389222_sub1.flagstat:md5,ad8159a7683b9c3057bf54ccb77a2ddc", - "SRR389222_sub1.stats:md5,7df53a3d66857dd289911a13fed8497d", + "SRR389222_sub1.stats:md5,21e116cc6374c4a2d55d2c383cd4281e", "SRR389222_sub2.flagstat:md5,e768b4f54014d62b096152c44f920bc1", - "SRR389222_sub2.stats:md5,b0426a921c56ed8a86ade65adf7ed6dd", + "SRR389222_sub2.stats:md5,75cb0f1e953843150e3b217ec3f271da", "SRR389222_sub3.flagstat:md5,39516f7d2c77e6fef9543edf8cddbcd6", - "SRR389222_sub3.stats:md5,7c6053d7f87cbaa8782fc41a53d932fe", + "SRR389222_sub3.stats:md5,6876e424094ecdf302bd2097f8e21b1c", "SRR389222_sub1.markdup.sorted_CpG.bedGraph:md5,0944ad0e3b05791bf6dfd06be7ca4af1", "SRR389222_sub2.markdup.sorted_CpG.bedGraph:md5,0a3c19af55d1625e91903ae25d1d2098", "SRR389222_sub3.markdup.sorted_CpG.bedGraph:md5,a91e6b039e9581a9d2ef6f6057918b39", @@ -247,8 +247,8 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.0" + "nextflow": "24.10.2" }, - "timestamp": "2024-12-02T11:55:47.061799" + "timestamp": "2024-12-05T05:51:21.384545" } } From 07ae4e2218ddfe9287af49c1d4f41549c88d31b2 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Thu, 5 Dec 2024 07:32:09 +0000 Subject: [PATCH 60/89] setup-nextflow use v2.0.0 --- .github/actions/nf-test/action.yml | 2 +- .github/workflows/download_pipeline.yml | 2 +- .github/workflows/linting.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index 2745a4d5..c2a18008 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -26,7 +26,7 @@ runs: java-version: "17" - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2 + uses: nf-core/setup-nextflow@v2.0.0 with: version: "${{ env.NXF_VERSION }}" diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml index 713dc3e7..4f109b3a 100644 --- a/.github/workflows/download_pipeline.yml +++ b/.github/workflows/download_pipeline.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Nextflow - uses: nf-core/setup-nextflow@v2 + uses: nf-core/setup-nextflow@v2.0.0 - name: Disk space cleanup uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index a502573c..eae17ff2 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -34,7 +34,7 @@ jobs: uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v2 + uses: nf-core/setup-nextflow@v2.0.0 - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: From c1b68abc01a1ed5b4193ee9e6f225726c142df90 Mon Sep 17 00:00:00 2001 From: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:29:48 +0530 Subject: [PATCH 61/89] Update .github/workflows/linting.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- .github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index eae17ff2..ae0fcf47 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -34,7 +34,7 @@ jobs: uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v2.0.0 + uses: nf-core/setup-nextflow@v57267e6c67300b3e82e0850adc6df50be8e16a26 - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: From 50f0f351d32d2e86ae11c976deb7f2f6c6055bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 5 Dec 2024 09:26:49 +0100 Subject: [PATCH 62/89] Update .github/workflows/linting.yml --- .github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index ae0fcf47..c6cc67e2 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -34,7 +34,7 @@ jobs: uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v57267e6c67300b3e82e0850adc6df50be8e16a26 + uses: nf-core/setup-nextflow@57267e6c67300b3e82e0850adc6df50be8e16a26 - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: From 67024a76061c58edee136c2a1cb10ce1ee0bd10a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 5 Dec 2024 10:10:15 +0100 Subject: [PATCH 63/89] going back to uriginal setup-nextflow version --- .github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index c6cc67e2..a502573c 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -34,7 +34,7 @@ jobs: uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@57267e6c67300b3e82e0850adc6df50be8e16a26 + uses: nf-core/setup-nextflow@v2 - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: From 4f5132a03b9484e8e9d023ae803515835956d75b Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 5 Dec 2024 10:13:14 +0100 Subject: [PATCH 64/89] try v2.1.1 --- .github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index a502573c..7f51eb08 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -34,7 +34,7 @@ jobs: uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v2 + uses: nf-core/setup-nextflow@v2.1.1 - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: From c4caa1e40854b1431a540354f5dae5a505e4297c Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 5 Dec 2024 10:32:19 +0100 Subject: [PATCH 65/89] try 2.1.2 --- .github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 7f51eb08..e3fbe376 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -34,7 +34,7 @@ jobs: uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v2.1.1 + uses: nf-core/setup-nextflow@v2.1.2 - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: From b883c8987b55eb3f00e8f95f423564a557e65b05 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 5 Dec 2024 10:42:55 +0100 Subject: [PATCH 66/89] try 2.1.3 --- .github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index e3fbe376..afb4e471 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -34,7 +34,7 @@ jobs: uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v2.1.2 + uses: nf-core/setup-nextflow@v2.1.3 - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: From 91e04d6afca462811d207b3d67c7d85090756755 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 5 Dec 2024 10:45:08 +0100 Subject: [PATCH 67/89] reset to broader v2 now that it is working again --- .github/actions/nf-test/action.yml | 2 +- .github/workflows/download_pipeline.yml | 2 +- .github/workflows/linting.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index c2a18008..2745a4d5 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -26,7 +26,7 @@ runs: java-version: "17" - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2.0.0 + uses: nf-core/setup-nextflow@v2 with: version: "${{ env.NXF_VERSION }}" diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml index 4f109b3a..713dc3e7 100644 --- a/.github/workflows/download_pipeline.yml +++ b/.github/workflows/download_pipeline.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Nextflow - uses: nf-core/setup-nextflow@v2.0.0 + uses: nf-core/setup-nextflow@v2 - name: Disk space cleanup uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index afb4e471..a502573c 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -34,7 +34,7 @@ jobs: uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v2.1.3 + uses: nf-core/setup-nextflow@v2 - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: From 1134404665d400eec56b9caa651a1b44b0efe1fa Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 5 Dec 2024 10:50:19 +0100 Subject: [PATCH 68/89] try again 2.1.3 explicitly --- .github/actions/nf-test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index 2745a4d5..ba3317d0 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -26,7 +26,7 @@ runs: java-version: "17" - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2 + uses: nf-core/setup-nextflow@v2.1.3 with: version: "${{ env.NXF_VERSION }}" From 0f5883923a7f215e60ad076cc7ca4696ac6f5b1e Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 5 Dec 2024 11:08:14 +0100 Subject: [PATCH 69/89] try 2.1.4 --- .github/actions/nf-test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index ba3317d0..380fee48 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -26,7 +26,7 @@ runs: java-version: "17" - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2.1.3 + uses: nf-core/setup-nextflow@v2.1.4 with: version: "${{ env.NXF_VERSION }}" From aaa58d1469ecd2b2c1ccffeeb562201dcaf4dc85 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 5 Dec 2024 11:12:23 +0100 Subject: [PATCH 70/89] and now reset back to v2 --- .github/actions/nf-test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index 380fee48..2745a4d5 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -26,7 +26,7 @@ runs: java-version: "17" - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2.1.4 + uses: nf-core/setup-nextflow@v2 with: version: "${{ env.NXF_VERSION }}" From ddb99ae30947760ad1f0bc3dd1e472977e49e979 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Thu, 5 Dec 2024 10:53:57 +0000 Subject: [PATCH 71/89] increase max_shards to 15 --- .github/workflows/nf-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index c5253281..985d5bdc 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -51,7 +51,7 @@ jobs: NFT_VER: ${{ env.NFT_VER }} with: tags: "cpu" - max_shards: 5 + max_shards: 15 - name: Set outputs id: set-outputs From ef2cdcbeec257f36ba5242df5d631fb3b4e083dc Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Tue, 10 Dec 2024 10:45:41 +1300 Subject: [PATCH 72/89] Corrected test labels, GPU tests with latest-everything, corrected env vars and excluded cancelled from confirm-pass --- .github/actions/nf-test/action.yml | 2 +- .github/workflows/nf-test-gpu.yml | 16 +++++++++------- .github/workflows/nf-test.yml | 16 +++++++++------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index 2745a4d5..9bc0bd4e 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -28,7 +28,7 @@ runs: - name: Setup Nextflow uses: nf-core/setup-nextflow@v2 with: - version: "${{ env.NXF_VERSION }}" + version: "${{ env.NXF_VER }}" - name: Set up Python uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index e483576e..bbc8b4af 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -25,8 +25,6 @@ env: NXF_ANSI_LOG: false NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity - # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver - NXF_VER: "24.10.2" jobs: get-shards: @@ -57,19 +55,23 @@ jobs: echo "total_shards=${{ steps.set-shards.outputs.total_shards }}" >> $GITHUB_ENV nf-test-gpu: - runs-on: "gpu" - name: "GPU Test | ${{ matrix.profile }} | ${{ matrix.shard }}" needs: [get-shards] + runs-on: "gpu" + name: "GPU Test | ${{ matrix.profile }} | ${{ matrix.shard }} | ${{ matrix.NXF_VER }} | ${{ matrix.filters }}" + env: + NXF_VER: ${{ matrix.NXF_VER }} + # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" strategy: fail-fast: false matrix: + profile: [docker_self_hosted, singularity] + shard: ${{ fromJson(needs.get-shards.outputs.shard) }} NXF_VER: - "24.04.2" - shard: ${{ fromJson(needs.get-shards.outputs.shard) }} + - "latest-everything" filters: [pipeline] - profile: [docker_self_hosted, singularity] steps: - name: Check out pipeline code @@ -92,7 +94,7 @@ jobs: if: always() steps: - name: All tests ok - if: ${{ success() || !contains(needs.*.result, 'failure') }} + if: ${{ ( success() || !contains(needs.*.result, 'failure') ) && !cancelled() }} run: exit 0 - name: One or more tests failed if: ${{ contains(needs.*.result, 'failure') }} diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 985d5bdc..33b8ddb7 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -28,8 +28,6 @@ env: NXF_ANSI_LOG: false NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity - # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver - NXF_VER: "24.10.2" jobs: get-shards: @@ -60,20 +58,24 @@ jobs: echo "total_shards=${{ steps.set-shards.outputs.total_shards }}" >> $GITHUB_ENV nf-test: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - name: "Test | ${{ matrix.profile }} | ${{ matrix.shard }}" needs: [get-shards] + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + name: "Test | ${{ matrix.profile }} | ${{ matrix.shard }} | ${{ matrix.NXF_VER }} | ${{ matrix.filters }}" + env: + NXF_VER: ${{ matrix.NXF_VER }} + # Only run on push if this is the nf-core dev branch (merged PRs) if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" + strategy: fail-fast: false matrix: + profile: [conda, docker_self_hosted, singularity] + shard: ${{ fromJson(needs.get-shards.outputs.shard) }} NXF_VER: - "24.04.2" - "latest-everything" - shard: ${{ fromJson(needs.get-shards.outputs.shard) }} filters: [pipeline] - profile: [conda, docker_self_hosted, singularity] steps: - name: Check out pipeline code @@ -96,7 +98,7 @@ jobs: if: always() steps: - name: All tests ok - if: ${{ success() || !contains(needs.*.result, 'failure') }} + if: ${{ ( success() || !contains(needs.*.result, 'failure') ) && !cancelled() }} run: exit 0 - name: One or more tests failed if: ${{ contains(needs.*.result, 'failure') }} From 05cb5c9ea68ad2f50e9d33b5504a771fa97327ea Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Tue, 10 Dec 2024 11:04:43 +1300 Subject: [PATCH 73/89] Disabled latest-everything for now --- .github/workflows/nf-test-gpu.yml | 1 - .github/workflows/nf-test.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index bbc8b4af..85978237 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -70,7 +70,6 @@ jobs: shard: ${{ fromJson(needs.get-shards.outputs.shard) }} NXF_VER: - "24.04.2" - - "latest-everything" filters: [pipeline] steps: diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 33b8ddb7..cb05c4b0 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -74,7 +74,6 @@ jobs: shard: ${{ fromJson(needs.get-shards.outputs.shard) }} NXF_VER: - "24.04.2" - - "latest-everything" filters: [pipeline] steps: From 5d8cc4bdbe44e3f5b601b9e8dc57f72fc58c42e5 Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Tue, 10 Dec 2024 11:42:39 +1300 Subject: [PATCH 74/89] Bumped nxf min version to 24.10.2 --- .github/workflows/nf-test-gpu.yml | 13 +++++++++---- .github/workflows/nf-test.yml | 13 +++++++++---- CHANGELOG.md | 1 + README.md | 2 +- .../nf-core/bismark/align/tests/main.nf.test.snap | 14 +++++++------- .../genomepreparation/tests/main.nf.test.snap | 2 +- modules/nf-core/gunzip/tests/main.nf.test.snap | 8 ++++---- .../preseq/lcextrap/tests/main.nf.test.snap | 8 ++++---- nextflow.config | 2 +- .../tests/main.nf.test.snap | 14 +++++++------- tests/bismark_hisat.nf.test.snap | 2 +- tests/bismark_run_preseq.nf.test.snap | 2 +- tests/bwameth.nf.test.snap | 2 +- tests/default.nf.test.snap | 2 +- tests/nextflow.config | 2 +- 15 files changed, 49 insertions(+), 38 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 85978237..d0c58a83 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -69,7 +69,7 @@ jobs: profile: [docker_self_hosted, singularity] shard: ${{ fromJson(needs.get-shards.outputs.shard) }} NXF_VER: - - "24.04.2" + - "24.10.2" filters: [pipeline] steps: @@ -92,13 +92,18 @@ jobs: needs: [nf-test-gpu] if: always() steps: - - name: All tests ok - if: ${{ ( success() || !contains(needs.*.result, 'failure') ) && !cancelled() }} - run: exit 0 - name: One or more tests failed if: ${{ contains(needs.*.result, 'failure') }} run: exit 1 + - name: One or more tests cancelled + if: ${{ contains(needs.*.result, 'cancelled') }} + run: exit 1 + + - name: All tests ok + if: ${{ contains(needs.*.result, 'success') }} + run: exit 0 + - name: debug-print if: always() run: | diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index cb05c4b0..000adc69 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -73,7 +73,7 @@ jobs: profile: [conda, docker_self_hosted, singularity] shard: ${{ fromJson(needs.get-shards.outputs.shard) }} NXF_VER: - - "24.04.2" + - "24.10.2" filters: [pipeline] steps: @@ -96,13 +96,18 @@ jobs: needs: [nf-test] if: always() steps: - - name: All tests ok - if: ${{ ( success() || !contains(needs.*.result, 'failure') ) && !cancelled() }} - run: exit 0 - name: One or more tests failed if: ${{ contains(needs.*.result, 'failure') }} run: exit 1 + - name: One or more tests cancelled + if: ${{ contains(needs.*.result, 'cancelled') }} + run: exit 1 + + - name: All tests ok + if: ${{ contains(needs.*.result, 'success') }} + run: exit 0 + - name: debug-print if: always() run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index abea1580..93b5971d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - 🔧 reorg individual configs to `conf/modules/` named configs [#459](https://github.com/nf-core/methylseq/pull/469) - 🔧 `run_preseq` param + skip preseq/lcextrap module by default [#458](https://github.com/nf-core/methylseq/pull/470) - 🔧 `run_qualimap` param + skip qualimap module by default [#367](https://github.com/nf-core/methylseq/pull/471) +- 🔧 Raised Nextflow version requirement to `24.10.2` ## [v2.7.1](https://github.com/nf-core/methylseq/releases/tag/2.7.1) - [2024-10-27] diff --git a/README.md b/README.md index 00202575..07a6f9f9 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![GitHub Actions Linting Status](https://github.com/nf-core/methylseq/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/methylseq/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/methylseq/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.1343417-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.1343417) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.2-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.10.2-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) diff --git a/modules/nf-core/bismark/align/tests/main.nf.test.snap b/modules/nf-core/bismark/align/tests/main.nf.test.snap index 2073fabe..5dfdfbfe 100644 --- a/modules/nf-core/bismark/align/tests/main.nf.test.snap +++ b/modules/nf-core/bismark/align/tests/main.nf.test.snap @@ -16,7 +16,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-20T07:06:10.385317868" }, @@ -43,7 +43,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-20T07:12:14.540260123" }, @@ -70,7 +70,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-20T07:04:42.515898978" }, @@ -91,7 +91,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-20T06:50:16.487384391" }, @@ -112,7 +112,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-20T06:51:06.34107409" }, @@ -139,7 +139,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-20T07:12:59.868431446" }, @@ -198,7 +198,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-20T07:16:21.454058443" } diff --git a/modules/nf-core/bismark/genomepreparation/tests/main.nf.test.snap b/modules/nf-core/bismark/genomepreparation/tests/main.nf.test.snap index 4fd18496..6feb91ac 100644 --- a/modules/nf-core/bismark/genomepreparation/tests/main.nf.test.snap +++ b/modules/nf-core/bismark/genomepreparation/tests/main.nf.test.snap @@ -147,7 +147,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-25T15:32:55.125985002" } diff --git a/modules/nf-core/gunzip/tests/main.nf.test.snap b/modules/nf-core/gunzip/tests/main.nf.test.snap index 069967e7..8e6df73a 100644 --- a/modules/nf-core/gunzip/tests/main.nf.test.snap +++ b/modules/nf-core/gunzip/tests/main.nf.test.snap @@ -28,7 +28,7 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nextflow": "24.10.2" }, "timestamp": "2024-06-25T11:35:10.861293" }, @@ -61,7 +61,7 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nextflow": "24.10.2" }, "timestamp": "2024-06-25T11:35:05.857145" }, @@ -94,7 +94,7 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nextflow": "24.10.2" }, "timestamp": "2023-10-17T15:35:37.690477896" }, @@ -127,7 +127,7 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nextflow": "24.10.2" }, "timestamp": "2024-06-25T11:33:32.921739" } diff --git a/modules/nf-core/preseq/lcextrap/tests/main.nf.test.snap b/modules/nf-core/preseq/lcextrap/tests/main.nf.test.snap index c725d8e8..fdaa4572 100644 --- a/modules/nf-core/preseq/lcextrap/tests/main.nf.test.snap +++ b/modules/nf-core/preseq/lcextrap/tests/main.nf.test.snap @@ -13,7 +13,7 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nextflow": "24.10.2" }, "timestamp": "2023-11-23T17:20:40.735535" }, @@ -23,7 +23,7 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nextflow": "24.10.2" }, "timestamp": "2023-11-23T17:20:51.981746" }, @@ -110,7 +110,7 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nextflow": "24.10.2" }, "timestamp": "2023-11-23T17:20:40.72985" }, @@ -128,7 +128,7 @@ ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nextflow": "24.10.2" }, "timestamp": "2023-11-23T17:20:51.998533" }, diff --git a/nextflow.config b/nextflow.config index 153740e3..04c8c9a3 100644 --- a/nextflow.config +++ b/nextflow.config @@ -299,7 +299,7 @@ manifest { homePage = 'https://github.com/nf-core/methylseq' description = """Methylation (Bisulfite-Sequencing) Best Practice analysis pipeline, part of the nf-core community.""" mainScript = 'main.nf' - nextflowVersion = '!>=24.04.2' + nextflowVersion = '!>=24.10.2' version = '2.8.0dev' doi = '10.5281/zenodo.1343417' } diff --git a/subworkflows/nf-core/fasta_index_bismark_bwameth/tests/main.nf.test.snap b/subworkflows/nf-core/fasta_index_bismark_bwameth/tests/main.nf.test.snap index 2bb64c16..dc0fcc34 100644 --- a/subworkflows/nf-core/fasta_index_bismark_bwameth/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fasta_index_bismark_bwameth/tests/main.nf.test.snap @@ -52,7 +52,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-26T16:24:46.519054694" }, @@ -101,7 +101,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-26T16:26:40.995024346" }, @@ -149,7 +149,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-26T16:27:06.257485123" }, @@ -210,7 +210,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-26T16:26:00.163900219" }, @@ -271,7 +271,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-26T16:25:35.90378279" }, @@ -314,7 +314,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-26T16:26:14.893224201" }, @@ -371,7 +371,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-26T16:25:20.094964567" } diff --git a/tests/bismark_hisat.nf.test.snap b/tests/bismark_hisat.nf.test.snap index 157689c9..d69b4477 100644 --- a/tests/bismark_hisat.nf.test.snap +++ b/tests/bismark_hisat.nf.test.snap @@ -5,7 +5,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-27T16:52:51.583828756" }, diff --git a/tests/bismark_run_preseq.nf.test.snap b/tests/bismark_run_preseq.nf.test.snap index 682358fa..3e390000 100644 --- a/tests/bismark_run_preseq.nf.test.snap +++ b/tests/bismark_run_preseq.nf.test.snap @@ -368,7 +368,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-27T17:06:26.829540988" } diff --git a/tests/bwameth.nf.test.snap b/tests/bwameth.nf.test.snap index 73919fc3..64df254a 100644 --- a/tests/bwameth.nf.test.snap +++ b/tests/bwameth.nf.test.snap @@ -302,7 +302,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-27T17:25:00.644734614" } diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 774cfb9b..7d3d0967 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -5,7 +5,7 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.1" + "nextflow": "24.10.2" }, "timestamp": "2024-11-27T17:37:46.406666357" }, diff --git a/tests/nextflow.config b/tests/nextflow.config index 5b09a39e..6766b79a 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -36,7 +36,7 @@ profiles { // Impose same minimum Nextflow version as the pipeline for testing manifest { - nextflowVersion = '!>=24.04.2' + nextflowVersion = '!>=24.10.2' } // Disable all Nextflow reporting options From 1e7746398824666e7052a9314dc92122215db661 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Mon, 9 Dec 2024 23:50:49 +0000 Subject: [PATCH 75/89] update qualimap paths in nftignore --- tests/.nftignore | 31 +++--- tests/bismark_run_qualimap.nf.test.snap | 133 +----------------------- tests/bwameth_run_qualimap.nf.test.snap | 107 +------------------ 3 files changed, 18 insertions(+), 253 deletions(-) diff --git a/tests/.nftignore b/tests/.nftignore index 5c3f0a21..2b6598d3 100644 --- a/tests/.nftignore +++ b/tests/.nftignore @@ -1,21 +1,22 @@ -{bismark,bismark_hisat}/alignments/logs/*.txt -{bismark,bismark_hisat,bwameth}/{alignments,deduplicated}/*.{bam,bam.bai} -{bismark,bismark_hisat}/deduplicated/logs/*.txt -{bismark,bismark_hisat}/{reports,summary}/*.{html,txt} -bwameth/deduplicated/picard_metrics/*.txt +*/alignments/logs/*.txt +*/{alignments,deduplicated}/*.{bam,bam.bai} +*/deduplicated/logs/*.txt +*/{reports,summary}/*.{html,txt} +*/deduplicated/picard_metrics/*.txt fastqc/*.html fastqc/zips/*.zip -multiqc/{bismark,bismark_hisat,bwameth}/multiqc_data/*.{log,json} -multiqc/{bismark,bismark_hisat,bwameth}/multiqc_data/multiqc_fastqc.txt -multiqc/{bismark,bismark_hisat,bwameth}/multiqc_data/multiqc_general_stats.txt -multiqc/{bismark,bismark_hisat,bwameth}/multiqc_data/multiqc_qualimap_bamqc_genome_results.txt -multiqc/{bismark,bismark_hisat,bwameth}/multiqc_data/multiqc_software_versions.txt -multiqc/{bismark,bismark_hisat,bwameth}/multiqc_data/multiqc_sources.txt -multiqc/{bismark,bismark_hisat,bwameth}/multiqc_report.html -multiqc/{bismark,bismark_hisat,bwameth}/multiqc_plots/{pdf,png,svg}/*.{pdf,png,svg} +multiqc/*/multiqc_data/*.{log,json} +multiqc/*/multiqc_data/multiqc_fastqc.txt +multiqc/*/multiqc_data/multiqc_general_stats.txt +multiqc/*/multiqc_data/multiqc_qualimap_bamqc_genome_results.txt +multiqc/*/multiqc_data/multiqc_software_versions.txt +multiqc/*/multiqc_data/multiqc_sources.txt +multiqc/*/multiqc_report.html +multiqc/*/multiqc_plots/{pdf,png,svg}/*.{pdf,png,svg} pipeline_info/*.{html,json,txt,yml} -{bismark,bismark_hisat,bwameth}/qualimap/bamqc/*/qualimapReport.html -qualimap/**/css/* +*/qualimap/bamqc/*/qualimapReport.html +*/qualimap/bamqc/**/*.{pdf,png,svg} +*/qualimap/bamqc/*/css/* qualimap/**/images_qualimapReport/* qualimap/**/raw_data_qualimapReport/* trimgalore/fastqc/*.html diff --git a/tests/bismark_run_qualimap.nf.test.snap b/tests/bismark_run_qualimap.nf.test.snap index 78214c46..d65d8ae4 100644 --- a/tests/bismark_run_qualimap.nf.test.snap +++ b/tests/bismark_run_qualimap.nf.test.snap @@ -520,42 +520,7 @@ "SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,94f18ee855566ef9842b4909e3d3dd88", "SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,7c3526e022e97e49677f7428badef420", "SRR389222_sub3_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,e8d2bf21731b6ed517c56385061b1dee", - "agogo.css:md5,bd757b1a7ce6fdc0288ba148680f4583", - "ajax-loader.gif:md5,ae6667053ad118020b8e68ccf307b519", - "basic.css:md5,25b2823342c0604924a2870eeb4e7e94", - "bgfooter.png:md5,ed01bb040346e4623cc87de331ddb4e1", - "bgtop.png:md5,626a50532dc6e5adbdd968746ef318e8", - "comment-bright.png:md5,0c850bb4920b581bf5e5dba5fa493a64", - "comment-close.png:md5,2635dda49c823e8122d4d11ed385f33d", - "comment.png:md5,882e40f3d6a16c6ed35659b105251525", - "doctools.js:md5,5ff571aa60e63f69c1890283e240ff8d", - "down-pressed.png:md5,ebe8979581eda700fb234a73c661a4b9", - "down.png:md5,f6f3c819cc7ca27d7fd3347e5e7ffe0f", - "file.png:md5,6587e59c55e626744eb6fc11129d99a7", - "jquery.js:md5,10092eee563dec2dca82b77d2cf5a1ae", - "minus.png:md5,8d572395aa95c89584a09813ada4dfa1", - "plus.png:md5,0125e6faa04e2cf0141a2d599d3bb220", - "pygments.css:md5,d625a0adb949f181bd0d3f1432b0fa7f", - "qualimap_logo_small.png:md5,7526f145a97be4682fd59e27dda4f71b", - "report.css:md5,7a5f09eaf7c176f966f4e8854168b812", - "searchtools.js:md5,d550841adeedc8ed47c40ee607620937", - "underscore.js:md5,db5ba047a66617d4cd3e8c5099cc51db", - "up-pressed.png:md5,8ea9bd109342f87fee97943b479c6f7e", - "up.png:md5,ecc373278454cc8ecc12d6ca69e55b36", - "websupport.js:md5,9e61e1e8a7433c56bd7e5a615affcf85", "genome_results.txt:md5,e2a57140a56e0e4a1905c70084ccbb41", - "genome_coverage_0to50_histogram.png:md5,f337e5205a6eca1b4e1b699cdec2ca9e", - "genome_coverage_across_reference.png:md5,cb236b2257837bb1c3e507a39bbea505", - "genome_coverage_histogram.png:md5,e4278f90d203c49fc65dd89ca986511f", - "genome_coverage_quotes.png:md5,105c926528fd0a641955bcc2a7361f4b", - "genome_gc_content_per_window.png:md5,ae0bb58187b0ed58f8ea386d138f4af6", - "genome_homopolymer_indels.png:md5,2f2f85d6051d00171b90b57586c4bf89", - "genome_insert_size_across_reference.png:md5,ec76e7f994c19c4b3128c1723d23a36b", - "genome_insert_size_histogram.png:md5,35410d385a13734a221db49bcbcb8125", - "genome_mapping_quality_across_reference.png:md5,98de0bd88a1675614eb222334eee02a1", - "genome_mapping_quality_histogram.png:md5,74776b1e7e8c8876af1cf89d9dbb5f31", - "genome_reads_content_per_read_position.png:md5,c6920dc357411880132c207c7d0aa68d", - "genome_uniq_read_starts_histogram.png:md5,8f7549a63eee3d3f022699aaa5bb858d", "coverage_across_reference.txt:md5,2bb9eb828acfd71ace8f1e6a2ca124b4", "coverage_histogram.txt:md5,6ee761660a6072e5b6b75b7fe8d2c69e", "duplication_rate_histogram.txt:md5,d5f25ad3572b98880b6904dcf4b3ed25", @@ -567,39 +532,7 @@ "mapped_reads_nucleotide_content.txt:md5,2e1916f7c31072e65529c44069be69bd", "mapping_quality_across_reference.txt:md5,7b29de82fbdad93249121e694f09abb2", "mapping_quality_histogram.txt:md5,9dc98cc4e42f82a21f438c349e7e55db", - "agogo.css:md5,bd757b1a7ce6fdc0288ba148680f4583", - "ajax-loader.gif:md5,ae6667053ad118020b8e68ccf307b519", - "basic.css:md5,25b2823342c0604924a2870eeb4e7e94", - "bgfooter.png:md5,ed01bb040346e4623cc87de331ddb4e1", - "bgtop.png:md5,626a50532dc6e5adbdd968746ef318e8", - "comment-bright.png:md5,0c850bb4920b581bf5e5dba5fa493a64", - "comment-close.png:md5,2635dda49c823e8122d4d11ed385f33d", - "comment.png:md5,882e40f3d6a16c6ed35659b105251525", - "doctools.js:md5,5ff571aa60e63f69c1890283e240ff8d", - "down-pressed.png:md5,ebe8979581eda700fb234a73c661a4b9", - "down.png:md5,f6f3c819cc7ca27d7fd3347e5e7ffe0f", - "file.png:md5,6587e59c55e626744eb6fc11129d99a7", - "jquery.js:md5,10092eee563dec2dca82b77d2cf5a1ae", - "minus.png:md5,8d572395aa95c89584a09813ada4dfa1", - "plus.png:md5,0125e6faa04e2cf0141a2d599d3bb220", - "pygments.css:md5,d625a0adb949f181bd0d3f1432b0fa7f", - "qualimap_logo_small.png:md5,7526f145a97be4682fd59e27dda4f71b", - "report.css:md5,7a5f09eaf7c176f966f4e8854168b812", - "searchtools.js:md5,d550841adeedc8ed47c40ee607620937", - "underscore.js:md5,db5ba047a66617d4cd3e8c5099cc51db", - "up-pressed.png:md5,8ea9bd109342f87fee97943b479c6f7e", - "up.png:md5,ecc373278454cc8ecc12d6ca69e55b36", - "websupport.js:md5,9e61e1e8a7433c56bd7e5a615affcf85", "genome_results.txt:md5,69a3dc8568127c50b0decb29757230ee", - "genome_coverage_0to50_histogram.png:md5,40ae43b035246671b104579d1f58c721", - "genome_coverage_across_reference.png:md5,30add2685de78a8d32d8db96d57d5b08", - "genome_coverage_histogram.png:md5,39476f1d524ea056cb6523c4d43014b2", - "genome_coverage_quotes.png:md5,e500432b1c4100d09f1f66011577d071", - "genome_gc_content_per_window.png:md5,9809c001b92268555415b0de60628850", - "genome_mapping_quality_across_reference.png:md5,2da419564a9d8c71e84e8d9037839711", - "genome_mapping_quality_histogram.png:md5,7ba0db9375d09fd49946b9bca4ba92b4", - "genome_reads_content_per_read_position.png:md5,1a6a7717ea948d2518808f72f58ffdef", - "genome_uniq_read_starts_histogram.png:md5,395413ab1d0e10af72b116fd8cb12685", "coverage_across_reference.txt:md5,081ddc3be3349b0c57d9676c03d84803", "coverage_histogram.txt:md5,47b75803a6925e883e254bb91d6220d2", "duplication_rate_histogram.txt:md5,3d06b34e40d1fdeb55c4ad19dc6ddad1", @@ -608,39 +541,7 @@ "mapped_reads_nucleotide_content.txt:md5,2a2d04e030d228b9e6c3267e6f9e6225", "mapping_quality_across_reference.txt:md5,33942e63dd23b4c0e307cd2f8c2fb299", "mapping_quality_histogram.txt:md5,5d14a661b9d3d59e4dfcb841084a6def", - "agogo.css:md5,bd757b1a7ce6fdc0288ba148680f4583", - "ajax-loader.gif:md5,ae6667053ad118020b8e68ccf307b519", - "basic.css:md5,25b2823342c0604924a2870eeb4e7e94", - "bgfooter.png:md5,ed01bb040346e4623cc87de331ddb4e1", - "bgtop.png:md5,626a50532dc6e5adbdd968746ef318e8", - "comment-bright.png:md5,0c850bb4920b581bf5e5dba5fa493a64", - "comment-close.png:md5,2635dda49c823e8122d4d11ed385f33d", - "comment.png:md5,882e40f3d6a16c6ed35659b105251525", - "doctools.js:md5,5ff571aa60e63f69c1890283e240ff8d", - "down-pressed.png:md5,ebe8979581eda700fb234a73c661a4b9", - "down.png:md5,f6f3c819cc7ca27d7fd3347e5e7ffe0f", - "file.png:md5,6587e59c55e626744eb6fc11129d99a7", - "jquery.js:md5,10092eee563dec2dca82b77d2cf5a1ae", - "minus.png:md5,8d572395aa95c89584a09813ada4dfa1", - "plus.png:md5,0125e6faa04e2cf0141a2d599d3bb220", - "pygments.css:md5,d625a0adb949f181bd0d3f1432b0fa7f", - "qualimap_logo_small.png:md5,7526f145a97be4682fd59e27dda4f71b", - "report.css:md5,7a5f09eaf7c176f966f4e8854168b812", - "searchtools.js:md5,d550841adeedc8ed47c40ee607620937", - "underscore.js:md5,db5ba047a66617d4cd3e8c5099cc51db", - "up-pressed.png:md5,8ea9bd109342f87fee97943b479c6f7e", - "up.png:md5,ecc373278454cc8ecc12d6ca69e55b36", - "websupport.js:md5,9e61e1e8a7433c56bd7e5a615affcf85", "genome_results.txt:md5,13d933d61649b837de72a418d5126c1b", - "genome_coverage_0to50_histogram.png:md5,dd2484bbca9a7c6f55def856746afa46", - "genome_coverage_across_reference.png:md5,f8b9f96f7e9de8d870e489f347272252", - "genome_coverage_histogram.png:md5,4c077ab4bc677b24650397f4c697b59c", - "genome_coverage_quotes.png:md5,8860c47649c597df8271868122cf167d", - "genome_gc_content_per_window.png:md5,5ea2ba99c19eaaa24140dc41f30fd427", - "genome_mapping_quality_across_reference.png:md5,60413ab77c118c3242f8cb1c6b5a0f35", - "genome_mapping_quality_histogram.png:md5,017451f46bc875ac58e7906bf7979956", - "genome_reads_content_per_read_position.png:md5,fd94ec5bfe5ca4adfa6ebdeaebe25157", - "genome_uniq_read_starts_histogram.png:md5,2f2555ef88c3bda05315469e684f5b19", "coverage_across_reference.txt:md5,a76b3da0ebb2f8bd6c36ec1902cf13f2", "coverage_histogram.txt:md5,ca854c05e9619233b7d95d44051cab80", "duplication_rate_histogram.txt:md5,530bb05c648d610227e121e1e74f41f8", @@ -649,39 +550,7 @@ "mapped_reads_nucleotide_content.txt:md5,ffdced213cd2a96408c4b9794092bd80", "mapping_quality_across_reference.txt:md5,a15709e1860fdd7deaf86dfa0d461144", "mapping_quality_histogram.txt:md5,feafb188d85e486d16374117565d88d1", - "agogo.css:md5,bd757b1a7ce6fdc0288ba148680f4583", - "ajax-loader.gif:md5,ae6667053ad118020b8e68ccf307b519", - "basic.css:md5,25b2823342c0604924a2870eeb4e7e94", - "bgfooter.png:md5,ed01bb040346e4623cc87de331ddb4e1", - "bgtop.png:md5,626a50532dc6e5adbdd968746ef318e8", - "comment-bright.png:md5,0c850bb4920b581bf5e5dba5fa493a64", - "comment-close.png:md5,2635dda49c823e8122d4d11ed385f33d", - "comment.png:md5,882e40f3d6a16c6ed35659b105251525", - "doctools.js:md5,5ff571aa60e63f69c1890283e240ff8d", - "down-pressed.png:md5,ebe8979581eda700fb234a73c661a4b9", - "down.png:md5,f6f3c819cc7ca27d7fd3347e5e7ffe0f", - "file.png:md5,6587e59c55e626744eb6fc11129d99a7", - "jquery.js:md5,10092eee563dec2dca82b77d2cf5a1ae", - "minus.png:md5,8d572395aa95c89584a09813ada4dfa1", - "plus.png:md5,0125e6faa04e2cf0141a2d599d3bb220", - "pygments.css:md5,d625a0adb949f181bd0d3f1432b0fa7f", - "qualimap_logo_small.png:md5,7526f145a97be4682fd59e27dda4f71b", - "report.css:md5,7a5f09eaf7c176f966f4e8854168b812", - "searchtools.js:md5,d550841adeedc8ed47c40ee607620937", - "underscore.js:md5,db5ba047a66617d4cd3e8c5099cc51db", - "up-pressed.png:md5,8ea9bd109342f87fee97943b479c6f7e", - "up.png:md5,ecc373278454cc8ecc12d6ca69e55b36", - "websupport.js:md5,9e61e1e8a7433c56bd7e5a615affcf85", "genome_results.txt:md5,235830239cfc43d8e84c049b1e9b180a", - "genome_coverage_0to50_histogram.png:md5,2cfd88b3aa6b417f73c0fc39f1a6f633", - "genome_coverage_across_reference.png:md5,7ee66fcdc90f02a84ae95e4a4ac8c7c3", - "genome_coverage_histogram.png:md5,c4caed1ef14844f6da4cc7b92543b379", - "genome_coverage_quotes.png:md5,66d0e8e1c0885b5f8ccf4683ebadbcd8", - "genome_gc_content_per_window.png:md5,2112a5d7c78711258303ef6fdc088346", - "genome_mapping_quality_across_reference.png:md5,2a066966adca0027fce00dbabf12f4b1", - "genome_mapping_quality_histogram.png:md5,a22ee10fcb9e34a84f607b089855fb92", - "genome_reads_content_per_read_position.png:md5,9fa406f353148448f6b9ac11880bc6de", - "genome_uniq_read_starts_histogram.png:md5,cf2fe3e30e11e7d6eb2db679cd12dc03", "coverage_across_reference.txt:md5,fb9c989fc6c15d0a8fd0a58b8d3c20ce", "coverage_histogram.txt:md5,8a3f45461442657433914c6a5ecb1301", "duplication_rate_histogram.txt:md5,ec9a705fd3a300b126d556afa14c326b", @@ -748,6 +617,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-12-02T12:34:53.964190449" + "timestamp": "2024-12-09T23:38:20.640992232" } } \ No newline at end of file diff --git a/tests/bwameth_run_qualimap.nf.test.snap b/tests/bwameth_run_qualimap.nf.test.snap index 1c83d052..cd181cf4 100644 --- a/tests/bwameth_run_qualimap.nf.test.snap +++ b/tests/bwameth_run_qualimap.nf.test.snap @@ -380,43 +380,7 @@ "SRR389222_sub2.stats:md5,75cb0f1e953843150e3b217ec3f271da", "SRR389222_sub3.flagstat:md5,39516f7d2c77e6fef9543edf8cddbcd6", "SRR389222_sub3.stats:md5,6876e424094ecdf302bd2097f8e21b1c", - "agogo.css:md5,bd757b1a7ce6fdc0288ba148680f4583", - "ajax-loader.gif:md5,ae6667053ad118020b8e68ccf307b519", - "basic.css:md5,25b2823342c0604924a2870eeb4e7e94", - "bgfooter.png:md5,ed01bb040346e4623cc87de331ddb4e1", - "bgtop.png:md5,626a50532dc6e5adbdd968746ef318e8", - "comment-bright.png:md5,0c850bb4920b581bf5e5dba5fa493a64", - "comment-close.png:md5,2635dda49c823e8122d4d11ed385f33d", - "comment.png:md5,882e40f3d6a16c6ed35659b105251525", - "doctools.js:md5,5ff571aa60e63f69c1890283e240ff8d", - "down-pressed.png:md5,ebe8979581eda700fb234a73c661a4b9", - "down.png:md5,f6f3c819cc7ca27d7fd3347e5e7ffe0f", - "file.png:md5,6587e59c55e626744eb6fc11129d99a7", - "jquery.js:md5,10092eee563dec2dca82b77d2cf5a1ae", - "minus.png:md5,8d572395aa95c89584a09813ada4dfa1", - "plus.png:md5,0125e6faa04e2cf0141a2d599d3bb220", - "pygments.css:md5,d625a0adb949f181bd0d3f1432b0fa7f", - "qualimap_logo_small.png:md5,7526f145a97be4682fd59e27dda4f71b", - "report.css:md5,7a5f09eaf7c176f966f4e8854168b812", - "searchtools.js:md5,d550841adeedc8ed47c40ee607620937", - "underscore.js:md5,db5ba047a66617d4cd3e8c5099cc51db", - "up-pressed.png:md5,8ea9bd109342f87fee97943b479c6f7e", - "up.png:md5,ecc373278454cc8ecc12d6ca69e55b36", - "websupport.js:md5,9e61e1e8a7433c56bd7e5a615affcf85", "genome_results.txt:md5,1ee2e29ac41c2cd7e6a0f00741e8d16e", - "genome_coverage_0to50_histogram.png:md5,2c5cd211837b2d4fe01ddad354d5211e", - "genome_coverage_across_reference.png:md5,eb361fdf3a6b96f91c6fea5083cff81f", - "genome_coverage_histogram.png:md5,4edfa52e16b71c63b80a802cb16a2f04", - "genome_coverage_quotes.png:md5,0297a2d5a33e5c0cd6d40961d33139ac", - "genome_gc_content_per_window.png:md5,aa7ea66cb94cd582db83e744203a137f", - "genome_homopolymer_indels.png:md5,e9406a58d8df6b1b930636b07f2a6767", - "genome_insert_size_across_reference.png:md5,3900fed8a1775605adcdcb9acd499b80", - "genome_insert_size_histogram.png:md5,33957d27ec5b7180cc8447532cbfef57", - "genome_mapping_quality_across_reference.png:md5,4d663703c8d8b967ce2372e457e6bba9", - "genome_mapping_quality_histogram.png:md5,4c06a9a29fba3dc84a2b37e736ada597", - "genome_reads_clipping_profile.png:md5,31c0938199fce4207269e12d158f63dc", - "genome_reads_content_per_read_position.png:md5,e655bfeb69a21f69b5dbdb181dd1586a", - "genome_uniq_read_starts_histogram.png:md5,f45743d65d23c9a53902480b018e828c", "coverage_across_reference.txt:md5,c4c7f930ad577d15de1358226acab252", "coverage_histogram.txt:md5,cddf42d662856425649c196c731f048b", "duplication_rate_histogram.txt:md5,764cb7ceebd9f6c40d68fac844d8645e", @@ -429,77 +393,8 @@ "mapped_reads_nucleotide_content.txt:md5,87d4e3b86ac646498e986e8b5e05226e", "mapping_quality_across_reference.txt:md5,c483c7c5c75cc05ab696743934ae006b", "mapping_quality_histogram.txt:md5,97b8ae36235e0dabc17f2b0697e9d078", - "agogo.css:md5,bd757b1a7ce6fdc0288ba148680f4583", - "ajax-loader.gif:md5,ae6667053ad118020b8e68ccf307b519", - "basic.css:md5,25b2823342c0604924a2870eeb4e7e94", - "bgfooter.png:md5,ed01bb040346e4623cc87de331ddb4e1", - "bgtop.png:md5,626a50532dc6e5adbdd968746ef318e8", - "comment-bright.png:md5,0c850bb4920b581bf5e5dba5fa493a64", - "comment-close.png:md5,2635dda49c823e8122d4d11ed385f33d", - "comment.png:md5,882e40f3d6a16c6ed35659b105251525", - "doctools.js:md5,5ff571aa60e63f69c1890283e240ff8d", - "down-pressed.png:md5,ebe8979581eda700fb234a73c661a4b9", - "down.png:md5,f6f3c819cc7ca27d7fd3347e5e7ffe0f", - "file.png:md5,6587e59c55e626744eb6fc11129d99a7", - "jquery.js:md5,10092eee563dec2dca82b77d2cf5a1ae", - "minus.png:md5,8d572395aa95c89584a09813ada4dfa1", - "plus.png:md5,0125e6faa04e2cf0141a2d599d3bb220", - "pygments.css:md5,d625a0adb949f181bd0d3f1432b0fa7f", - "qualimap_logo_small.png:md5,7526f145a97be4682fd59e27dda4f71b", - "report.css:md5,7a5f09eaf7c176f966f4e8854168b812", - "searchtools.js:md5,d550841adeedc8ed47c40ee607620937", - "underscore.js:md5,db5ba047a66617d4cd3e8c5099cc51db", - "up-pressed.png:md5,8ea9bd109342f87fee97943b479c6f7e", - "up.png:md5,ecc373278454cc8ecc12d6ca69e55b36", - "websupport.js:md5,9e61e1e8a7433c56bd7e5a615affcf85", "genome_results.txt:md5,2c755b15ae5ad4a1d29395320a6264ca", - "agogo.css:md5,bd757b1a7ce6fdc0288ba148680f4583", - "ajax-loader.gif:md5,ae6667053ad118020b8e68ccf307b519", - "basic.css:md5,25b2823342c0604924a2870eeb4e7e94", - "bgfooter.png:md5,ed01bb040346e4623cc87de331ddb4e1", - "bgtop.png:md5,626a50532dc6e5adbdd968746ef318e8", - "comment-bright.png:md5,0c850bb4920b581bf5e5dba5fa493a64", - "comment-close.png:md5,2635dda49c823e8122d4d11ed385f33d", - "comment.png:md5,882e40f3d6a16c6ed35659b105251525", - "doctools.js:md5,5ff571aa60e63f69c1890283e240ff8d", - "down-pressed.png:md5,ebe8979581eda700fb234a73c661a4b9", - "down.png:md5,f6f3c819cc7ca27d7fd3347e5e7ffe0f", - "file.png:md5,6587e59c55e626744eb6fc11129d99a7", - "jquery.js:md5,10092eee563dec2dca82b77d2cf5a1ae", - "minus.png:md5,8d572395aa95c89584a09813ada4dfa1", - "plus.png:md5,0125e6faa04e2cf0141a2d599d3bb220", - "pygments.css:md5,d625a0adb949f181bd0d3f1432b0fa7f", - "qualimap_logo_small.png:md5,7526f145a97be4682fd59e27dda4f71b", - "report.css:md5,7a5f09eaf7c176f966f4e8854168b812", - "searchtools.js:md5,d550841adeedc8ed47c40ee607620937", - "underscore.js:md5,db5ba047a66617d4cd3e8c5099cc51db", - "up-pressed.png:md5,8ea9bd109342f87fee97943b479c6f7e", - "up.png:md5,ecc373278454cc8ecc12d6ca69e55b36", - "websupport.js:md5,9e61e1e8a7433c56bd7e5a615affcf85", "genome_results.txt:md5,2294badeccd2e8870fd6c60d115687b0", - "agogo.css:md5,bd757b1a7ce6fdc0288ba148680f4583", - "ajax-loader.gif:md5,ae6667053ad118020b8e68ccf307b519", - "basic.css:md5,25b2823342c0604924a2870eeb4e7e94", - "bgfooter.png:md5,ed01bb040346e4623cc87de331ddb4e1", - "bgtop.png:md5,626a50532dc6e5adbdd968746ef318e8", - "comment-bright.png:md5,0c850bb4920b581bf5e5dba5fa493a64", - "comment-close.png:md5,2635dda49c823e8122d4d11ed385f33d", - "comment.png:md5,882e40f3d6a16c6ed35659b105251525", - "doctools.js:md5,5ff571aa60e63f69c1890283e240ff8d", - "down-pressed.png:md5,ebe8979581eda700fb234a73c661a4b9", - "down.png:md5,f6f3c819cc7ca27d7fd3347e5e7ffe0f", - "file.png:md5,6587e59c55e626744eb6fc11129d99a7", - "jquery.js:md5,10092eee563dec2dca82b77d2cf5a1ae", - "minus.png:md5,8d572395aa95c89584a09813ada4dfa1", - "plus.png:md5,0125e6faa04e2cf0141a2d599d3bb220", - "pygments.css:md5,d625a0adb949f181bd0d3f1432b0fa7f", - "qualimap_logo_small.png:md5,7526f145a97be4682fd59e27dda4f71b", - "report.css:md5,7a5f09eaf7c176f966f4e8854168b812", - "searchtools.js:md5,d550841adeedc8ed47c40ee607620937", - "underscore.js:md5,db5ba047a66617d4cd3e8c5099cc51db", - "up-pressed.png:md5,8ea9bd109342f87fee97943b479c6f7e", - "up.png:md5,ecc373278454cc8ecc12d6ca69e55b36", - "websupport.js:md5,9e61e1e8a7433c56bd7e5a615affcf85", "genome_results.txt:md5,4da1144933d094185d02f7482c120804", "versions.yml:md5,bb8bc6f11e6ca441cdb653f077b870c2", "Ecoli_10K_methylated.markdup.sorted_CpG.bedGraph:md5,d3bce01078b6e7749f3906bf9faad5b5", @@ -581,6 +476,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.2" }, - "timestamp": "2024-12-04T04:39:07.483507526" + "timestamp": "2024-12-09T23:50:03.744243522" } } \ No newline at end of file From 336d466e0ba3bd914dde1a3a60698adce9dd4a15 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 10 Dec 2024 03:58:04 +0000 Subject: [PATCH 76/89] remove test folder contents only --- .github/actions/nf-test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index 9bc0bd4e..b5391c7f 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -85,4 +85,4 @@ runs: if: always() shell: bash run: | - sudo rm -rf /home/ubuntu/tests/ + sudo rm -rf /home/ubuntu/tests/* From 18b389cddddc64a3ff32e595f96f0f906a968da6 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 10 Dec 2024 09:52:09 +0000 Subject: [PATCH 77/89] use ubuntu-latest as default runner --- .github/workflows/nf-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 000adc69..1c8e33b4 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -14,7 +14,7 @@ on: options: - "ubuntu-latest" - "self-hosted" - default: "self-hosted" + default: "ubuntu-latest" # Cancel if a newer run is started concurrency: From e98c5de8725060da4aa54a4dbc9f4114ddb81a8d Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 10 Dec 2024 09:57:24 +0000 Subject: [PATCH 78/89] use ubuntu-latest as default runner --- .github/workflows/nf-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 1c8e33b4..2a0a8626 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -70,7 +70,7 @@ jobs: strategy: fail-fast: false matrix: - profile: [conda, docker_self_hosted, singularity] + profile: [conda, docker, singularity] shard: ${{ fromJson(needs.get-shards.outputs.shard) }} NXF_VER: - "24.10.2" From 231d9d18fb393af40d224f53ecad2a1c4df8a13f Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 10 Dec 2024 10:16:48 +0000 Subject: [PATCH 79/89] use ubuntu-latest as default runner --- .github/workflows/nf-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 2a0a8626..9e7eef26 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -59,7 +59,7 @@ jobs: nf-test: needs: [get-shards] - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + runs-on: ${{ github.event.inputs.runners }} name: "Test | ${{ matrix.profile }} | ${{ matrix.shard }} | ${{ matrix.NXF_VER }} | ${{ matrix.filters }}" env: NXF_VER: ${{ matrix.NXF_VER }} From f8787df9e7a3591513288ce1989346a3df25a003 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 10 Dec 2024 10:19:16 +0000 Subject: [PATCH 80/89] use ubuntu-latest as default runner --- .github/workflows/nf-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 9e7eef26..4af442fe 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -59,7 +59,7 @@ jobs: nf-test: needs: [get-shards] - runs-on: ${{ github.event.inputs.runners }} + runs-on: "ubuntu-latest" name: "Test | ${{ matrix.profile }} | ${{ matrix.shard }} | ${{ matrix.NXF_VER }} | ${{ matrix.filters }}" env: NXF_VER: ${{ matrix.NXF_VER }} From 57f57569e36f2cc8ddc5dae3bf64de96a89aad01 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 10 Dec 2024 11:11:47 +0000 Subject: [PATCH 81/89] add latest-everything to matrix --- .github/workflows/nf-test-gpu.yml | 2 ++ .github/workflows/nf-test.yml | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index d0c58a83..3b001888 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -25,6 +25,8 @@ env: NXF_ANSI_LOG: false NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver + NXF_VER: "24.10.2" jobs: get-shards: diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 4af442fe..9f0f0190 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -28,6 +28,8 @@ env: NXF_ANSI_LOG: false NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver + NXF_VER: "24.10.2" jobs: get-shards: @@ -73,7 +75,8 @@ jobs: profile: [conda, docker, singularity] shard: ${{ fromJson(needs.get-shards.outputs.shard) }} NXF_VER: - - "24.10.2" + - "24.04.2" + - "latest-everything" filters: [pipeline] steps: From c23d1ccef78713ba7bc519b2d96c24e800302a92 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 10 Dec 2024 11:17:52 +0000 Subject: [PATCH 82/89] correct NXF VER for nf-test --- .github/workflows/nf-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 9f0f0190..94867285 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -75,7 +75,7 @@ jobs: profile: [conda, docker, singularity] shard: ${{ fromJson(needs.get-shards.outputs.shard) }} NXF_VER: - - "24.04.2" + - "24.10.2" - "latest-everything" filters: [pipeline] From 18119abd3bc80cc42c11a470a700fb35b38b12f9 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Tue, 10 Dec 2024 11:31:27 +0000 Subject: [PATCH 83/89] rm latest-everything from matrix --- .github/workflows/nf-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 94867285..bbc212f2 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -76,7 +76,6 @@ jobs: shard: ${{ fromJson(needs.get-shards.outputs.shard) }} NXF_VER: - "24.10.2" - - "latest-everything" filters: [pipeline] steps: From 099fd574ac065a094b37cd3cda377e63d650179b Mon Sep 17 00:00:00 2001 From: Sateesh Date: Wed, 11 Dec 2024 14:17:24 +0000 Subject: [PATCH 84/89] use gpu profile in tests config --- .github/workflows/nf-test-gpu.yml | 2 +- tests/nextflow.config | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 3b001888..c2786638 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -68,7 +68,7 @@ jobs: strategy: fail-fast: false matrix: - profile: [docker_self_hosted, singularity] + profile: [gpu] shard: ${{ fromJson(needs.get-shards.outputs.shard) }} NXF_VER: - "24.10.2" diff --git a/tests/nextflow.config b/tests/nextflow.config index 6766b79a..a899e619 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -27,6 +27,12 @@ process { } profiles { + gpu { + docker.runOptions = '-u $(id -u):$(id -g) --gpus all' + apptainer.runOptions = '--nv' + singularity.runOptions = '--nv' + use_gpu = true + } docker_self_hosted { docker.enabled = true docker.fixOwnership = true From e6cd57be4074704a3e15ce884cf06c0683b9db69 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Wed, 11 Dec 2024 14:38:21 +0000 Subject: [PATCH 85/89] update test gpu configs --- .github/workflows/nf-test-gpu.yml | 4 ++-- tests/nextflow.config | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index c2786638..a3f64372 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -59,7 +59,7 @@ jobs: nf-test-gpu: needs: [get-shards] runs-on: "gpu" - name: "GPU Test | ${{ matrix.profile }} | ${{ matrix.shard }} | ${{ matrix.NXF_VER }} | ${{ matrix.filters }}" + name: "Test | ${{ matrix.profile }} | ${{ matrix.shard }} | ${{ matrix.NXF_VER }} | ${{ matrix.filters }}" env: NXF_VER: ${{ matrix.NXF_VER }} @@ -68,7 +68,7 @@ jobs: strategy: fail-fast: false matrix: - profile: [gpu] + profile: [singularity_gpu, docker_gpu] shard: ${{ fromJson(needs.get-shards.outputs.shard) }} NXF_VER: - "24.10.2" diff --git a/tests/nextflow.config b/tests/nextflow.config index a899e619..c0c2ba8a 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -27,16 +27,17 @@ process { } profiles { - gpu { - docker.runOptions = '-u $(id -u):$(id -g) --gpus all' - apptainer.runOptions = '--nv' + singularity_gpu { + singularity.enabled = true + singularity.autoMounts = true singularity.runOptions = '--nv' use_gpu = true } - docker_self_hosted { + docker_gpu { docker.enabled = true docker.fixOwnership = true - docker.runOptions = '--platform=linux/amd64' + docker.runOptions = '-u $(id -u):$(id -g) --gpus all' + use_gpu = true } } From 9703f605ee636bb2d157ef8b0964e021ca79b315 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Thu, 12 Dec 2024 01:17:48 +0000 Subject: [PATCH 86/89] use gpu+docker/singularity profile for nf-test --- .github/workflows/nf-test-gpu.yml | 4 ++-- tests/nextflow.config | 15 --------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index a3f64372..464af97c 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -68,7 +68,7 @@ jobs: strategy: fail-fast: false matrix: - profile: [singularity_gpu, docker_gpu] + profile: [docker, singularity] shard: ${{ fromJson(needs.get-shards.outputs.shard) }} NXF_VER: - "24.10.2" @@ -83,7 +83,7 @@ jobs: - name: Run nf-test Action uses: ./.github/actions/nf-test with: - profile: ${{ matrix.profile }},gpu + profile: ${{ matrix.profile }}+gpu shard: ${{ matrix.shard }} total_shards: ${{ needs.get-shards.outputs.total_shards }} filters: ${{ matrix.filters }} diff --git a/tests/nextflow.config b/tests/nextflow.config index c0c2ba8a..13136c1e 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -26,21 +26,6 @@ process { ] } -profiles { - singularity_gpu { - singularity.enabled = true - singularity.autoMounts = true - singularity.runOptions = '--nv' - use_gpu = true - } - docker_gpu { - docker.enabled = true - docker.fixOwnership = true - docker.runOptions = '-u $(id -u):$(id -g) --gpus all' - use_gpu = true - } -} - // Impose same minimum Nextflow version as the pipeline for testing manifest { nextflowVersion = '!>=24.10.2' From b73ea64830c0f4d7dd1580e914faa0a31065a7f8 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Thu, 12 Dec 2024 01:26:00 +0000 Subject: [PATCH 87/89] use gpu,docker/singularity profile for nf-test --- .github/workflows/nf-test-gpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 464af97c..67723d77 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -83,7 +83,7 @@ jobs: - name: Run nf-test Action uses: ./.github/actions/nf-test with: - profile: ${{ matrix.profile }}+gpu + profile: ${{ matrix.profile }},gpu shard: ${{ matrix.shard }} total_shards: ${{ needs.get-shards.outputs.total_shards }} filters: ${{ matrix.filters }} From 32e716f43188ad0e64a6fb64f94fc9d5d0c2d06c Mon Sep 17 00:00:00 2001 From: Sateesh Date: Thu, 12 Dec 2024 01:27:37 +0000 Subject: [PATCH 88/89] use gpu,docker/singularity profile for nf-test --- .github/workflows/nf-test-gpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 67723d77..1eb05c0b 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -59,7 +59,7 @@ jobs: nf-test-gpu: needs: [get-shards] runs-on: "gpu" - name: "Test | ${{ matrix.profile }} | ${{ matrix.shard }} | ${{ matrix.NXF_VER }} | ${{ matrix.filters }}" + name: "GPU Test | ${{ matrix.profile }} | ${{ matrix.shard }} | ${{ matrix.NXF_VER }} | ${{ matrix.filters }}" env: NXF_VER: ${{ matrix.NXF_VER }} From 57db3d88b43aad1b6f6d746d459c6d5ea7ae3c27 Mon Sep 17 00:00:00 2001 From: Sateesh Date: Thu, 12 Dec 2024 01:37:34 +0000 Subject: [PATCH 89/89] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93b5971d..51eca1f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - 🔧 `run_preseq` param + skip preseq/lcextrap module by default [#458](https://github.com/nf-core/methylseq/pull/470) - 🔧 `run_qualimap` param + skip qualimap module by default [#367](https://github.com/nf-core/methylseq/pull/471) - 🔧 Raised Nextflow version requirement to `24.10.2` +- 🔧 Update the CI for pipeline-level bwameth GPU Tests [#481](https://github.com/nf-core/methylseq/pull/478) ## [v2.7.1](https://github.com/nf-core/methylseq/releases/tag/2.7.1) - [2024-10-27]