Merge pull request #475 from nf-core/fasta-align-dedup-bwameth

Fasta align dedup bwameth

.github/actions/nf-test-shard/action.yml

@@ -0,0 +1,60 @@
+name: "Get number of shards"
+description: "Get the number of nf-test shards for the current CI job"
+inputs:
+  tags:
+    description: "Tags to pass as argument for nf-test --tag parameter"
+    required: true
+  max_shards:
+    description: "Maximum number of shards allowed"
+    required: true
+outputs:
+  shard:
+    description: "Array of shard numbers"
+    value: ${{ steps.shards.outputs.shard }}
+  total_shards:
+    description: "Total number of shards"
+    value: ${{ steps.shards.outputs.total_shards }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install nf-test
+      uses: nf-core/setup-nf-test@v1
+      with:
+        version: ${{ env.NFT_VER }}
+        install-pdiff: true
+
+    - name: Get number of shards
+      id: shards
+      shell: bash
+      run: |
+        # Run nf-test to get the number of related tests
+        nftest_output=$(nf-test test --dry-run --changed-since HEAD^ --filter pipeline --tag ${{ inputs.tags }})
+        echo "nf-test dry-run output: $nftest_output"
+
+        # Default values for shard and total_shards
+        shard="[]"
+        total_shards=0
+
+        # Check if there are related tests
+        if echo "$nftest_output" | grep -q 'Nothing to do'; then
+          echo "No related tests found."
+        else
+          # Extract the number of related tests
+          number_of_shards=$(echo "$nftest_output" | sed -n 's|.*Executed \([0-9]*\) tests.*|\1|p')
+          if [[ -n "$number_of_shards" && "$number_of_shards" -gt 0 ]]; then
+            shards_to_run=$(( $number_of_shards < ${{ inputs.max_shards }} ? $number_of_shards : ${{ inputs.max_shards }} ))
+            shard=$(seq 1 "$shards_to_run" | jq -R . | jq -c -s .)
+            total_shards="$shards_to_run"
+          else
+            echo "Unexpected output format. Falling back to default values."
+          fi
+        fi
+
+        # Write to GitHub Actions outputs
+        echo "shard=$shard" >> $GITHUB_OUTPUT
+        echo "total_shards=$total_shards" >> $GITHUB_OUTPUT
+
+        # Debugging output
+        echo "Final shard array: $shard"
+        echo "Total number of shards: $total_shards"

.github/actions/nf-test/action.yml

@@ -0,0 +1,88 @@
+name: "nf-test Action"
+description: "Runs nf-test with common setup steps"
+inputs:
+  profile:
+    description: "Profile to use"
+    required: true
+  shard:
+    description: "Shard number for this CI job"
+    required: true
+  total_shards:
+    description: "Total number of test shards(NOT the total number of matrix jobs)"
+    required: true
+  filters:
+    description: "Filter test cases by specified types (e.g., module, pipeline, workflow or function)"
+    required: true
+  tags:
+    description: "Tags to pass as argument for nf-test --tag parameter"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4
+      with:
+        distribution: "temurin"
+        java-version: "17"
+
+    - name: Setup Nextflow
+      uses: nf-core/setup-nextflow@v2
+      with:
+        version: "${{ env.NXF_VER }}"
+
+    - name: Set up Python
+      uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
+      with:
+        python-version: "3.11"
+
+    - name: Install nf-test
+      uses: nf-core/setup-nf-test@v1
+      with:
+        version: "${{ env.NFT_VER }}"
+        install-pdiff: true
+
+    - name: Setup apptainer
+      if: contains(inputs.profile, 'singularity')
+      uses: eWaterCycle/setup-apptainer@main
+
+    - name: Set up Singularity
+      if: contains(inputs.profile, 'singularity')
+      shell: bash
+      run: |
+        mkdir -p $NXF_SINGULARITY_CACHEDIR
+        mkdir -p $NXF_SINGULARITY_LIBRARYDIR
+
+    - name: Conda setup
+      if: ${{inputs.profile == 'conda'}}
+      uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3
+      with:
+        auto-update-conda: true
+        conda-solver: libmamba
+        conda-remove-defaults: true
+
+    - name: Run nf-test
+      shell: bash
+      run: |
+        NFT_WORKDIR=~ \
+        nf-test test \
+          --ci \
+          --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \
+          --changed-since HEAD^ \
+          --profile=${{ inputs.profile }} \
+          --filter ${{ inputs.filters }} \
+          --tap=test.tap \
+          --verbose \
+          --tag ${{ inputs.tags }}
+
+      # TODO If no test.tap, then make one to spoof?
+    - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1
+      if: ${{ inputs.path != '' }}
+      with:
+        path: >-
+          test.tap
+
+    - name: Clean up
+      if: always()
+      shell: bash
+      run: |
+        sudo rm -rf /home/ubuntu/tests/*

.github/workflows/nf-test-gpu.yml

@@ -0,0 +1,113 @@
+name: Run GPU nf-tests
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      runners:
+        description: "Runners to test on"
+        type: string
+        default: "gpu"
+
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver
+  NFT_VER: "0.9.2"
+  NXF_ANSI_LOG: false
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
+  NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
+  # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver
+  NXF_VER: "24.10.2"
+
+jobs:
+  get-shards:
+    runs-on: ubuntu-latest
+    name: "Get Shards"
+    outputs:
+      shard: ${{ steps.set-shards.outputs.shard }}
+      total_shards: ${{ steps.set-shards.outputs.total_shards }}
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        with:
+          fetch-depth: 0
+
+      - name: run nf-test-shard action
+        id: set-shards
+        uses: ./.github/actions/nf-test-shard
+        env:
+          NFT_VER: ${{ env.NFT_VER }}
+        with:
+          tags: "gpu"
+          max_shards: 2
+
+      - name: Set outputs
+        id: set-outputs
+        run: |
+          echo "shard=${{ steps.set-shards.outputs.shard }}" >> $GITHUB_ENV
+          echo "total_shards=${{ steps.set-shards.outputs.total_shards }}" >> $GITHUB_ENV
+
+  nf-test-gpu:
+    needs: [get-shards]
+    runs-on: "gpu"
+    name: "GPU Test | ${{ matrix.profile }} | ${{ matrix.shard }} | ${{ matrix.NXF_VER }} | ${{ matrix.filters }}"
+    env:
+      NXF_VER: ${{ matrix.NXF_VER }}
+
+    # Only run on push if this is the nf-core dev branch (merged PRs)
+    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}"
+    strategy:
+      fail-fast: false
+      matrix:
+        profile: [docker, singularity]
+        shard: ${{ fromJson(needs.get-shards.outputs.shard) }}
+        NXF_VER:
+          - "24.10.2"
+        filters: [pipeline]
+
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        with:
+          fetch-depth: 0
+
+      - name: Run nf-test Action
+        uses: ./.github/actions/nf-test
+        with:
+          profile: ${{ matrix.profile }},gpu
+          shard: ${{ matrix.shard }}
+          total_shards: ${{ needs.get-shards.outputs.total_shards }}
+          filters: ${{ matrix.filters }}
+          tags: "gpu"
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [nf-test-gpu]
+    if: always()
+    steps:
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: One or more tests cancelled
+        if: ${{ contains(needs.*.result, 'cancelled') }}
+        run: exit 1
+
+      - name: All tests ok
+        if: ${{ contains(needs.*.result, 'success') }}
+        run: exit 0
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"