Merge branch 'develop' of github.com:mskcc-omics-workflows/modules in…

…to bugfix/netmhc_feature_names

.github/PULL_REQUEST_TEMPLATE.md

@@ -27,7 +27,7 @@ Closes #XXX <!-- If this PR fixes an issue, please link it here! -->
 - [ ] Follow the parameters requirements.
 - [ ] Follow the input/output options guidelines.
 - [ ] Add a resource `label`.
-- [ ] Use Jfrog if possible to fulfil software requirements.
+- [ ] Use Jfrog if possible to fulfill software requirements.
 - Ensure that the test works with either Docker / Singularity. Conda CI tests can be quite flaky:
   - For modules:
     - [ ] `nf-core modules --git-remote https://github.com/mskcc-omics-workflows/modules.git -b <module_branch> test <MODULE> --profile docker`

.github/actions/nf-test-action/action.yml

@@ -0,0 +1,97 @@
+name: "nf-test Action"
+description: "Runs nf-test with common setup steps"
+inputs:
+  profile:
+    description: "Profile to use"
+    required: true
+  shard:
+    description: "Shard number for this CI job"
+    required: true
+  total_shards:
+    description: "Total number of test shards(NOT the total number of matrix jobs)"
+    required: true
+  paths:
+    description: "Test paths"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4
+      with:
+        distribution: "temurin"
+        java-version: "17"
+
+    - name: Setup Nextflow
+      uses: nf-core/setup-nextflow@v2
+      with:
+        version: "${{ env.NXF_VERSION }}"
+
+    - name: Set up Python
+      uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
+      with:
+        python-version: "3.11"
+
+    - name: Install nf-test
+      uses: nf-core/setup-nf-test@v1
+      with:
+        version: "${{ env.NFT_VER }}"
+        install-pdiff: true
+
+    - name: Setup apptainer
+      if: contains(inputs.profile, 'singularity')
+      uses: eWaterCycle/setup-apptainer@main
+
+    - name: Set up Singularity
+      if: contains(inputs.profile, 'singularity')
+      shell: bash
+      run: |
+        mkdir -p $NXF_SINGULARITY_CACHEDIR
+        mkdir -p $NXF_SINGULARITY_LIBRARYDIR
+
+    - name: Conda setup
+      if: ${{inputs.profile == 'conda'}}
+      uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3
+      with:
+        auto-update-conda: true
+        conda-solver: libmamba
+        conda-remove-defaults: true
+
+    # Set up secrets
+    - name: Set up nextflow secrets
+      if: env.SENTIEON_ENCRYPTION_KEY != '' && env.SENTIEON_LICENSE_MESSAGE != ''
+      shell: bash
+      run: |
+        python -m pip install cryptography
+        nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
+
+      # TODO Skip failing conda tests and document their failures
+      # https://github.com/nf-core/modules/issues/7017
+    - name: Run nf-test
+      shell: bash
+      env:
+        SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }}
+        SENTIEON_AUTH_MECH: "GitHub Actions - token"
+      run: |
+        NFT_WORKDIR=~ \
+        nf-test test \
+          --profile=${{ inputs.profile }} \
+          --tap=test.tap \
+          --verbose \
+          --ci \
+          --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \
+          --filter process,workflow \
+          ${{ inputs.paths }}
+
+      # TODO If no test.tap, then make one to spoof?
+    - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1
+      if: ${{ inputs.path != '' }}
+      with:
+        path: >-
+          test.tap
+
+    - name: Clean up
+      if: always()
+      shell: bash
+      run: |
+        sudo rm -rf /home/ubuntu/tests/

.github/conda_skip.yml

@@ -0,0 +1,74 @@
+exclude:
+  - path: modules/msk/nf-test
+  - profile: conda
+    path: modules/msk/gbcms
+  - profile: conda
+    path: modules/msk/polysolver
+  - profile: conda
+    path: modules/msk/snppileup
+  - profile: conda
+    path: modules/msk/facets
+  - profile: conda
+    path: modules/msk/mutect1
+  - profile: conda
+    path: modules/msk/phylowgs/createinput
+  - profile: conda
+    path: modules/msk/phylowgs/parsecnvs
+  - profile: conda
+    path: modules/msk/phylowgs/multievolve
+  - profile: conda
+    path: modules/msk/phylowgs/writeresults
+  - profile: conda
+    path: modules/msk/netmhcpan4
+  - profile: conda
+    path: modules/msk/neoantigeninput
+  - profile: conda
+    path: modules/msk/neoantigenediting/aligntoIEDB
+  - profile: conda
+    path: modules/msk/neoantigenediting/computeFitness
+  - profile: conda
+    path: modules/msk/subworkflows/neoantigen_editing
+  - profile: conda
+    path: modules/msk/subworkflows/phylowgs
+  - profile: conda
+    path: modules/msk/netmhcstabpan
+  - profile: conda
+    path: modules/msk/ppflagfixer
+  - profile: conda
+    path: modules/msk/neoantigenutils/generatemutfasta
+  - profile: conda
+    path: modules/msk/neoantigenutils/formatnetmhcpan
+  - profile: conda
+    path: modules/msk/neoantigenutils/generatehlastring
+  - profile: conda
+    path: modules/msk/neoantigenutils/neoantigeninput
+  - profile: conda
+    path: modules/msk/subworkflows/netmhcstabandpan
+  - profile: conda
+    path: modules/msk/salmon/index
+  - profile: conda
+    path: modules/msk/genomenexus/vcf2maf
+  - profile: conda
+    path: modules/msk/genomenexus/annotationpipeline
+  - profile: conda
+    path: modules/msk/ppflagfixer
+  - profile: conda
+    path: modules/msk/pvmaf/tagtraceback
+  - profile: conda
+    path: modules/msk/pvmaf/concat
+  - profile: conda
+    path: modules/msk/genotypevariants/all
+  - profile: conda
+    path: modules/msk/subworkflows/traceback
+  - profile: conda
+    path: modules/msk/salmon/index
+  - profile: conda
+    path: modules/msk/subworkflows/genome_nexus
+  - profile: conda
+    path: modules/msk/salmon/quant
+  - profile: conda
+    path: modules/msk/rediscoverte
+  - profile: conda
+    path: modules/msk/neosv
+  - profile: conda
+    path: modules/msk/netmhc3

.github/workflows/nf-test.yml

@@ -0,0 +1,121 @@
+name: Run nf-test
+on:
+  push:
+    branches:
+      # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging
+      - "renovate/**" # branches Renovate creates
+  pull_request:
+    branches: [master, develop]
+  merge_group:
+    types: [checks_requested]
+    branches: [master, develop]
+  workflow_dispatch: {}
+
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver
+  NFT_VER: "0.9.2"
+  NXF_ANSI_LOG: false
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
+  NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
+  # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver
+  NXF_VER: "24.10.2"
+
+jobs:
+  nf-test-changes:
+    name: nf-test-changes
+    runs-on: ubuntu-latest
+    outputs:
+      # Expose detected tags as 'modules' and 'workflows' output variables
+      paths: ${{ steps.list.outputs.components }}
+      modules: ${{ steps.outputs.outputs.modules }}
+      subworkflows: ${{ steps.outputs.outputs.subworkflows}}
+      # Prod for version bumping
+    steps:
+      - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
+        run: |
+          ls -la ./
+          rm -rf ./* || true
+          rm -rf ./.??* || true
+          ls -la ./
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          fetch-depth: 0
+
+      - name: List nf-test files
+        id: list
+        uses: adamrtalbot/detect-nf-test-changes@de3c3c8e113031b4f15a3c1104b5f135e8346997 # v0.0.6
+        with:
+          head: ${{ github.sha }}
+          base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
+          n_parents: 0
+          exclude_tags: "gpu"
+
+      - name: Separate modules and subworkflows
+        id: outputs
+        run: |
+          echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT
+          echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
+      - name: debug
+        run: |
+          echo ${{ steps.list.outputs.components }}
+          echo ${{ steps.outputs.outputs.modules }}
+          echo ${{ steps.outputs.outputs.subworkflows }}
+  nf-test:
+    runs-on: ubuntu-latest
+    name: "Test | ${{ matrix.profile }} | ${{ matrix.shard }}"
+    needs: nf-test-changes
+    if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4, 5]
+        profile: [docker, singularity]
+    env:
+      NXF_ANSI_LOG: false
+      TOTAL_SHARDS: 5
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          fetch-depth: 0
+      - name: Run nf-test Action
+        uses: ./.github/actions/nf-test-action
+        env:
+          SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
+          SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
+          SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }}
+          SENTIEON_AUTH_MECH: "GitHub Actions - token"
+        with:
+          profile: ${{ matrix.profile }}
+          shard: ${{ matrix.shard }}
+          total_shards: ${{ env.TOTAL_SHARDS }}
+          paths: "${{ join(fromJson(needs.nf-test-changes.outputs.paths), ' ') }}"
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [nf-test]
+    if: always()
+    steps:
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: One or more tests cancelled
+        if: ${{ contains(needs.*.result, 'cancelled') }}
+        run: exit 1
+
+      - name: All tests ok
+        if: ${{ contains(needs.*.result, 'success') }}
+        run: exit 0
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"