downstream-tests-on-comment.yml

name: downstream-tests-on-comment

# This workflow is triggered by the comment '/test-downstream' inside a PR
# The workflow then:
# 1. modifies the original comment into:
#   """
#   /test-downstream
# 
#   Automatic edit:
#   Running downstream tests for commit <commit>, follow the run [here](<link>).
#   """
# 2. runs the unit test of all common modules with the last commit before that comment
# 3. replies in a comment with which modules/ OSes failed (successes are omitted).

on:
  issue_comment:
    types:
    - created

jobs:
  pr-info:
    if: ${{ github.event.issue.pull_request && github.event.comment.body == '/test-downstream'}}
    runs-on: ubuntu-latest
    
    outputs:
      commit:     ${{ steps.get-commit.outputs.commit }}

    steps:
    
    - name: Get PR info
      id: get-pr-info
      uses: actions/github-script@v5
      with:
        script: |
          const { owner, repo, number } = context.issue;
          const pr = await github.rest.pulls.get({
            owner,
            repo,
            pull_number: number,
          });
          // console.log(pr) // for debugging
          return pr
      
    - name: Print metadata and set PR SHA
      id: get-commit
      env:
        pr_info: ${{ steps.get-pr-info.outputs.result }}
      run: |
        # this step is mainly there for debugging
        base=$(echo ${pr_info} | jq '.data["base"]["label"]')
        head=$(echo ${pr_info} | jq '.data["head"]["label"]')
        commit=$(echo ${pr_info} | jq -r '.data["head"]["sha"]')
        
        printf '%-15s: %s\n' \
            repository ${{ github.event.repository.name }}      \
            base       ${base}                                  \
            head       ${head}                                  \
            commit     ${commit}                                \
            github.sha ${{ github.sha }}                        \
            comment    ${{ github.event.comment.html_url }}     \
               
        echo "::set-output name=commit::${commit}"
              
      shell: bash

    - name: Update initial comment
      uses: actions/github-script@v5
      with:
        script: |
          body = "${{ github.event.comment.body }}".concat("\n\nAutomatic edit:\nRunning downstream tests for commit ${{ steps.get-commit.outputs.commit }}, follow the run [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}).");
          console.log(body);
          github.rest.issues.updateComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: ${{ github.event.comment.id }},
              body: body
            });


  unit-tests:
    runs-on: ${{ matrix.os }}
    # this might not be necessary, but it's more user friendly if the comment is edited first (otherwise pr-info and unit-tests run concurrently).
    needs: pr-info
    continue-on-error: true
    
    name: ${{ matrix.os }} (${{ matrix.r }} (${{ matrix.module }})

    strategy:
      fail-fast: false
      matrix:
        r: [4.3.1]
        os: [ubuntu-latest, macOS-latest, windows-latest]
        # only common modules
        module: [jasp-stats/jaspDescriptives, jasp-stats/jaspTTests, jasp-stats/jaspAnova, jasp-stats/jaspRegression, jasp-stats/jaspFrequencies, jasp-stats/jaspFactor]
        # for debugging
        # os: [ubuntu-latest]
        # module: [jasp-stats/jaspDescriptives, jasp-stats/jaspTTests]

#    # outputs is generated with the following R code. It does not matter if some outputs are unused.
#     oses      <- c("ubuntu-latest", "macOS-latest", "windows-latest")
#     modules   <- c("jasp-stats/jaspDescriptives", "jasp-stats/jaspTTests", "jasp-stats/jaspAnova", "jasp-stats/jaspRegression", "jasp-stats/jaspFrequencies", "jasp-stats/jaspFactor")

#     base <- "      %1$s-%2$s: ${{ steps.update-output.outputs.%1$s-%2$s }}"
#     for (os in oses) for (module in modules)
#       cat(sprintf(base, os, basename(module)), "\n")
    outputs:
      ubuntu-latest-jaspDescriptives: ${{ steps.update-output.outputs.ubuntu-latest-jaspDescriptives }} 
      ubuntu-latest-jaspTTests: ${{ steps.update-output.outputs.ubuntu-latest-jaspTTests }} 
      ubuntu-latest-jaspAnova: ${{ steps.update-output.outputs.ubuntu-latest-jaspAnova }} 
      ubuntu-latest-jaspRegression: ${{ steps.update-output.outputs.ubuntu-latest-jaspRegression }} 
      ubuntu-latest-jaspFrequencies: ${{ steps.update-output.outputs.ubuntu-latest-jaspFrequencies }} 
      ubuntu-latest-jaspFactor: ${{ steps.update-output.outputs.ubuntu-latest-jaspFactor }} 
      macOS-latest-jaspDescriptives: ${{ steps.update-output.outputs.macOS-latest-jaspDescriptives }} 
      macOS-latest-jaspTTests: ${{ steps.update-output.outputs.macOS-latest-jaspTTests }} 
      macOS-latest-jaspAnova: ${{ steps.update-output.outputs.macOS-latest-jaspAnova }} 
      macOS-latest-jaspRegression: ${{ steps.update-output.outputs.macOS-latest-jaspRegression }} 
      macOS-latest-jaspFrequencies: ${{ steps.update-output.outputs.macOS-latest-jaspFrequencies }} 
      macOS-latest-jaspFactor: ${{ steps.update-output.outputs.macOS-latest-jaspFactor }} 
      windows-latest-jaspDescriptives: ${{ steps.update-output.outputs.windows-latest-jaspDescriptives }} 
      windows-latest-jaspTTests: ${{ steps.update-output.outputs.windows-latest-jaspTTests }} 
      windows-latest-jaspAnova: ${{ steps.update-output.outputs.windows-latest-jaspAnova }} 
      windows-latest-jaspRegression: ${{ steps.update-output.outputs.windows-latest-jaspRegression }} 
      windows-latest-jaspFrequencies: ${{ steps.update-output.outputs.windows-latest-jaspFrequencies }} 
      windows-latest-jaspFactor: ${{ steps.update-output.outputs.windows-latest-jaspFactor }} 

    env:
      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}

    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ needs.pr-info.outputs.commit }}

      - uses: r-lib/actions/setup-r@v2
        with:
         r-version: ${{ matrix.r }}

      - name: clone module ${{ matrix.module }}
        run: git clone "https://github.com/${{ matrix.module }}.git"
        shell: bash

      - uses: jasp-stats/jasp-actions/setup-test-env@master

      - name: Install igraph from source
        if: ${{ matrix.os  == 'ubuntu-latest' && matrix.module == 'jasp-stats/jaspFactor' }}
        run: install.packages("igraph", type = "source", repos = "https://cloud.r-project.org")
        shell: Rscript {0}
       
      - name: Install ${{ github.event.repository.name }}
        run: remotes::install_local(".")
        shell: Rscript {0}

      - name: Run unit tests
        run: |
          setwd(basename("${{ matrix.module }}"))
          source("tests/testthat.R")
        shell: Rscript {0}

#       - name: fake unit tests for debugging
#         run: |
#           if [ ${{ matrix.module }} == 'jasp-stats/jaspDescriptives' ]; then
#             exit 1
#           fi
#         shell: bash
       
      - name: set error if tests fail
        id: update-output
        if: ${{ failure() }}
        run: |
          moduleName=$(basename ${{ matrix.module }})
          nm="${{ matrix.os }}-${moduleName}"
          echo "nm = $nm"
          echo "::set-output name=${nm}::error"
        shell: bash


  report-results:
    runs-on: ubuntu-latest
    needs: [pr-info, unit-tests]
    
    steps:

      - name: Aggregate results & post comment output
        id: aggregated-results
        env:
          NEEDS: ${{ toJSON(needs) }}
        uses: actions/github-script@v5
        with:
          script: |

            const { NEEDS } = process.env
            console.log(`${NEEDS}`)

            results=JSON.parse(NEEDS);
            console.log(results)
            
            outputs=results["unit-tests"]["outputs"];
            for (const prop in outputs) { console.log(`${prop}: ${outputs[prop]}`);}
            
            if (Object.keys(outputs).length === 0) {
              console.log("All downstream tests succeeded")
              body = "All downstream tests for commit ${{ needs.pr-info.outputs.commit }} succeeded! ✅"
            } else {
              console.log("Some unit tests failed")
            
              obj = {}; // dict from module to os
              for (const key in outputs) {
                var [os, module] = key.split(/\-(?=[^\-]+$)/);
                console.log(os, "|", module)
                if (!(module in obj))
                  obj[module] = new Set();
                obj[module].add(os);
              }
              
              body = "The following downstream tests for commit ${{ needs.pr-info.outputs.commit }} failed:\n| Module | windows-latest | macOS-latest | ubuntu-latest |\n| ---: | :---: | :---: | :---: |"
              oses = ["windows-latest", "macOS-latest", "ubuntu-latest"];
              for (const module in obj) {
                str = "\n| ".concat(module, " | ", oses.map(os => obj[module].has(os) ? "❌" : "✅").join(" | "));
                body = body.concat(str);
                console.log(body)
              }
            }
            
            body = body.concat("\n\nThe full log is available [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}).")
            
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: body,
            });