.github/workflows/test.yml: Improve test workflow

.github/workflows/ci.yml

@@ -6,6 +6,9 @@ on:
       - 'v*'
       - '!v*-rc*'
 
+concurrency:
+  group: ${{ github.workflow }}
+
 jobs:
   build:
     uses: ./.github/workflows/build.yml

.github/workflows/develop.yml

@@ -5,6 +5,9 @@ on:
     tags:
       - 'v*-rc*'
 
+concurrency:
+  group: ${{ github.workflow }}
+
 jobs:
   build:
     uses: ./.github/workflows/build.yml

.github/workflows/test.yml

@@ -3,57 +3,114 @@ name: Run DTS tests
 on:
   pull_request:
     branches:
-      - 'main'
+      - "main"
+
+concurrency:
+  group: ${{ github.workflow }}
+
 jobs:
   build-dts:
     uses: ./.github/workflows/build.yml
     with:
       cacheless: false
+
+  checkout-osfv:
+    runs-on: dts-builder
+    steps:
+      - name: Checkout OSFV repo
+        uses: actions/checkout@v4
+        with:
+          repository: "Dasharo/open-source-firmware-validation"
+          path: "open-source-firmware-validation"
+          submodules: "recursive"
+          ref: "develop"
+
   run-tests:
     name: Run DTS tests
-    needs: build-dts
+    needs: [checkout-osfv, build-dts]
     runs-on:
       labels: dts-builder
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - test: msi_comm
+            dpp_download_key: $MSI_DOWNLOAD
+            dpp_logs_key: $MSI_LOGS
+            log_file: output_msi.log
+
+          - test: msi_dpp
+            dpp_download_key: $MSI_DOWNLOAD
+            dpp_logs_key: $MSI_LOGS
+            log_file: output_msi.log
+
+          - test: msi_heads
+            dpp_download_key: $MSI_HEADS_DOWNLOAD
+            dpp_logs_key: $MSI_HEADS_LOGS
+            log_file: output_msi_heads.log
+
+          - test: optiplex_dpp
+            dpp_download_key: $OPTIPLEX_DOWNLOAD
+            dpp_logs_key: $OPTIPLEX_LOGS
+            log_file: output_nc_heads.log
+
+          - test: novacustom_heads
+            dpp_download_key: $NOVACUSTOM_HEADS_DOWNLOAD
+            dpp_logs_key: $NOVACUSTOM_HEADS_LOGS
+            log_file: output_nc_heads.log
+
+          - test: novacustom_comm
+            dpp_download_key: $NOVACUSTOM_HEADS_DOWNLOAD
+            dpp_logs_key: $NOVACUSTOM_HEADS_LOGS
+            log_file: output_nc_comm.log
+
+          - test: pcengines_dpp
+            dpp_download_key: $PCENGINES_DOWNLOAD
+            dpp_logs_key: $PCENGINES_LOGS
+            log_file: output_pcengines.log
+
+          - test: pcengines_seabios
+            dpp_download_key: $PCENGINES_SEABIOS_DOWNLOAD
+            dpp_logs_key: $PCENGINES_LOGS
+            log_file: output_pcengines_seabios.log
+
     steps:
-      - name: Checkout OSFV repo
-        uses: actions/checkout@v4
-        with:
-          repository: 'Dasharo/open-source-firmware-validation'
-          path: 'open-source-firmware-validation'
-          submodules: 'recursive'
-          ref: 'develop'
       - name: Run IPXE server
         shell: bash
         run: |
-          cd open-source-firmware-validation
-          ./scripts/ci/ipxe-run.sh ../build/tmp/deploy/images/genericx86-64/dts-base-image-genericx86-64.cpio.gz ../build/tmp/deploy/images/genericx86-64/bzImage &
+          ./open-source-firmware-validation/scripts/ci/ipxe-run.sh \
+          build/tmp/deploy/images/genericx86-64/dts-base-image-genericx86-64.cpio.gz \
+          build/tmp/deploy/images/genericx86-64/bzImage &
+
       - name: Install requirements
         shell: bash
         run: |
           cd open-source-firmware-validation
-          python3 -m virtualenv venv
+          if [ ! -d "venv" ]; then
+            python3 -m venv venv
+          fi
           source venv/bin/activate
           pip install -r requirements.txt
+
       - name: Run QEMU
         shell: bash
         id: run_qemu
         run: |
           cd open-source-firmware-validation/scripts/ci
-          mkdir qemu-data
+          mkdir -p qemu-data
           touch qemu-data/hdd.qcow2
           ./qemu-run.sh nographic os &
+
       - name: Create directory for logs
         shell: bash
-        id: log_dirs
+        id: create_log_dirs
         run: |
-          timestamp=$(date -u +%Y-%m-%dT%H:%M:%S%Z)
-          directory="/tmp/dts-test-ci-${timestamp}"
-          mkdir $directory
-          echo "directory=$directory" >> "$GITHUB_OUTPUT"
-      - name: Run tests
+          LOG_DIR="/tmp/dts-test-ci-${{ github.run_id }}/${{ matrix.test }}"
+          mkdir -p $LOG_DIR
+          echo "LOG_DIR=$LOG_DIR" >> $GITHUB_ENV
+
+      - name: Run ${{ matrix.test }}
         shell: bash
-        env:
-          LOG_DIR: ${{ steps.log_dirs.outputs.directory }}
         run: |
           cd open-source-firmware-validation
           source venv/bin/activate
@@ -67,43 +124,28 @@ jobs:
           source ~/.secrets/dpp-keys
           ip_addr=$(ip -o -4 addr list eno2 | awk '{print $4}' | cut -d/ -f1)
 
-          robot -L TRACE -v config:qemu -v rte_ip:127.0.0.1 -v snipeit:no \
-          -v dpp_password:$DPP_PASSWORD -v dpp_download_key:$MSI_DOWNLOAD \
-          -v dpp_logs_key:$MSI_LOGS -v boot_dts_from_ipxe_shell:True \
-          -v dts_ipxe_link:http://${ip_addr}:4321/dts.ipxe \
-          -i "msi_comm" -i "msi_dpp" dts/dts-e2e.robot 2>&1 | tee $LOG_DIR/output_msi.log | grep "| PASS |\|| FAIL |"
-
-          robot -L TRACE -v config:qemu -v rte_ip:127.0.0.1 -v snipeit:no \
-          -v dpp_password:$DPP_PASSWORD -v dpp_download_key:$MSI_HEADS_DOWNLOAD \
-          -v dpp_logs_key:$MSI_HEADS_LOGS -v boot_dts_from_ipxe_shell:True \
-          -v dts_ipxe_link:http://${ip_addr}:4321/dts.ipxe \
-          -i "msi_heads" dts/dts-e2e.robot 2>&1 | tee $LOG_DIR/output_msi_heads.log | grep "| PASS |\|| FAIL |"
-
-          robot -L TRACE -v config:qemu -v rte_ip:127.0.0.1 -v snipeit:no \
-          -v dpp_password:$DPP_PASSWORD -v dpp_download_key:$OPTIPLEX_DOWNLOAD \
-          -v dpp_logs_key:$OPTIPLEX_LOGS -v boot_dts_from_ipxe_shell:True \
-          -v dts_ipxe_link:http://${ip_addr}:4321/dts.ipxe \
-          -i "optiplex_dpp" dts/dts-e2e.robot 2>&1 | tee $LOG_DIR/output_optiplex.log | grep "| PASS |\|| FAIL |"
-
-          robot -L TRACE -v config:qemu -v rte_ip:127.0.0.1 -v snipeit:no \
-          -v dpp_password:$DPP_PASSWORD -v dpp_download_key:$NOVACUSTOM_HEADS_DOWNLOAD \
-          -v dpp_logs_key:$NOVACUSTOM_HEADS_LOGS -v boot_dts_from_ipxe_shell:True \
-          -v dts_ipxe_link:http://${ip_addr}:4321/dts.ipxe \
-          -i "novacustom_heads" -i "novacustom_comm" dts/dts-e2e.robot 2>&1 | tee $LOG_DIR/output_nc_heads.log | grep "| PASS |\|| FAIL |"
+          echo "::add-mask::${{matrix.dpp_download_key}}"
+          echo "::add-mask::${{matrix.dpp_logs_key}}"
+          echo "::add-mask::$DPP_PASSWORD"
 
-          robot -L TRACE -v config:qemu -v rte_ip:127.0.0.1 -v snipeit:no \
-          -v dpp_password:$DPP_PASSWORD -v dpp_download_key:$PCENGINES_DOWNLOAD \
-          -v dpp_logs_key:$PCENGINES_LOGS -v boot_dts_from_ipxe_shell:True \
+          RTE_IP="127.0.0.1" CONFIG="qemu" ./scripts/run.sh dts/dts-e2e.robot -- \
+          -L TRACE \
+          -v snipeit:no \
+          -v dpp_password:$DPP_PASSWORD \
+          -v dpp_download_key:${{matrix.dpp_download_key}} \
+          -v dpp_logs_key:"${{matrix.dpp_logs_key}}" \
+          -v boot_dts_from_ipxe_shell:True \
           -v dts_ipxe_link:http://${ip_addr}:4321/dts.ipxe \
-          -i "pcengines_dpp" dts/dts-e2e.robot 2>&1 | tee $LOG_DIR/output_pcengines.log | grep "| PASS |\|| FAIL |"
+          -i ${{ matrix.test }} 2>&1 \
+          | tee ${{ env.LOG_DIR }}/${{ matrix.log_file }} | grep "| PASS |\|| FAIL |"
 
       - name: Copy log
         shell: bash
         if: ${{ !cancelled() }}
-        env:
-          LOG_DIR: ${{ steps.log_dirs.outputs.directory }}
         run: |
-          cp open-source-firmware-validation/log.html $LOG_DIR/log.html
+          cd open-source-firmware-validation
+          find logs -type f -name "*log.html" -exec mv {} "${{ env.LOG_DIR }}/${{ matrix.test }}_log.html" \;
+
   cleanup:
     name: Cleanup
     if: always()

.github/workflows/weekly.yml

@@ -5,6 +5,9 @@ on:
     - cron: '0 8 * * 6'
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}
+
 jobs:
   build:
     uses: ./.github/workflows/build.yml