From ac2394401e5ae08d27a145d2920207c74ef00944 Mon Sep 17 00:00:00 2001 From: Jingwei Wang <48091236+sky1122@users.noreply.github.com> Date: Thu, 23 Jan 2025 18:20:24 -0600 Subject: [PATCH] ci: ci only merged to main will upload the testing report (#1244) ci: only merged to main will upload the testing report --------- Signed-off-by: Jingwei Wang --- .github/workflows/e2e-linux.yaml | 99 +++++++++++++++++++++++++++++- .github/workflows/e2e-macos.yaml | 79 +++++++++++++++++++++++- .github/workflows/e2e-windows.yaml | 86 ++++++++++++++++++++++++++ Makefile | 20 ++++-- 4 files changed, 276 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e-linux.yaml b/.github/workflows/e2e-linux.yaml index ee3d3c1ff..d549bc248 100644 --- a/.github/workflows/e2e-linux.yaml +++ b/.github/workflows/e2e-linux.yaml @@ -32,6 +32,11 @@ jobs: "${{ inputs.version }}", "${{ inputs.runner-type }}", ] + outputs: + vars: ${{ steps.vars.outputs.has_creds}} + vm_report: ${{ steps.set-multiple-vars.outputs.VM_REPORT }} + container_report: ${{ steps.set-multiple-vars.outputs.CONTAINER_REPORT }} + vm_serial_report: ${{ steps.set-multiple-vars.outputs.VM_SERIAL_REPORT }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # checkout uses node version 20, which doesn't run on AL2 @@ -60,7 +65,7 @@ jobs: - name: configure aws credentials uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 # this action requires node20, skip on AL2 - if: ${{ steps.vars.outputs.has_creds == true && (!(startsWith(inputs.os, 'amazon') && inputs.version == '2' ))}} + if: ${{ steps.vars.outputs.has_creds == 'true' && (!(startsWith(inputs.os, 'amazon') && inputs.version == '2' ))}} with: role-to-assume: ${{ secrets.ROLE }} role-session-name: credhelper-test @@ -90,6 +95,9 @@ jobs: sudo systemctl start finch.service sudo systemctl start finch-buildkit.service sudo systemctl start finch-soci.service + - name: Set up REPORT_DIR + run: | + echo "REPORT_DIR=${{ github.workspace }}/reports" >> $GITHUB_ENV - name: Run e2e tests run: | git status @@ -98,7 +106,96 @@ jobs: eval "$(ssh-agent -s)" INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} sudo -E make test-e2e-container INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} sudo -E make test-e2e-vm + - name: Change ownership of reports + if: always() + run: | + if [ ! -d "$REPORT_DIR" ]; then + echo "Error: Directory $REPORT_DIR does not exist." + exit 1 + fi + + USER=$(whoami) + GROUP=$(id -gn) + + if sudo chown -R "$USER:$GROUP" "$REPORT_DIR"; then + echo "Ownership of $REPORT_DIR changed to $USER:$GROUP" + else + echo "Error: Failed to change ownership of $REPORT_DIR" + exit 1 + fi + - name: Set artifacts name outputs + if: ${{ steps.vars.outputs.has_creds == 'true' && (!(startsWith(inputs.os, 'amazon') && inputs.version == '2' ))}} + id: set-multiple-vars + run: | + echo "VM_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" >> $GITHUB_OUTPUT + echo "CONTAINER_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" >> $GITHUB_OUTPUT + echo "VM_SERIAL_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-serial-report.json" >> $GITHUB_OUTPUT + - name: Upload reports artifact + if: ${{ steps.vars.outputs.has_creds == 'true' && (!(startsWith(inputs.os, 'amazon') && inputs.version == '2' ))}} + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: linux-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + path: ${{ github.workspace }}/reports/${{ github.run_id }}-${{ github.run_attempt }}-*.json - name: Clean up repo AL2 if: ${{ (startsWith(inputs.os, 'amazon') && inputs.version == '2' && always() ) }} run: | rm -rf "${GITHUB_WORKSPACE}" + upload-linux-e2e-test-report: + needs: test + if: always() + runs-on: + [ + "self-hosted", + "linux", + "${{ inputs.arch }}", + "${{ inputs.version }}", + "${{ inputs.runner-type }}", + ] + steps: + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + if: (!(startsWith(inputs.os, 'amazon') && inputs.version == '2' )) && github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + with: + role-to-assume: ${{ secrets.ROLE }} + role-session-name: credhelper-test + aws-region: ${{ secrets.REGION }} + - name: download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.08 + if: (!(startsWith(inputs.os, 'amazon') && inputs.version == '2' )) && github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + with: + name: linux-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + path: linux-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + - name: Display structure of downloaded files + if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + run: ls -R linux-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + - name: Check and upload e2e tests reports to s3 bucket + if: (!(startsWith(inputs.os, 'amazon') && inputs.version == '2' )) && github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + run: | + VM_REPORT="${{ github.workspace }}/linux-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports/${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" + CONTAINER_REPORT="${{ github.workspace }}/linux-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports/${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" + VM_SERIAL_REPORT="${{ github.workspace }}/linux-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports/${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-serial-report.json" + + if [ -f "$VM_REPORT" ]; then + echo "VM report file exists. Uploading to S3..." + aws s3 cp "$VM_REPORT" "s3://finch-e2e-test-log-reports/linux-${{ inputs.arch }}/${{ needs.test.outputs.vm_report }}" + echo "VM report uploaded successfully." + else + echo "VM report file does not exist. Skipping upload." + fi + + if [ -f "$CONTAINER_REPORT" ]; then + echo "Container report file exists. Uploading to S3..." + aws s3 cp "$CONTAINER_REPORT" "s3://finch-e2e-test-log-reports/linux-${{ inputs.arch }}/${{ needs.test.outputs.container_report }}" + echo "Container report uploaded successfully." + else + echo "Container report file does not exist. Skipping upload." + fi + + if [ -f "$VM_SERIAL_REPORT" ]; then + echo "VM serial report file exists. Uploading to S3..." + aws s3 cp "$VM_SERIAL_REPORT" "s3://finch-e2e-test-log-reports/linux-${{ inputs.arch }}/${{ needs.test.outputs.vm_serial_report }}" + echo "VM serial report uploaded successfully." + else + echo "VM serial report file does not exist. Skipping upload." + fi + \ No newline at end of file diff --git a/.github/workflows/e2e-macos.yaml b/.github/workflows/e2e-macos.yaml index 51c875af9..9582b3515 100644 --- a/.github/workflows/e2e-macos.yaml +++ b/.github/workflows/e2e-macos.yaml @@ -35,6 +35,11 @@ jobs: "${{ inputs.version }}", "${{ inputs.runner-type }}", ] + outputs: + vars: ${{ steps.vars.outputs.has_creds}} + vm_report: ${{ steps.set-multiple-vars.outputs.VM_REPORT }} + container_report: ${{ steps.set-multiple-vars.outputs.CONTAINER_REPORT }} + vm_serial_report: ${{ steps.set-multiple-vars.outputs.VM_SERIAL_REPORT }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -53,7 +58,7 @@ jobs: echo "has_creds=$has_creds" >> $GITHUB_OUTPUT - name: configure aws credentials uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 - if: ${{ steps.vars.outputs.has_creds == true }} + if: ${{ steps.vars.outputs.has_creds == 'true' }} with: role-to-assume: ${{ secrets.ROLE }} role-session-name: credhelper-test @@ -84,3 +89,75 @@ jobs: git clean -f -d REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} make ${{ inputs.test-command }} shell: zsh {0} + - name: Set artifacts name outputs + if: always() + id: set-multiple-vars + run: | + echo "VM_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" >> $GITHUB_OUTPUT + echo "CONTAINER_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" >> $GITHUB_OUTPUT + echo "VM_SERIAL_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-serial-report.json" >> $GITHUB_OUTPUT + - name: Upload reports artifact + if: always() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + path: ${{ github.workspace }}/reports/${{ github.run_id }}-${{ github.run_attempt }}-*.json + upload-macos-e2e-test-report: + needs: test + if: always() + runs-on: + [ + "self-hosted", + "macos", + "${{ inputs.arch }}", + "${{ inputs.version }}", + "${{ inputs.runner-type }}", + ] + steps: + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + if: ${{ needs.test.outputs.vars == 'true' }} && github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + with: + role-to-assume: ${{ secrets.ROLE }} + role-session-name: credhelper-test + aws-region: ${{ secrets.REGION }} + - name: download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.08 + if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + with: + name: macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + path: macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + - name: Display structure of downloaded files + if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + run: ls -R macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + - name: Check and upload e2e tests reports to s3 bucket + if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + run: | + VM_REPORT="${{ github.workspace }}/macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports/${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" + CONTAINER_REPORT="${{ github.workspace }}/macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports/${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" + VM_SERIAL_REPORT="${{ github.workspace }}/macos-${{ inputs.version }}-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports/${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-serial-report.json" + + if [ -f "$VM_REPORT" ]; then + echo "VM report file exists. Uploading to S3..." + aws s3 cp "$VM_REPORT" "s3://finch-e2e-test-log-reports/macos-${{ inputs.arch }}/${{ needs.test.outputs.vm_report }}" + echo "VM report uploaded successfully." + else + echo "VM report file does not exist. Skipping upload." + fi + + if [ -f "$CONTAINER_REPORT" ]; then + echo "Container report file exists. Uploading to S3..." + aws s3 cp "$CONTAINER_REPORT" "s3://finch-e2e-test-log-reports/macos-${{ inputs.arch }}/${{ needs.test.outputs.container_report }}" + echo "Container report uploaded successfully." + else + echo "Container report file does not exist. Skipping upload." + fi + + if [ -f "$VM_SERIAL_REPORT" ]; then + echo "VM serial report file exists. Uploading to S3..." + aws s3 cp "$VM_SERIAL_REPORT" "s3://finch-e2e-test-log-reports/macos-${{ inputs.arch }}/${{ needs.test.outputs.vm_serial_report }}" + echo "VM serial report uploaded successfully." + else + echo "VM serial report file does not exist. Skipping upload." + fi + \ No newline at end of file diff --git a/.github/workflows/e2e-windows.yaml b/.github/workflows/e2e-windows.yaml index 59c62cfb5..770e530a3 100644 --- a/.github/workflows/e2e-windows.yaml +++ b/.github/workflows/e2e-windows.yaml @@ -88,7 +88,21 @@ jobs: git status git clean -f -d + make ${{ inputs.test-command }} + - name: Set artifacts name outputs + if: always() + id: set-multiple-vars + run: | + "VM_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" >> $env:GITHUB_OUTPUT + "CONTAINER_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" >> $env:GITHUB_OUTPUT + "VM_SERIAL_REPORT=${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-serial-report.json" >> $env:GITHUB_OUTPUT + - name: Upload reports artifact + if: always() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: windows-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + path: ${{ github.workspace }}/reports/${{ github.run_id }}-${{ github.run_attempt }}-*.json - name: Remove Finch VM and Clean Up Previous Environment if: ${{ always() }} timeout-minutes: 2 @@ -97,3 +111,75 @@ jobs: make clean cd deps/finch-core && make clean exit 0 # Cleanup may set the exit code e.g. if a file doesn't exist; just ignore + upload-windows-e2e-test-report: + needs: test + if: always() + timeout-minutes: 180 + runs-on: + [ + "self-hosted", + "windows", + "${{ inputs.arch }}", + "${{ inputs.runner-type }}", + ] + steps: + - name: Set output variables + id: vars + run: | + $has_creds="${{ (github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name) && github.actor != 'dependabot[bot]'}}" + echo "has_creds=$has_creds" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + exit 0 # if $has_creds is false, powershell will exit with code 1 and this step will fail + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + if: | + env.has_creds == 'true' && + github.event_name == 'pull_request' && + github.event.action == 'closed' && + github.event.pull_request.merged == true + with: + role-to-assume: ${{ secrets.ROLE }} + role-session-name: credhelper-test + aws-region: ${{ secrets.REGION }} + - name: download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.08 + if: | + github.event_name == 'pull_request' && + github.event.action == 'closed' && + github.event.pull_request.merged == true + with: + name: windows-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + path: windows-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports + - name: Check and upload e2e tests reports to s3 bucket + if: | + github.event_name == 'pull_request' && + github.event.action == 'closed' && + github.event.pull_request.merged == true + run: | + $env:VM_REPORT = "${{ github.workspace }}\windows-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports\${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" + $env:CONTAINER_REPORT = "${{ github.workspace }}\windows-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports\${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" + $env:VM_SERIAL_REPORT = "${{ github.workspace }}\windows-${{ inputs.test-command }}-${{ inputs.arch }}-${{ github.run_id }}-${{ github.run_attempt }}-e2e-reports\${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-serial-report.json" + + if (Test-Path $env:VM_REPORT) { + Write-Host "VM report file exists. Uploading to S3..." + aws s3 cp $env:VM_REPORT "s3://finch-e2e-test-log-reports/windows-${{ inputs.arch }}/${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-report.json" + Write-Host "VM report uploaded successfully." + } else { + Write-Host "VM report file does not exist. Skipping upload." + } + + if (Test-Path $env:CONTAINER_REPORT) { + Write-Host "Container report file exists. Uploading to S3..." + aws s3 cp $env:CONTAINER_REPORT "s3://finch-e2e-test-log-reports/windows-${{ inputs.arch }}/${{ github.run_id }}-${{ github.run_attempt }}-e2e-container-report.json" + Write-Host "Container report uploaded successfully." + } else { + Write-Host "Container report file does not exist. Skipping upload." + } + + if (Test-Path $env:VM_SERIAL_REPORT) { + Write-Host "VM serial report file exists. Uploading to S3..." + aws s3 cp $env:VM_SERIAL_REPORT "s3://finch-e2e-test-log-reports/windows-${{ inputs.arch }}/${{ github.run_id }}-${{ github.run_attempt }}-e2e-vm-serial-report.json" + Write-Host "VM serial report uploaded successfully." + } else { + Write-Host "VM serial report file does not exist. Skipping upload." + } + \ No newline at end of file diff --git a/Makefile b/Makefile index 165828311..a0b0803ba 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,9 @@ DEST := $(shell echo "$(DESTDIR)/$(PREFIX)" | sed 's:///*:/:g; s://*$$::') BINDIR ?= /usr/local/bin OUTDIR ?= $(CURDIR)/_output OS_OUTDIR ?= $(OUTDIR)/os +REPORT_DIR ?= $(CURDIR)/reports +RUN_ID ?= $(GITHUB_RUN_ID) +RUN_ATTEMPT ?= $(GITHUB_RUN_ATTEMPT) OUTPUT_DIRECTORIES := $(OUTDIR) $(OS_OUTDIR) $(OUTPUT_DIRECTORIES): @@ -286,20 +289,25 @@ test-unit: # # Container tests and VM tests can be run in any order, but they must be run sequentially. # For more details, see the package-level comment of the e2e package. + +.PHONY: create-report-dir +create-report-dir: + mkdir -p $(REPORT_DIR) + .PHONY: test-e2e test-e2e: test-e2e-vm-serial test-e2e-container .PHONY: test-e2e-vm-serial -test-e2e-vm-serial: - go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 --installed="$(INSTALLED)" +test-e2e-vm-serial: create-report-dir + go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-serial-report.json --installed="$(INSTALLED)" .PHONY: test-e2e-container -test-e2e-container: - go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/container -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 --installed="$(INSTALLED)" +test-e2e-container: create-report-dir + go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/container -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-container-report.json --installed="$(INSTALLED)" .PHONY: test-e2e-vm -test-e2e-vm: - go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 --installed="$(INSTALLED)" --registry="$(REGISTRY)" +test-e2e-vm: create-report-dir + go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-report.json --installed="$(INSTALLED)" --registry="$(REGISTRY)" .PHONY: test-benchmark test-benchmark: