From 7331ab3827f2a61ad8bf11b8305424be24a9d495 Mon Sep 17 00:00:00 2001 From: Maria Ines Parnisari Date: Thu, 2 Nov 2023 11:34:20 -0700 Subject: [PATCH 1/6] ci: run benchmarks on every pr and alert if perf is worse --- .github/workflows/master.yaml | 47 ++++++++++++++++++++++++ .github/workflows/on-pull-request.yml | 51 +++++++++++++++++++++++++++ Makefile | 4 +++ 3 files changed, 102 insertions(+) create mode 100644 .github/workflows/master.yaml diff --git a/.github/workflows/master.yaml b/.github/workflows/master.yaml new file mode 100644 index 00000000..3ff13eb3 --- /dev/null +++ b/.github/workflows/master.yaml @@ -0,0 +1,47 @@ +name: Master +on: + push: + branches: + - master + +permissions: + contents: read + +jobs: + go-bench: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v4 + with: + go-version-file: './go.mod' + cache-dependency-path: './go.sum' + check-latest: true + + - name: Run benchmark and store the output to a file + run: | + set -o pipefail + make bench | tee bench_output.txt + + - name: Get benchmark as JSON + uses: benchmark-action/github-action-benchmark@v1 + with: + # What benchmark tool the output.txt came from + tool: 'go' + # Where the output from the benchmark tool is stored + output-file-path: bench_output.txt + # Write benchmarks to this file + external-data-json-path: ./cache/benchmark-data.json + # Workflow will fail when an alert happens + fail-on-alert: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-on-alert: true + + - name: Save benchmark JSON to cache + uses: actions/cache/save@v3 + with: + path: ./cache/benchmark-data.json + # Save with commit hash to avoid "cache already exists" + key: ${{ github.sha }}-go-benchmark diff --git a/.github/workflows/on-pull-request.yml b/.github/workflows/on-pull-request.yml index 3f0d8191..8559e0e6 100644 --- a/.github/workflows/on-pull-request.yml +++ b/.github/workflows/on-pull-request.yml @@ -44,3 +44,54 @@ jobs: - name: Test run: go test -v -race -p=1 -count=1 + go-bench: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 # to be able to retrieve the last commit in master branch + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version-file: './go.mod' + cache-dependency-path: './go.sum' + check-latest: true + + - name: Run benchmark and store the output to a file + run: | + set -o pipefail + make bench | tee ${{ github.sha }}_bench_output.txt + + - name: Get Main branch SHA + id: get-main-branch-sha + run: | + SHA=$(git rev-parse origin/main) + echo "sha=$SHA" >> $GITHUB_OUTPUT + + - name: Get benchmark JSON from main branch + uses: actions/cache/restore@v3 + with: + path: ./cache/benchmark-data.json + key: ${{ steps.get-main-branch-sha.outputs.sha }}-go-benchmark + fail-on-cache-miss: true + + - name: Compare benchmarks + uses: benchmark-action/github-action-benchmark@v1 + with: + # What benchmark tool the output.txt came from + tool: 'go' + # Where the output from the benchmark tool is stored + output-file-path: ${{ github.sha }}_bench_output.txt + # Where the benchmarks in main are (to compare) + external-data-json-path: ./cache/benchmark-data.json + # Do not save the data + save-data-file: false + # Workflow will fail when an alert happens + fail-on-alert: true + github-token: ${{ secrets.GITHUB_TOKEN }} + # Enable Job Summary for PRs + summary-always: true + diff --git a/Makefile b/Makefile index c0580763..619ff279 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,10 @@ test: go tool cover -html coverage.out -o coverage.html; \ exit $$ret) +.PHONY: bench +bench: + go test ./... -bench . -benchtime 10s -timeout 0 -run=XXX -cpu 1 -benchmem + .PHONY: docker docker: docker build --build-arg VERSION=$(VERSION) -t ghcr.io/mailgun/gubernator:$(VERSION) . From 5178e4e123544876018f179f1412bbabc2c188c2 Mon Sep 17 00:00:00 2001 From: Maria Ines Parnisari Date: Thu, 2 Nov 2023 11:57:44 -0700 Subject: [PATCH 2/6] reduce benchtime to 5s --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 619ff279..cce3c88c 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ test: .PHONY: bench bench: - go test ./... -bench . -benchtime 10s -timeout 0 -run=XXX -cpu 1 -benchmem + go test ./... -bench . -benchtime 5s -timeout 0 -run=XXX -cpu 1 -benchmem .PHONY: docker docker: From 5f40339477f265cf686030328cd9c46bc5e8d57c Mon Sep 17 00:00:00 2001 From: Maria Ines Parnisari Date: Thu, 2 Nov 2023 12:36:13 -0700 Subject: [PATCH 3/6] increase job timeout --- .github/workflows/on-pull-request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-pull-request.yml b/.github/workflows/on-pull-request.yml index 8559e0e6..ded935a3 100644 --- a/.github/workflows/on-pull-request.yml +++ b/.github/workflows/on-pull-request.yml @@ -46,7 +46,7 @@ jobs: run: go test -v -race -p=1 -count=1 go-bench: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 30 steps: - name: Checkout code uses: actions/checkout@v3 From adde360da228c36fd999c7d10bd8116b197216a3 Mon Sep 17 00:00:00 2001 From: Maria Ines Parnisari Date: Wed, 8 Nov 2023 22:47:43 -0500 Subject: [PATCH 4/6] cache CPU + run benchmarks using multiple CPUs --- .github/workflows/master.yaml | 3 ++- .github/workflows/on-pull-request.yml | 3 ++- Makefile | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/master.yaml b/.github/workflows/master.yaml index 3ff13eb3..0977635d 100644 --- a/.github/workflows/master.yaml +++ b/.github/workflows/master.yaml @@ -44,4 +44,5 @@ jobs: with: path: ./cache/benchmark-data.json # Save with commit hash to avoid "cache already exists" - key: ${{ github.sha }}-go-benchmark + # Save with OS to prevent comparing against results from different CPUs + key: ${{ github.sha }}-${{ runner.os }}-go-benchmark diff --git a/.github/workflows/on-pull-request.yml b/.github/workflows/on-pull-request.yml index ded935a3..01536544 100644 --- a/.github/workflows/on-pull-request.yml +++ b/.github/workflows/on-pull-request.yml @@ -75,11 +75,12 @@ jobs: uses: actions/cache/restore@v3 with: path: ./cache/benchmark-data.json - key: ${{ steps.get-main-branch-sha.outputs.sha }}-go-benchmark + key: ${{ steps.get-main-branch-sha.outputs.sha }}-${{ runner.os }}-go-benchmark fail-on-cache-miss: true - name: Compare benchmarks uses: benchmark-action/github-action-benchmark@v1 + if: success() with: # What benchmark tool the output.txt came from tool: 'go' diff --git a/Makefile b/Makefile index cce3c88c..fde55c98 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ test: .PHONY: bench bench: - go test ./... -bench . -benchtime 5s -timeout 0 -run=XXX -cpu 1 -benchmem + go test ./... -bench . -benchtime 5s -timeout 0 -run=XXX -benchmem .PHONY: docker docker: From bd19a91974a2c38f9a8d09d004755cc1465de773 Mon Sep 17 00:00:00 2001 From: Maria Ines Parnisari Date: Tue, 21 Nov 2023 16:37:25 -0800 Subject: [PATCH 5/6] address comments --- .github/workflows/on-pull-request.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/on-pull-request.yml b/.github/workflows/on-pull-request.yml index 01536544..fced87ef 100644 --- a/.github/workflows/on-pull-request.yml +++ b/.github/workflows/on-pull-request.yml @@ -66,22 +66,23 @@ jobs: make bench | tee ${{ github.sha }}_bench_output.txt - name: Get Main branch SHA - id: get-main-branch-sha + id: get-master-branch-sha run: | - SHA=$(git rev-parse origin/main) + SHA=$(git rev-parse origin/master) echo "sha=$SHA" >> $GITHUB_OUTPUT - name: Get benchmark JSON from main branch uses: actions/cache/restore@v3 with: path: ./cache/benchmark-data.json - key: ${{ steps.get-main-branch-sha.outputs.sha }}-${{ runner.os }}-go-benchmark + key: ${{ steps.get-master-branch-sha.outputs.sha }}-${{ runner.os }}-go-benchmark fail-on-cache-miss: true - name: Compare benchmarks uses: benchmark-action/github-action-benchmark@v1 if: success() with: + alert-threshold: "200%" # What benchmark tool the output.txt came from tool: 'go' # Where the output from the benchmark tool is stored From ad1c5f402a525485694e33db93819e089dd07e94 Mon Sep 17 00:00:00 2001 From: Maria Ines Parnisari Date: Thu, 23 Nov 2023 10:37:00 -0800 Subject: [PATCH 6/6] run against master OR just run but don't compare results --- .github/workflows/on-pull-request.yml | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/on-pull-request.yml b/.github/workflows/on-pull-request.yml index fced87ef..412d4b15 100644 --- a/.github/workflows/on-pull-request.yml +++ b/.github/workflows/on-pull-request.yml @@ -65,29 +65,29 @@ jobs: set -o pipefail make bench | tee ${{ github.sha }}_bench_output.txt - - name: Get Main branch SHA + - name: Get Master branch SHA id: get-master-branch-sha run: | SHA=$(git rev-parse origin/master) echo "sha=$SHA" >> $GITHUB_OUTPUT - - name: Get benchmark JSON from main branch + - name: Get benchmark JSON from Master branch + id: cache uses: actions/cache/restore@v3 with: path: ./cache/benchmark-data.json key: ${{ steps.get-master-branch-sha.outputs.sha }}-${{ runner.os }}-go-benchmark - fail-on-cache-miss: true - - name: Compare benchmarks + - name: Compare benchmarks with master uses: benchmark-action/github-action-benchmark@v1 - if: success() + if: steps.cache.outputs.cache-hit == 'true' with: alert-threshold: "200%" # What benchmark tool the output.txt came from tool: 'go' # Where the output from the benchmark tool is stored output-file-path: ${{ github.sha }}_bench_output.txt - # Where the benchmarks in main are (to compare) + # Where the benchmarks in master are (to compare) external-data-json-path: ./cache/benchmark-data.json # Do not save the data save-data-file: false @@ -97,3 +97,16 @@ jobs: # Enable Job Summary for PRs summary-always: true + - name: Run benchmarks but don't compare to master branch + uses: benchmark-action/github-action-benchmark@v1 + if: steps.cache.outputs.cache-hit != 'true' + with: + # What benchmark tool the output.txt came from + tool: 'go' + # Where the output from the benchmark tool is stored + output-file-path: ${{ github.sha }}_bench_output.txt + # Write benchmarks to this file, do not publish to GitHub Pages + save-data-file: false + external-data-json-path: ./cache/benchmark-data.json + # Enable Job Summary for PRs + summary-always: true