Benchmark #210
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Benchmark libgit2 against the git reference implementation. | |
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| suite: | |
| description: Benchmark suite to run | |
| debug: | |
| type: boolean | |
| description: Debugging output | |
| schedule: | |
| - cron: '15 4 * * *' | |
| permissions: | |
| contents: read | |
| jobs: | |
| # Run our benchmarks. We build a matrix with the various build | |
| # targets and their details. Unlike our CI builds, we run these | |
| # directly on the VM instead of in containers since we do not | |
| # need the breadth of platform diversity. | |
| build: | |
| # Only run scheduled workflows on the main repository; prevents people | |
| # from using build minutes on their forks. | |
| #TODO | |
| # if: github.repository == 'libgit2/libgit2' | |
| strategy: | |
| matrix: | |
| platform: | |
| - name: "Linux (clang, OpenSSL)" | |
| id: linux | |
| os: ubuntu-latest | |
| setup-script: ubuntu | |
| env: | |
| CC: clang | |
| CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=Release | |
| CMAKE_BUILD_OPTIONS: --config Release | |
| #TODO | |
| # - name: "macOS" | |
| # id: macos | |
| # os: macos-latest | |
| # setup-script: osx | |
| # env: | |
| # CC: clang | |
| # CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=Release | |
| # CMAKE_BUILD_OPTIONS: --config Release | |
| # PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig | |
| # - name: "Windows (amd64, Visual Studio)" | |
| # id: windows | |
| # os: windows-2022 | |
| # setup-script: win32 | |
| # env: | |
| # ARCH: amd64 | |
| # CMAKE_GENERATOR: Visual Studio 17 2022 | |
| # CMAKE_OPTIONS: -A x64 -DDEPRECATE_HARD=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=Release | |
| # CMAKE_BUILD_OPTIONS: --config Release | |
| fail-fast: false | |
| name: "Benchmark ${{ matrix.platform.name }}" | |
| env: ${{ matrix.platform.env }} | |
| runs-on: ${{ matrix.platform.os }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: source | |
| fetch-depth: 0 | |
| - name: Set up benchmark environment | |
| run: source/ci/setup-${{ matrix.platform.setup-script }}-benchmark.sh | |
| shell: bash | |
| if: matrix.platform.setup-script != '' | |
| - name: Clone resource repositories | |
| run: | | |
| # we need a superior way to package the benchmark resources; lfs | |
| # is too expensive | |
| # git lfs install | |
| # git clone https://github.com/libgit2/benchmark-resources resources | |
| git clone --bare https://github.com/git/git resources/git | |
| # avoid linux temporarily; the linux blame benchmarks are simply | |
| # too slow to use | |
| # git clone --bare https://github.com/torvalds/linux resources/linux | |
| - name: Build | |
| run: | | |
| mkdir build && cd build | |
| ../source/ci/build.sh | |
| shell: bash | |
| - name: Benchmark | |
| run: | | |
| # avoid benchmark resource path currently | |
| #export BENCHMARK_RESOURCES_PATH="$(pwd)/resources" | |
| export BENCHMARK_GIT_PATH="$(pwd)/resources/git" | |
| # avoid linux temporarily; the linux blame benchmarks are simply | |
| # too slow to use | |
| # export BENCHMARK_LINUX_PATH="$(pwd)/resources/linux" | |
| if [[ "$(uname -s)" == MINGW* ]]; then | |
| GIT2_CLI="$(cygpath -w $(pwd))\\build\\Release\\git2" | |
| else | |
| GIT2_CLI="$(pwd)/build/git2" | |
| fi | |
| if [ "${{ github.event.inputs.suite }}" != "" ]; then | |
| SUITE_FLAG="--suite ${{ github.event.inputs.suite }}" | |
| fi | |
| if [ "${{ github.event.inputs.debug }}" = "true" ]; then | |
| DEBUG_FLAG="--debug" | |
| fi | |
| mkdir benchmark && cd benchmark | |
| ../source/tests/benchmarks/benchmark.sh \ | |
| ${SUITE_FLAG} ${DEBUG_FLAG} \ | |
| --baseline-cli "git" --cli "${GIT2_CLI}" --name libgit2 \ | |
| --json benchmarks.json --flamegraph --zip benchmarks.zip | |
| shell: bash | |
| - name: Upload results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-${{ matrix.platform.id }} | |
| path: benchmark | |
| if: always() | |
| # Publish the results | |
| publish: | |
| name: Publish results | |
| needs: [ build ] | |
| if: always() && github.repository == 'libgit2/libgit2' && github.event_name == 'schedule' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out benchmark repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: libgit2/benchmarks | |
| path: site | |
| fetch-depth: 0 | |
| ssh-key: ${{ secrets.BENCHMARKS_PUBLISH_KEY }} | |
| - name: Download test results | |
| uses: actions/download-artifact@v4 | |
| - name: Generate API | |
| run: | | |
| # Move today's benchmark run into the right place | |
| #TODO | |
| #for platform in linux macos windows; do | |
| for platform in linux; do | |
| TIMESTAMP=$(jq .time.start < "benchmark-${platform}/benchmarks.json") | |
| TIMESTAMP_LEN=$(echo -n ${TIMESTAMP} | wc -c | xargs) | |
| DENOMINATOR=1 | |
| if [ "${TIMESTAMP_LEN}" = "19" ]; then | |
| DENOMINATOR="1000000000" | |
| elif [ "${TIMESTAMP_LEN}" = "13" ]; then | |
| DENOMINATOR="1000" | |
| else | |
| echo "unknown timestamp" | |
| exit 1 | |
| fi | |
| if [[ "$(uname -s)" == "Darwin" ]]; then | |
| DATE=$(date -R -r $(("${TIMESTAMP}/${DENOMINATOR}")) +"%Y-%m-%d") | |
| else | |
| DATE=$(date -d @$(("${TIMESTAMP}/${DENOMINATOR}")) +"%Y-%m-%d") | |
| fi | |
| # move the complete results in | |
| mkdir -p "site/public/api/runs/${DATE}" | |
| cp "benchmark-${platform}/benchmarks.json" "site/public/api/runs/${DATE}/${platform}.json" | |
| # unzip the individual results | |
| mkdir -p "site/public/api/runs/${DATE}/${platform}" | |
| unzip "benchmark-${platform}/benchmarks.zip" -d "site/public/api/runs/${DATE}/${platform}" '*.json' '*.svg' | |
| done | |
| (cd site && node scripts/aggregate.js) | |
| shell: bash | |
| # in debug mode, don't deploy the site; only create a zip file and | |
| # upload it for debugging | |
| - name: Upload site | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: site | |
| path: site | |
| # if: github.event.inputs.debug == 'true' | |
| # - name: Publish API | |
| # run: | | |
| # git config user.name 'Benchmark Site Generation' && | |
| # git config user.email '[email protected]' && | |
| # git add . && | |
| # git commit --allow-empty -m"benchmark update ${DATE}" && | |
| # git push origin main | |
| # shell: bash | |
| # working-directory: site | |
| # if: always() |