Benchmark #15
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| schedule: | |
| - cron: '30 3 * * *' | |
| workflow_dispatch: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| build_tquic: | |
| name: Build tquic | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| - name: Update rust | |
| run: rustup update | |
| - name: Build tquic | |
| run: | | |
| cargo build --all --release | |
| cp target/release/tquic_server tquic_server | |
| cp target/release/tquic_client tquic_client | |
| - name: Build start script | |
| run: | | |
| echo $'#!/bin/bash\nchmod u+x ./tquic_server\n./tquic_server --send-udp-payload-size 1350 --log-level OFF --root ./ --disable-stateless-reset -l 0.0.0.0:4433 -c ./cert.crt -k ./cert.key &' > start_tquic.sh | |
| chmod u+x start_tquic.sh | |
| - name: Upload tquic_server | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: tquic_server_bin | |
| path: | | |
| tquic_server | |
| start_tquic.sh | |
| - name: Upload tquic_client | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: tquic_client_bin | |
| path: tquic_client | |
| build_lsquic: | |
| name: Build lsquic | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: 'litespeedtech/lsquic' | |
| path: lsquic | |
| submodules: 'recursive' | |
| - name: Build lsquic | |
| run: | | |
| git clone https://boringssl.googlesource.com/boringssl | |
| cd boringssl | |
| git checkout 9fc1c33e9c21439ce5f87855a6591a9324e569fd | |
| cmake . && make | |
| BORINGSSL=$PWD | |
| cd ../lsquic | |
| sudo apt install libevent-dev | |
| cmake -DBORINGSSL_DIR=$BORINGSSL . | |
| make | |
| cp bin/http_server ../lsquic_server | |
| - name: Build start script | |
| run: | | |
| echo $'#!/bin/bash\nchmod u+x ./lsquic_server\n./lsquic_server -c tquic_benchmark,./cert.crt,./cert.key -s 0.0.0.0:4433 -r ./ -L crit > lsquic.log 2>&1 &' > start_lsquic.sh | |
| chmod u+x start_lsquic.sh | |
| - name: Upload lsquic server | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: lsquic_server_bin | |
| path: | | |
| lsquic_server | |
| start_lsquic.sh | |
| gen_cert: | |
| name: Generate cert | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Generate cert | |
| run: | | |
| openssl genrsa -out cert.key 2048 | |
| openssl req -new -x509 -key cert.key -out cert.crt -days 365 -subj "/CN=tquic_benchmark" | |
| - name: Upload cert | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cert | |
| path: cert.* | |
| gen_files: | |
| name: Generate files | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Generate files | |
| run: | | |
| head -c 1K /dev/urandom > file_1K | |
| head -c 15K /dev/urandom > file_15K | |
| head -c 50K /dev/urandom > file_50K | |
| head -c 2M /dev/urandom > file_2M | |
| - name: Upload files | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: files | |
| path: file_* | |
| run_long_conn: | |
| name: Run long connection scenario benchmark | |
| needs: [ build_tquic, build_lsquic, gen_cert, gen_files ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| server: [ tquic ] | |
| file: [ 15K, 50K, 2M ] | |
| conn: [ 10 ] | |
| stream: [ 1, 10 ] | |
| steps: | |
| - name: Download ${{ matrix.server }} server | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ matrix.server }}_server_bin | |
| - name: Download cert | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: cert | |
| - name: Download files | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: files | |
| - name: Download tquic_client | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: tquic_client_bin | |
| - name: Display structure of downloaded files | |
| run: ls -R | |
| - name: Start ${{ matrix.server }} server | |
| run: | | |
| sh start_${{ matrix.server }}.sh | |
| pgrep ${{ matrix.server }}_server | |
| - name: Benchmark ${{ matrix.server }} | |
| run: | | |
| chmod u+x ./tquic_client | |
| ./tquic_client https://tquic_benchmark:4433/file_${{ matrix.file }} --connect-to 127.0.0.1:4433 --threads ${{ matrix.conn }} --max-concurrent-conns 1 --max-concurrent-requests ${{ matrix.stream }} --max-requests-per-conn 0 --total-requests-per-thread 0 -d 10 --disable-stateless-reset --send-batch-size 1 --recv-udp-payload-size 1350 --send-udp-payload-size 1350 --log-level OFF | grep "finished in" | awk '{prinit $4}' > benchmark_long_${{ matrix.server }}_${{ matrix.file }}_${{ matrix.conn }}_${{ matrix.stream }} | |
| - name: Stop ${{ matrix.server }} server | |
| run: | | |
| killall ${{ matrix.server }}_server | |
| sleep 1 | |
| - name: Upload benchmark result | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark_long_${{ matrix.server }}_${{ matrix.file }}_${{ matrix.conn }}_${{ matrix.stream }} | |
| path: benchmark_long_* | |
| run_short_conn: | |
| name: Run short connection scenario benchmark | |
| needs: [ build_tquic, build_lsquic, gen_cert, gen_files ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| server: [ tquic, lsquic ] | |
| steps: | |
| - name: Download ${{ matrix.server }} server | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ matrix.server }}_server_bin | |
| - name: Download cert | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: cert | |
| - name: Download files | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: files | |
| - name: Download tquic_client | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: tquic_client_bin | |
| - name: Display structure of downloaded files | |
| run: ls -R | |
| - name: Start ${{ matrix.server }} server | |
| run: | | |
| sh start_${{ matrix.server }}.sh | |
| pgrep ${{ matrix.server }}_server | |
| - name: Benchmark ${{ matrix.server }} | |
| run: | | |
| chmod u+x ./tquic_client | |
| ./tquic_client https://tquic_benchmark:4433/file_1K --threads 10 --max-concurrent-conns 1 --max-concurrent-requests 1 --max-requests-per-conn 1 --total-requests-per-thread 0 -d 10 --disable-stateless-reset --send-batch-size 1 --recv-udp-payload-size 1350 --send-udp-payload-size 1350 --log-level OFF | grep "finished in" | awk '{prinit $4}' > benchmark_short_${{ matrix.server }}_1K_10_1 | |
| - name: Stop ${{ matrix.server }} server | |
| run: | | |
| killall ${{ matrix.server }}_server | |
| sleep 1 | |
| - name: Upload benchmark result | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark_short_${{ matrix.server }}_1K_10_1 | |
| path: benchmark_short_* | |
| result: | |
| runs-on: ubuntu-latest | |
| needs: [ run_long_conn, run_short_conn ] | |
| steps: | |
| - name: Download all benchmark results | |
| uses: actions/download-artifact@v4 | |
| - name: Display structure of downloaded files | |
| run: ls -R | |
| - name: Download plot tools | |
| uses: actions/checkout@v4 | |
| with: | |
| path: tools | |
| - name: Install dependencies | |
| run: | | |
| sudo apt install python3-matplotlib | |
| pip3 install prettytable termcolor | |
| - name: Plot and print all benchmark results | |
| run: python3 tools/.github/workflows/plot-benchmark.py . | |
| - name: Store all benchmark results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark_all | |
| path: benchmark_all.png | |