From 8818dac3d4d91d2449fc6930224d08fb4d787d02 Mon Sep 17 00:00:00 2001 From: xiaofei0800 Date: Sat, 11 May 2024 15:30:08 +0800 Subject: [PATCH] Optimize benchmark workflow. --- .github/workflows/plot-benchmark.py | 44 ++++--- .github/workflows/plot-fct.py | 2 +- .github/workflows/plot-goodput.py | 2 +- .github/workflows/plot-interop.py | 2 +- .github/workflows/tquic-benchmark.yml | 164 ++++++++++++++------------ 5 files changed, 120 insertions(+), 94 deletions(-) diff --git a/.github/workflows/plot-benchmark.py b/.github/workflows/plot-benchmark.py index 46e5b42a..d9391d17 100644 --- a/.github/workflows/plot-benchmark.py +++ b/.github/workflows/plot-benchmark.py @@ -12,10 +12,13 @@ # Benchmark scenarios SCENARIO = ["long", "short"] -# QUIC implements. +# QUIC implementations. # The first element is used as the normalization base. IMPLS = ["lsquic", "tquic"] +# Round of benchmark in one scenario. +ROUND = 5 + # File sizes in long connection scenario benchmark. LONG_FILE_SIZES = ["15K", "50K", "2M"] @@ -38,9 +41,9 @@ DAYS = 90 # Read data from benchmark result file. -def read_data(data_dir, scen, impl, size, conn, stream, date): - dirname = "benchmark_%s_%s_%s_%d_%d.%s" % (scen, impl, size, conn, stream, date) - filename = "benchmark_%s_%s_%s_%d_%d.%s" % (scen, impl, size, conn, stream, date) +def read_data(data_dir, scen, impl, size, conn, stream, round, date): + dirname = "benchmark_%s_%s_%d_%d.%s" % (scen, size, conn, stream, date) + filename = "benchmark_%s_%s_%s_%d_%d.%d.%s" % (scen, impl, size, conn, stream, round, date) path = os.path.join(data_dir, dirname, filename) try: with open(path) as f: @@ -49,10 +52,10 @@ def read_data(data_dir, scen, impl, size, conn, stream, date): except: return 0.0 -# Put benchmark result in array according to implement. +# Load benchmark results into array. def prepare_data(data_dir): titles = [' ' for _ in range((len(LONG_FILE_SIZES)*len(LONG_CONNS)*len(LONG_STREAMS) + len(SHORT_FILE_SIZES)*len(SHORT_CONNS)*len(SHORT_STREAMS)))] - result = [[[0.0 for _ in range(len(LONG_FILE_SIZES)*len(LONG_CONNS)*len(LONG_STREAMS) + len(SHORT_FILE_SIZES)*len(SHORT_CONNS)*len(SHORT_STREAMS))] for _ in range(len(IMPLS))] for _ in range(DAYS)] + result = [[[[0.0 for _ in range(len(LONG_FILE_SIZES)*len(LONG_CONNS)*len(LONG_STREAMS) + len(SHORT_FILE_SIZES)*len(SHORT_CONNS)*len(SHORT_STREAMS))] for _ in range(len(IMPLS))] for _ in range(DAYS)] for _ in range(ROUND) ] # Load long connection scenario result. I = len(LONG_FILE_SIZES) @@ -66,8 +69,9 @@ def prepare_data(data_dir): titles[i*J*K+j*K+k] = "long %s %d %d" % (LONG_FILE_SIZES[i], LONG_CONNS[j], LONG_STREAMS[k]) for n in range(N): for d in range(D): - date = (datetime.datetime.now() - datetime.timedelta(days=d)).strftime('%Y-%m-%d') - result[D-1-d][n][i*J*K+j*K+k] = read_data(data_dir, "long", IMPLS[n], LONG_FILE_SIZES[i], LONG_CONNS[j], LONG_STREAMS[k], date) + for r in range(ROUND): + date = (datetime.datetime.now() - datetime.timedelta(days=d)).strftime('%Y-%m-%d') + result[r][D-1-d][n][i*J*K+j*K+k] = read_data(data_dir, "long", IMPLS[n], LONG_FILE_SIZES[i], LONG_CONNS[j], LONG_STREAMS[k], r, date) # Load short connection scenario result. M = len(LONG_FILE_SIZES)*len(LONG_CONNS)*len(LONG_STREAMS) @@ -82,19 +86,23 @@ def prepare_data(data_dir): titles[M+i*J*K+j*K+k] = "short %s %d %d" % (SHORT_FILE_SIZES[i], SHORT_CONNS[j], SHORT_STREAMS[k]) for n in range(N): for d in range(D): - date = (datetime.datetime.now() - datetime.timedelta(days=d)).strftime('%Y-%m-%d') - result[D-1-d][n][M+i*J*K+j*K+k] = read_data(data_dir, "short", IMPLS[n], SHORT_FILE_SIZES[i], SHORT_CONNS[j], LONG_STREAMS[k], date) + for r in range(ROUND): + date = (datetime.datetime.now() - datetime.timedelta(days=d)).strftime('%Y-%m-%d') + result[r][D-1-d][n][M+i*J*K+j*K+k] = read_data(data_dir, "short", IMPLS[n], SHORT_FILE_SIZES[i], SHORT_CONNS[j], LONG_STREAMS[k], r, date) + + # Average by rounds. + result_avg = np.mean(np.array(result), axis=0).tolist() # Normalize benchmark result. for d in range(D): - base = result[d][0] - for i in range(1, len(result[d])): - result[d][i] = [round(x/y, 4) if y != 0 else 0 for x, y in zip(result[d][i], base)] - for i in range(len(result[d][0])): - if result[d][0][i] != 0: - result[d][0][i] = 1 - - return titles, result + base = result_avg[d][0] + for i in range(1, len(result_avg[d])): + result_avg[d][i] = [round(x/y, 4) if y != 0 else 0 for x, y in zip(result_avg[d][i], base)] + for i in range(len(result_avg[d][0])): + if result_avg[d][0][i] != 0: + result_avg[d][0][i] = 1 + + return titles, result_avg # Print benchmark performance result to stdout. def show(titles, result): diff --git a/.github/workflows/plot-fct.py b/.github/workflows/plot-fct.py index 2c61b6f7..d21ec5d4 100644 --- a/.github/workflows/plot-fct.py +++ b/.github/workflows/plot-fct.py @@ -7,7 +7,7 @@ import matplotlib.pyplot as plt import numpy as np -# QUIC implements +# QUIC implementations IMPLS = ["tquic", "gquiche", "lsquic", "picoquic", "quiche"] # Different modes diff --git a/.github/workflows/plot-goodput.py b/.github/workflows/plot-goodput.py index 0e8ddcc1..21b802ae 100644 --- a/.github/workflows/plot-goodput.py +++ b/.github/workflows/plot-goodput.py @@ -7,7 +7,7 @@ import matplotlib.pyplot as plt import numpy as np -# QUIC implements +# QUIC implementations IMPLS = ["tquic", "gquiche", "lsquic", "picoquic", "quiche"] # Different file sizes diff --git a/.github/workflows/plot-interop.py b/.github/workflows/plot-interop.py index eb308d42..b52fd9d2 100644 --- a/.github/workflows/plot-interop.py +++ b/.github/workflows/plot-interop.py @@ -8,7 +8,7 @@ import numpy as np from matplotlib.colors import ListedColormap -# QUIC implements +# QUIC implementations CLIENT_IMPLS = ["tquic", "lsquic", "quiche", "picoquic", "ngtcp2", "msquic", "s2n-quic", "quinn", "neqo", "kwik", "aioquic", "chrome", "go-x-net", "quic-go", "mvfst"] diff --git a/.github/workflows/tquic-benchmark.yml b/.github/workflows/tquic-benchmark.yml index a38e29b0..186195ab 100644 --- a/.github/workflows/tquic-benchmark.yml +++ b/.github/workflows/tquic-benchmark.yml @@ -14,6 +14,9 @@ jobs: runs-on: ubuntu-latest outputs: benchmark_date: ${{ steps.set-benchmark-date.outputs.benchmark_date }} + benchmark_impls: ${{ steps.set-implements.outputs.benchmark_impls }} + benchmark_rounds: ${{ steps.set-rounds.outputs.benchmark_rounds }} + benchmark_duration: ${{ steps.set-duration.outputs.benchmark_duration }} steps: - name: Set date id: set-benchmark-date @@ -21,6 +24,21 @@ jobs: BENCHMARK_DATE=$(date -u +"%Y-%m-%d") echo $BENCHMARK_DATE echo "benchmark_date=$BENCHMARK_DATE" >> $GITHUB_OUTPUT + - name: Set implements + id: set-implements + run: | + IMPLS="lsquic tquic" + echo "benchmark_impls=$IMPLS" >> $GITHUB_OUTPUT + - name: Set rounds + id: set-rounds + run: | + ROUNDS=5 + echo "benchmark_rounds=$ROUNDS" >> $GITHUB_OUTPUT + - name: Set benchmark duration + id: set-duration + run: | + DURATION=120 + echo "benchmark_duration=$DURATION" >> $GITHUB_OUTPUT build_tquic: name: Build tquic @@ -38,7 +56,7 @@ jobs: cp target/release/tquic_client tquic_client - name: Build start script run: | - echo $'#!/bin/bash\nchmod u+x ./tquic_server\n./tquic_server --send-udp-payload-size 1350 --log-level OFF --root ./ --disable-stateless-reset -l 0.0.0.0:4433 -c ./cert.crt -k ./cert.key &' > start_tquic.sh + echo $'#!/bin/bash\ncd "$(dirname "$0")"\nchmod u+x ./tquic_server\n./tquic_server --send-udp-payload-size 1350 --log-level OFF --root ../files --disable-stateless-reset -l 0.0.0.0:4433 -c ../cert/cert.crt -k ../cert/cert.key &' > start_tquic.sh chmod u+x start_tquic.sh - name: Upload tquic_server uses: actions/upload-artifact@v4 @@ -76,7 +94,7 @@ jobs: cp bin/http_server ../lsquic_server - name: Build start script run: | - echo $'#!/bin/bash\nchmod u+x ./lsquic_server\n./lsquic_server -c tquic_benchmark,./cert.crt,./cert.key -s 0.0.0.0:4433 -r ./ -L crit > lsquic.log 2>&1 &' > start_lsquic.sh + echo $'#!/bin/bash\ncd "$(dirname "$0")"\nchmod u+x ./lsquic_server\n./lsquic_server -c tquic_benchmark,../cert/cert.crt,../cert/cert.key -s 0.0.0.0:4433 -r ../files -L crit > lsquic.log 2>&1 &' > start_lsquic.sh chmod u+x start_lsquic.sh - name: Upload lsquic server uses: actions/upload-artifact@v4 @@ -122,46 +140,32 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - server: [ tquic, lsquic ] file: [ 15K, 50K, 2M ] conn: [ 10 ] stream: [ 1, 10 ] steps: - - name: Download ${{ matrix.server }} server - uses: actions/download-artifact@v4 - with: - name: ${{ matrix.server }}_server_bin - - name: Download cert - uses: actions/download-artifact@v4 - with: - name: cert - - name: Download files - uses: actions/download-artifact@v4 - with: - name: files - - name: Download tquic_client + - name: Download all uses: actions/download-artifact@v4 - with: - name: tquic_client_bin - name: Display structure of downloaded files run: ls -R - - name: Start ${{ matrix.server }} server - run: | - sh start_${{ matrix.server }}.sh - pgrep ${{ matrix.server }}_server - - name: Benchmark ${{ matrix.server }} - run: | - chmod u+x ./tquic_client - ./tquic_client https://tquic_benchmark:4433/file_${{ matrix.file }} --connect-to 127.0.0.1:4433 --threads ${{ matrix.conn }} --max-concurrent-conns 1 --max-concurrent-requests ${{ matrix.stream }} --max-requests-per-conn 0 --total-requests-per-thread 0 -d 600 --disable-stateless-reset --send-batch-size 1 --recv-udp-payload-size 1350 --send-udp-payload-size 1350 --log-level OFF > client.log 2>&1 - cat client.log | grep "finished in" | awk '{print $4}' > benchmark_long_${{ matrix.server }}_${{ matrix.file }}_${{ matrix.conn }}_${{ matrix.stream }}.${{ needs.config.outputs.benchmark_date }} - - name: Stop ${{ matrix.server }} server + - name: Benchmark run: | - killall ${{ matrix.server }}_server - sleep 1 + chmod u+x ./tquic_client_bin/tquic_client + for((round=0;round<${{ needs.config.outputs.benchmark_rounds }};round++));do + for impl in ${{ needs.config.outputs.benchmark_impls }};do + sh ${impl}_server_bin/start_${impl}.sh + pgrep ${impl}_server + sleep 1 + ./tquic_client_bin/tquic_client https://tquic_benchmark:4433/file_${{ matrix.file }} --connect-to 127.0.0.1:4433 --threads ${{ matrix.conn }} --max-concurrent-conns 1 --max-concurrent-requests ${{ matrix.stream }} --max-requests-per-conn 0 --total-requests-per-thread 0 -d ${{ needs.config.outputs.benchmark_duration }} --disable-stateless-reset --send-batch-size 1 --recv-udp-payload-size 1350 --send-udp-payload-size 1350 --log-level OFF > client.log 2>&1 + cat client.log | grep "finished in" | awk '{print $4}' > benchmark_long_${impl}_${{ matrix.file }}_${{ matrix.conn }}_${{ matrix.stream }}.${round}.${{ needs.config.outputs.benchmark_date }} + killall ${impl}_server + sleep 1 + done + done - name: Upload benchmark result uses: actions/upload-artifact@v4 with: - name: benchmark_long_${{ matrix.server }}_${{ matrix.file }}_${{ matrix.conn }}_${{ matrix.stream }}.${{ needs.config.outputs.benchmark_date }} + name: benchmark_long_${{ matrix.file }}_${{ matrix.conn }}_${{ matrix.stream }}.${{ needs.config.outputs.benchmark_date }} path: benchmark_long_* retention-days: 90 @@ -169,45 +173,29 @@ jobs: name: Run short connection scenario benchmark needs: [ config, build_tquic, build_lsquic, gen_cert, gen_files ] runs-on: ubuntu-latest - strategy: - matrix: - server: [ tquic, lsquic ] steps: - - name: Download ${{ matrix.server }} server - uses: actions/download-artifact@v4 - with: - name: ${{ matrix.server }}_server_bin - - name: Download cert - uses: actions/download-artifact@v4 - with: - name: cert - - name: Download files - uses: actions/download-artifact@v4 - with: - name: files - - name: Download tquic_client + - name: Download all uses: actions/download-artifact@v4 - with: - name: tquic_client_bin - name: Display structure of downloaded files run: ls -R - - name: Start ${{ matrix.server }} server - run: | - sh start_${{ matrix.server }}.sh - pgrep ${{ matrix.server }}_server - - name: Benchmark ${{ matrix.server }} + - name: Benchmark run: | - chmod u+x ./tquic_client - ./tquic_client https://tquic_benchmark:4433/file_1K --connect-to 127.0.0.1:4433 --threads 10 --max-concurrent-conns 1 --max-concurrent-requests 1 --max-requests-per-conn 1 --total-requests-per-thread 0 -d 600 --disable-stateless-reset --send-batch-size 1 --recv-udp-payload-size 1350 --send-udp-payload-size 1350 --log-level OFF > client.log 2>&1 - cat client.log | grep "finished in" | awk '{print $4}' > benchmark_short_${{ matrix.server }}_1K_10_1.${{ needs.config.outputs.benchmark_date }} - - name: Stop ${{ matrix.server }} server - run: | - killall ${{ matrix.server }}_server - sleep 1 + chmod u+x ./tquic_client_bin/tquic_client + for((round=0;round<${{ needs.config.outputs.benchmark_rounds }};round++));do + for impl in ${{ needs.config.outputs.benchmark_impls }};do + sh ${impl}_server_bin/start_${impl}.sh + pgrep ${impl}_server + sleep 1 + ./tquic_client_bin/tquic_client https://tquic_benchmark:4433/file_1K --connect-to 127.0.0.1:4433 --threads 10 --max-concurrent-conns 1 --max-concurrent-requests 1 --max-requests-per-conn 1 --total-requests-per-thread 0 -d ${{ needs.config.outputs.benchmark_duration }} --disable-stateless-reset --send-batch-size 1 --recv-udp-payload-size 1350 --send-udp-payload-size 1350 --log-level OFF > client.log 2>&1 + cat client.log | grep "finished in" | awk '{print $4}' > benchmark_short_${impl}_1K_10_1.${round}.${{ needs.config.outputs.benchmark_date }} + killall ${impl}_server + sleep 1 + done + done - name: Upload benchmark result uses: actions/upload-artifact@v4 with: - name: benchmark_short_${{ matrix.server }}_1K_10_1.${{ needs.config.outputs.benchmark_date }} + name: benchmark_short_1K_10_1.${{ needs.config.outputs.benchmark_date }} path: benchmark_short_* retention-days: 90 @@ -215,25 +203,55 @@ jobs: runs-on: ubuntu-latest needs: [ run_long_conn, run_short_conn ] steps: - - name: Download all benchmark results - uses: actions/download-artifact@v4 - - - name: Display structure of downloaded files - run: ls -R - - name: Download plot tools uses: actions/checkout@v4 + - name: Download all + uses: actions/download-artifact@v4 with: - path: tools - + path: benchmark_result + - name: Download latest benchmark history + working-directory: ./benchmark_result + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + SCENARIO=("long" "short") + LONG_FILE_SIZES=("15K" "50K" "2M") + SHORT_FILE_SIZES=("1K") + LONG_CONNS=10 + SHORT_CONNS=10 + LONG_STREAMS=(1 10) + SHORT_STREAMS=(1) + DAYS=90 + for ((i=1; i<$DAYS; i++)); do + date=$(date -d "-$i day" +%Y-%m-%d) + download_cmd="gh run download" + for scen in "${SCENARIO[@]}"; do + if [ "$scen" == "long" ]; then + FILE_SIZES=("${LONG_FILE_SIZES[@]}") + CONNS=$LONG_CONNS + STREAMS=("${LONG_STREAMS[@]}") + else + FILE_SIZES=("${SHORT_FILE_SIZES[@]}") + CONNS=$SHORT_CONNS + STREAMS=("${SHORT_STREAMS[@]}") + fi + for size in "${FILE_SIZES[@]}"; do + for stream in "${STREAMS[@]}"; do + download_cmd+=" -n benchmark_${scen}_${size}_${CONNS}_${stream}.${date}" + done + done + done + echo "$download_cmd" + eval "$download_cmd" || echo "" + done + - name: Display structure of downloaded files + run: ls -R benchmark_result - name: Install dependencies run: | sudo apt install python3-matplotlib pip3 install prettytable termcolor - - name: Plot and print all benchmark results - run: python3 tools/.github/workflows/plot-benchmark.py . - + run: python3 .github/workflows/plot-benchmark.py ./benchmark_result - name: Store all benchmark results uses: actions/upload-artifact@v4 with: