diff --git a/.github/workflows/plot-benchmark.py b/.github/workflows/plot-benchmark.py new file mode 100644 index 00000000..dea2291f --- /dev/null +++ b/.github/workflows/plot-benchmark.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +import os +import sys + +import json +import matplotlib.pyplot as plt +import numpy as np +import prettytable +import termcolor + +# Benchmark scenarios +SCENARIO = ["long", "short"] + +# QUIC implements +IMPLS = ["tquic", "lsquic"] + +# File sizes in long connection scenario benchmark +LONG_FILE_SIZES = ["15K", "50K", "2M"] + +# File sizes in short connection scenario benchmark +SHORT_FILE_SIZES = ["1K"] + +# Different concurrent connections +LONG_CONNS = [10] + +# Different concurrent connections +SHORT_CONNS = [10] + +# Different concurrent streams +LONG_STREAMS = [1, 10] + +# Different concurrent streams +SHORT_STREAMS = [1] + +# Read data from benchmark result file. +def read_data(data_dir, scen, impl, size, conn, stream): + dirname = "benchmark_%s_%s_%s_%d_%d" % (scen, impl, size, conn, stream) + filename = "benchmark_%s_%s_%s_%d_%d" % (scen, impl, size, conn, stream) + path = os.path.join(data_dir, dirname, filename) + try: + with open(path) as f: + data = f.read().strip() + return float(data) + except: + return 0.0 + +# Put benchmark result in array according to implement. +def prepare_data(data_dir): + titles = [' ' for _ in range((len(LONG_FILE_SIZES)*len(LONG_CONNS)*len(LONG_STREAMS) + len(SHORT_FILE_SIZES)*len(SHORT_CONNS)*len(SHORT_STREAMS)))] + result = [[0.0 for _ in range(len(LONG_FILE_SIZES)*len(LONG_CONNS)*len(LONG_STREAMS) + len(SHORT_FILE_SIZES)*len(SHORT_CONNS)*len(SHORT_STREAMS))] for _ in range(len(IMPLS))] + + + I = len(LONG_FILE_SIZES) + J = len(LONG_CONNS) + K = len(LONG_STREAMS) + N = len(IMPLS) + for i in range(I): + for j in range(J): + for k in range(K): + titles[i*J*K+j*K+k] = "long %s %d %d" % (LONG_FILE_SIZES[i], LONG_CONNS[j], LONG_STREAMS[k]) + for n in range(N): + result[n][i*J*K+j*K+k] = read_data(data_dir, "long", IMPLS[n], LONG_FILE_SIZES[i], LONG_CONNS[j], LONG_STREAMS[k]) + + M = len(LONG_FILE_SIZES)*len(LONG_CONNS)*len(LONG_STREAMS) + I = len(SHORT_FILE_SIZES) + J = len(SHORT_CONNS) + K = len(SHORT_STREAMS) + N = len(IMPLS) + for i in range(I): + for j in range(J): + for k in range(K): + titles[M+i*J*K+j*K+k] = "short %s %d %d" % (SHORT_FILE_SIZES[i], SHORT_CONNS[j], SHORT_STREAMS[k]) + for n in range(N): + result[n][M+i*J*K+j*K+k] = read_data(data_dir, "short", IMPLS[n], SHORT_FILE_SIZES[i], SHORT_CONNS[j], LONG_STREAMS[k]) + + return titles, result + +# Print benchmark performance result to stdout. +def show(titles, result): + titles.insert(0, '') + + table = prettytable.PrettyTable() + table.field_names = titles + + for i in range(len(result)): + colored_row_name = termcolor.colored(IMPLS[i], 'green') + table.add_row([colored_row_name] + result[i]) + + print(table) + +def plot(titles, result): + + N = len(titles) + M = len(result) + + width = 0.35 + gap = 0.5 + + ind = np.arange(N) * (width * M + gap) + + fig, ax = plt.subplots() + fig.set_size_inches(10, 5) + for i in range(M): + ax.bar(ind + i*width, result[i], width, label=IMPLS[i]) + + ax.set_ylabel('RPS') + ax.set_title('TQUIC benchmark') + ax.set_xticks(ind + width * M / 2) + ax.set_xticklabels(titles, rotation=45) + + ax.legend() + + plt.savefig("benchmark_all.png", dpi=300) + +if __name__ == '__main__': + if len(sys.argv) < 2: + print("Usage: %s [data_dir]" % (sys.argv[0])) + exit(1) + + data_dir= sys.argv[1] + titles, result = prepare_data(data_dir) + plot(titles, result) + show(titles, result) + + diff --git a/.github/workflows/plot-fct.py b/.github/workflows/plot-fct.py index 67b5f504..2c61b6f7 100644 --- a/.github/workflows/plot-fct.py +++ b/.github/workflows/plot-fct.py @@ -7,7 +7,7 @@ import matplotlib.pyplot as plt import numpy as np -# QUIC implementes +# QUIC implements IMPLS = ["tquic", "gquiche", "lsquic", "picoquic", "quiche"] # Different modes diff --git a/.github/workflows/plot-goodput.py b/.github/workflows/plot-goodput.py index b55dec6f..0e8ddcc1 100644 --- a/.github/workflows/plot-goodput.py +++ b/.github/workflows/plot-goodput.py @@ -7,7 +7,7 @@ import matplotlib.pyplot as plt import numpy as np -# QUIC implementes +# QUIC implements IMPLS = ["tquic", "gquiche", "lsquic", "picoquic", "quiche"] # Different file sizes diff --git a/.github/workflows/plot-interop.py b/.github/workflows/plot-interop.py index 1995ef29..eb308d42 100644 --- a/.github/workflows/plot-interop.py +++ b/.github/workflows/plot-interop.py @@ -8,7 +8,7 @@ import numpy as np from matplotlib.colors import ListedColormap -# QUIC implementes +# QUIC implements CLIENT_IMPLS = ["tquic", "lsquic", "quiche", "picoquic", "ngtcp2", "msquic", "s2n-quic", "quinn", "neqo", "kwik", "aioquic", "chrome", "go-x-net", "quic-go", "mvfst"] diff --git a/.github/workflows/tquic-benchmark.yml b/.github/workflows/tquic-benchmark.yml new file mode 100644 index 00000000..65470752 --- /dev/null +++ b/.github/workflows/tquic-benchmark.yml @@ -0,0 +1,221 @@ +name: Benchmark + +on: + schedule: + - cron: '30 3 * * *' + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +jobs: + build_tquic: + name: Build tquic + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Update rust + run: rustup update + - name: Build tquic + run: | + cargo build --all --release + cp target/release/tquic_server tquic_server + cp target/release/tquic_client tquic_client + - name: Build start script + run: | + echo $'#!/bin/bash\nchmod u+x ./tquic_server\n./tquic_server --send-udp-payload-size 1350 --log-level OFF --root ./ --disable-stateless-reset -l 127.0.0.1:4433 -c ./cert.crt -k ./cert.key &' > start_tquic.sh + chmod u+x start_tquic.sh + - name: Upload tquic_server + uses: actions/upload-artifact@v4 + with: + name: tquic_server_bin + path: | + tquic_server + start_tquic.sh + - name: Upload tquic_client + uses: actions/upload-artifact@v4 + with: + name: tquic_client_bin + path: tquic_client + + build_lsquic: + name: Build lsquic + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + repository: 'litespeedtech/lsquic' + path: lsquic + submodules: 'recursive' + - name: Build lsquic + run: | + git clone https://boringssl.googlesource.com/boringssl + cd boringssl + git checkout 9fc1c33e9c21439ce5f87855a6591a9324e569fd + cmake . && make + BORINGSSL=$PWD + cd ../lsquic + sudo apt install libevent-dev + cmake -DBORINGSSL_DIR=$BORINGSSL . + make + cp bin/http_server ../lsquic_server + - name: Build start script + run: | + echo $'#!/bin/bash\nchmod u+x ./lsquic_server\n./lsquic_server -c localhost,./cert.crt,./cert.key -s 127.0.0.1:4433 -r ./ -L crit > lsquic.log 2>&1 &' > start_lsquic.sh + chmod u+x start_lsquic.sh + - name: Upload lsquic server + uses: actions/upload-artifact@v4 + with: + name: lsquic_server_bin + path: | + lsquic_server + start_lsquic.sh + + gen_cert: + name: Generate cert + runs-on: ubuntu-latest + steps: + - name: Generate cert + run: | + openssl genrsa -out cert.key 2048 + openssl req -new -x509 -key cert.key -out cert.crt -days 365 -subj "/CN=tquic_benchmark" + - name: Upload cert + uses: actions/upload-artifact@v4 + with: + name: cert + path: cert.* + + gen_files: + name: Generate files + runs-on: ubuntu-latest + steps: + - name: Generate files + run: | + head -c 1K /dev/urandom > file_1K + head -c 15K /dev/urandom > file_15K + head -c 50K /dev/urandom > file_50K + head -c 2M /dev/urandom > file_2M + - name: Upload files + uses: actions/upload-artifact@v4 + with: + name: files + path: file_* + + run_long_conn: + name: Run long connection scenario benchmark + needs: [ build_tquic, build_lsquic, gen_cert, gen_files ] + runs-on: ubuntu-latest + strategy: + matrix: + server: [ tquic, lsquic ] + file: [ 15K, 50K, 2M ] + conn: [ 10 ] + stream: [ 1, 10 ] + steps: + - name: Download ${{ matrix.server }} server + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.server }}_server_bin + - name: Download cert + uses: actions/download-artifact@v4 + with: + name: cert + - name: Download files + uses: actions/download-artifact@v4 + with: + name: files + - name: Download tquic_client + uses: actions/download-artifact@v4 + with: + name: tquic_client_bin + - name: Start ${{ matrix.server }} server + run: | + sh start_${{ matrix.server }}.sh + pgrep ${{ matrix.server }}_server + - name: Benchmark ${{ matrix.server }} + run: | + chmod u+x ./tquic_client + ./tquic_client https://localhost:4433/file_${{ matrix.file }} --threads ${{ matrix.conn }} --max-concurrent-conns 1 --max-concurrent-requests ${{ matrix.stream }} --max-requests-per-conn 0 --total-requests-per-thread 0 -d 10 --disable-stateless-reset --send-batch-size 1 --recv-udp-payload-size 1350 --send-udp-payload-size 1350 --log-level OFF | grep "finished in" | awk '{prinit $4}' > benchmark_long_${{ matrix.server }}_${{ matrix.file }}_${{ matrix.conn }}_${{ matrix.stream }} + - name: Stop ${{ matrix.server }} server + run: | + killall ${{ matrix.server }}_server + sleep 1 + - name: Upload benchmark result + uses: actions/upload-artifact@v4 + with: + name: benchmark_long_${{ matrix.server }}_${{ matrix.file }}_${{ matrix.conn }}_${{ matrix.stream }} + path: benchmark_long_* + + run_short_conn: + name: Run short connection scenario benchmark + needs: [ build_tquic, build_lsquic, gen_cert, gen_files ] + runs-on: ubuntu-latest + strategy: + matrix: + server: [ tquic, lsquic ] + steps: + - name: Download ${{ matrix.server }} server + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.server }}_server_bin + - name: Download cert + uses: actions/download-artifact@v4 + with: + name: cert + - name: Download files + uses: actions/download-artifact@v4 + with: + name: files + - name: Download tquic_client + uses: actions/download-artifact@v4 + with: + name: tquic_client_bin + - name: Start ${{ matrix.server }} server + run: | + sh start_${{ matrix.server }}.sh + pgrep ${{ matrix.server }}_server + - name: Benchmark ${{ matrix.server }} + run: | + chmod u+x ./tquic_client + ./tquic_client https://localhost:4433/file_1K --threads 10 --max-concurrent-conns 1 --max-concurrent-requests 1 --max-requests-per-conn 1 --total-requests-per-thread 0 -d 10 --disable-stateless-reset --send-batch-size 1 --recv-udp-payload-size 1350 --send-udp-payload-size 1350 --log-level OFF | grep "finished in" | awk '{prinit $4}' > benchmark_short_${{ matrix.server }}_1K_10_1 + - name: Stop ${{ matrix.server }} server + run: | + killall ${{ matrix.server }}_server + sleep 1 + - name: Upload benchmark result + uses: actions/upload-artifact@v4 + with: + name: benchmark_short_${{ matrix.server }}_1K_${{ matrix.conn }}_1 + path: benchmark_short_* + + result: + runs-on: ubuntu-latest + needs: [ run_long_conn, run_short_conn ] + steps: + - name: Download all benchmark results + uses: actions/download-artifact@v4 + + - name: Display structure of downloaded files + run: ls -R + + - name: Download plot tools + uses: actions/checkout@v4 + with: + path: tools + + - name: Install dependencies + run: | + sudo apt install python3-matplotlib + pip3 install prettytable termcolor + + - name: Plot and print all benchmark results + run: python3 tools/.github/workflows/plot-benchmark.py . + + - name: Store all benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark_all + path: benchmark_all.png + diff --git a/.github/workflows/tquic-fct.yml b/.github/workflows/tquic-fct.yml index 1a9ff85c..acc26f0d 100644 --- a/.github/workflows/tquic-fct.yml +++ b/.github/workflows/tquic-fct.yml @@ -37,7 +37,7 @@ jobs: cd quic-interop-runner pip3 install -r requirements.txt - - name: Install dependences + - name: Install dependencies run: | sudo modprobe ip6table_filter sudo add-apt-repository -y ppa:wireshark-dev/stable @@ -76,7 +76,7 @@ jobs: with: path: tools - - name: Install dependences + - name: Install dependencies run: | sudo apt install python3-matplotlib diff --git a/.github/workflows/tquic-goodput.yml b/.github/workflows/tquic-goodput.yml index 20ef8a9b..7402fe99 100644 --- a/.github/workflows/tquic-goodput.yml +++ b/.github/workflows/tquic-goodput.yml @@ -36,7 +36,7 @@ jobs: cd quic-interop-runner pip3 install -r requirements.txt - - name: Install dependences + - name: Install dependencies run: | sudo modprobe ip6table_filter sudo add-apt-repository -y ppa:wireshark-dev/stable @@ -74,7 +74,7 @@ jobs: with: path: tools - - name: Install dependences + - name: Install dependencies run: | sudo apt install python3-matplotlib diff --git a/.github/workflows/tquic-interop-all.yml b/.github/workflows/tquic-interop-all.yml index 36aa9ab3..b818e932 100644 --- a/.github/workflows/tquic-interop-all.yml +++ b/.github/workflows/tquic-interop-all.yml @@ -95,7 +95,7 @@ jobs: cd quic-interop-runner pip3 install -r requirements.txt - - name: Install dependences + - name: Install dependencies run: | sudo modprobe ip6table_filter sudo add-apt-repository -y ppa:wireshark-dev/stable @@ -141,7 +141,7 @@ jobs: with: path: tools - - name: Install dependences + - name: Install dependencies run: | sudo apt install python3-matplotlib diff --git a/.github/workflows/tquic-interop-main.yaml b/.github/workflows/tquic-interop-main.yaml index 06beec80..ba93ca6f 100644 --- a/.github/workflows/tquic-interop-main.yaml +++ b/.github/workflows/tquic-interop-main.yaml @@ -42,7 +42,7 @@ jobs: cd quic-interop-runner pip3 install -r requirements.txt - - name: Install dependences + - name: Install dependencies run: | sudo modprobe ip6table_filter sudo add-apt-repository -y ppa:wireshark-dev/stable