Add benchmark workflow.

.github/workflows/plot-benchmark.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+import json
+import matplotlib.pyplot as plt
+import numpy as np
+import prettytable
+import termcolor
+
+# Benchmark scenarios
+SCENARIO = ["long", "short"]
+
+# QUIC implements
+IMPLS = ["tquic", "lsquic"]
+
+# File sizes in long connection scenario benchmark
+LONG_FILE_SIZES = ["15K", "50K", "2M"]
+
+# File sizes in short connection scenario benchmark
+SHORT_FILE_SIZES = ["1K"]
+
+# Different concurrent connections
+LONG_CONNS = [10]
+
+# Different concurrent connections
+SHORT_CONNS = [10]
+
+# Different concurrent streams
+LONG_STREAMS = [1, 10]
+
+# Different concurrent streams
+SHORT_STREAMS = [1]
+
+# Read data from benchmark result file.
+def read_data(data_dir, scen, impl, size, conn, stream):
+    dirname = "benchmark_%s_%s_%s_%d_%d" % (scen, impl, size, conn, stream)
+    filename = "benchmark_%s_%s_%s_%d_%d" % (scen, impl, size, conn, stream)
+    path = os.path.join(data_dir, dirname, filename)
+    try:
+        with open(path) as f:
+            data = f.read().strip()
+            return float(data)
+    except:
+        return 0.0
+
+# Put benchmark result in array according to implement.
+def prepare_data(data_dir):
+    titles = [' ' for _ in range((len(LONG_FILE_SIZES)*len(LONG_CONNS)*len(LONG_STREAMS) + len(SHORT_FILE_SIZES)*len(SHORT_CONNS)*len(SHORT_STREAMS)))]
+    result = [[0.0 for _ in range(len(LONG_FILE_SIZES)*len(LONG_CONNS)*len(LONG_STREAMS) + len(SHORT_FILE_SIZES)*len(SHORT_CONNS)*len(SHORT_STREAMS))] for _ in range(len(IMPLS))]
+
+
+    I = len(LONG_FILE_SIZES)
+    J = len(LONG_CONNS)
+    K = len(LONG_STREAMS)
+    N = len(IMPLS)
+    for i in range(I):
+        for j in range(J):
+            for k in range(K):
+                titles[i*J*K+j*K+k] = "long %s %d %d" % (LONG_FILE_SIZES[i], LONG_CONNS[j], LONG_STREAMS[k])
+                for n in range(N):
+                    result[n][i*J*K+j*K+k] = read_data(data_dir, "long", IMPLS[n], LONG_FILE_SIZES[i], LONG_CONNS[j], LONG_STREAMS[k])
+
+    M = len(LONG_FILE_SIZES)*len(LONG_CONNS)*len(LONG_STREAMS)
+    I = len(SHORT_FILE_SIZES)
+    J = len(SHORT_CONNS)
+    K = len(SHORT_STREAMS)
+    N = len(IMPLS)
+    for i in range(I):
+        for j in range(J):
+            for k in range(K):
+                titles[M+i*J*K+j*K+k] = "short %s %d %d" % (SHORT_FILE_SIZES[i], SHORT_CONNS[j], SHORT_STREAMS[k])
+                for n in range(N):
+                    result[n][M+i*J*K+j*K+k] = read_data(data_dir, "short", IMPLS[n], SHORT_FILE_SIZES[i], SHORT_CONNS[j], LONG_STREAMS[k])
+
+    return titles, result
+
+# Print benchmark performance result to stdout.
+def show(titles, result):
+    titles.insert(0, '')
+
+    table = prettytable.PrettyTable()
+    table.field_names = titles
+
+    for i in range(len(result)):
+        colored_row_name = termcolor.colored(IMPLS[i], 'green')
+        table.add_row([colored_row_name] + result[i])
+
+    print(table)
+
+def plot(titles, result):
+
+    N = len(titles)
+    M = len(result)
+
+    width = 0.35
+    gap = 0.5
+
+    ind = np.arange(N) * (width * M + gap)
+
+    fig, ax = plt.subplots()
+    fig.set_size_inches(10, 5)
+    for i in range(M):
+        ax.bar(ind + i*width, result[i], width, label=IMPLS[i])
+
+    ax.set_ylabel('RPS')
+    ax.set_title('TQUIC benchmark')
+    ax.set_xticks(ind + width * M / 2)
+    ax.set_xticklabels(titles, rotation=45)
+
+    ax.legend()
+
+    plt.savefig("benchmark_all.png", dpi=300)
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print("Usage: %s [data_dir]" % (sys.argv[0]))
+        exit(1)
+
+    data_dir= sys.argv[1]
+    titles, result = prepare_data(data_dir)
+    plot(titles, result)
+    show(titles, result)
+
+

.github/workflows/plot-fct.py

@@ -7,7 +7,7 @@
 import matplotlib.pyplot as plt
 import numpy as np
 
-# QUIC implementes
+# QUIC implements
 IMPLS = ["tquic", "gquiche", "lsquic", "picoquic", "quiche"]
 
 # Different modes

.github/workflows/plot-goodput.py

@@ -7,7 +7,7 @@
 import matplotlib.pyplot as plt
 import numpy as np
 
-# QUIC implementes
+# QUIC implements
 IMPLS = ["tquic", "gquiche", "lsquic", "picoquic", "quiche"]
 
 # Different file sizes

.github/workflows/plot-interop.py

@@ -8,7 +8,7 @@
 import numpy as np
 from matplotlib.colors import ListedColormap
 
-# QUIC implementes
+# QUIC implements
 CLIENT_IMPLS = ["tquic", "lsquic", "quiche", "picoquic", "ngtcp2", "msquic",
                 "s2n-quic", "quinn", "neqo", "kwik", "aioquic", "chrome",
                 "go-x-net", "quic-go", "mvfst"]

.github/workflows/tquic-benchmark.yml

@@ -0,0 +1,221 @@
+name: Benchmark
+
+on:
+  schedule:
+    - cron: '30 3 * * *'
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build_tquic:
+    name: Build tquic
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: 'recursive'
+    - name: Update rust
+      run: rustup update
+    - name: Build tquic
+      run: |
+        cargo build --all --release
+        cp target/release/tquic_server tquic_server
+        cp target/release/tquic_client tquic_client
+    - name: Build start script
+      run: |
+        echo $'#!/bin/bash\nchmod u+x ./tquic_server\n./tquic_server --send-udp-payload-size 1350 --log-level OFF --root ./ --disable-stateless-reset -l 127.0.0.1:4433 -c ./cert.crt -k ./cert.key &' > start_tquic.sh
+        chmod u+x start_tquic.sh
+    - name: Upload tquic_server
+      uses: actions/upload-artifact@v4
+      with:
+        name: tquic_server_bin
+        path: |
+          tquic_server
+          start_tquic.sh
+    - name: Upload tquic_client
+      uses: actions/upload-artifact@v4
+      with:
+        name: tquic_client_bin
+        path: tquic_client
+
+  build_lsquic:
+    name: Build lsquic
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        repository: 'litespeedtech/lsquic'
+        path: lsquic
+        submodules: 'recursive'
+    - name: Build lsquic
+      run: |
+        git clone https://boringssl.googlesource.com/boringssl
+        cd boringssl
+        git checkout 9fc1c33e9c21439ce5f87855a6591a9324e569fd
+        cmake . &&  make
+        BORINGSSL=$PWD
+        cd ../lsquic
+        sudo apt install libevent-dev
+        cmake -DBORINGSSL_DIR=$BORINGSSL .
+        make
+        cp bin/http_server ../lsquic_server
+    - name: Build start script
+      run: |
+        echo $'#!/bin/bash\nchmod u+x ./lsquic_server\n./lsquic_server -c localhost,./cert.crt,./cert.key -s 127.0.0.1:4433 -r ./ -L crit > lsquic.log 2>&1 &' > start_lsquic.sh
+        chmod u+x start_lsquic.sh
+    - name: Upload lsquic server
+      uses: actions/upload-artifact@v4
+      with:
+        name: lsquic_server_bin
+        path: |
+          lsquic_server
+          start_lsquic.sh
+
+  gen_cert:
+    name: Generate cert
+    runs-on: ubuntu-latest
+    steps:
+    - name: Generate cert
+      run: |
+        openssl genrsa -out cert.key 2048
+        openssl req -new -x509 -key cert.key -out cert.crt -days 365 -subj "/CN=tquic_benchmark"
+    - name: Upload cert
+      uses: actions/upload-artifact@v4
+      with:
+        name: cert
+        path: cert.*
+
+  gen_files:
+    name: Generate files
+    runs-on: ubuntu-latest
+    steps:
+    - name: Generate files
+      run: |
+        head -c 1K /dev/urandom > file_1K
+        head -c 15K /dev/urandom > file_15K
+        head -c 50K /dev/urandom > file_50K
+        head -c 2M /dev/urandom > file_2M
+    - name: Upload files
+      uses: actions/upload-artifact@v4
+      with:
+        name: files
+        path: file_*
+
+  run_long_conn:
+    name: Run long connection scenario benchmark
+    needs: [ build_tquic, build_lsquic, gen_cert, gen_files ]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        server: [ tquic, lsquic ]
+        file: [ 15K, 50K, 2M ]
+        conn: [ 10 ]
+        stream: [ 1, 10 ]
+    steps:
+    - name: Download ${{ matrix.server }} server
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ matrix.server }}_server_bin
+    - name: Download cert
+      uses: actions/download-artifact@v4
+      with:
+        name: cert
+    - name: Download files
+      uses: actions/download-artifact@v4
+      with:
+        name: files
+    - name: Download tquic_client
+      uses: actions/download-artifact@v4
+      with:
+        name: tquic_client_bin
+    - name: Start ${{ matrix.server }} server
+      run: |
+        sh start_${{ matrix.server }}.sh
+        pgrep ${{ matrix.server }}_server
+    - name: Benchmark ${{ matrix.server }}
+      run: |
+        chmod u+x ./tquic_client
+        ./tquic_client https://localhost:4433/file_${{ matrix.file }} --threads ${{ matrix.conn }} --max-concurrent-conns 1 --max-concurrent-requests ${{ matrix.stream }} --max-requests-per-conn 0 --total-requests-per-thread 0 -d 10 --disable-stateless-reset --send-batch-size 1 --recv-udp-payload-size 1350 --send-udp-payload-size 1350 --log-level OFF | grep "finished in" | awk '{prinit $4}' > benchmark_long_${{ matrix.server }}_${{ matrix.file }}_${{ matrix.conn }}_${{ matrix.stream }}
+    - name: Stop ${{ matrix.server }} server
+      run: |
+        killall ${{ matrix.server }}_server
+        sleep 1
+    - name: Upload benchmark result
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark_long_${{ matrix.server }}_${{ matrix.file }}_${{ matrix.conn }}_${{ matrix.stream }}
+        path: benchmark_long_*
+
+  run_short_conn:
+    name: Run short connection scenario benchmark
+    needs: [ build_tquic, build_lsquic, gen_cert, gen_files ]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        server: [ tquic, lsquic ]
+    steps:
+    - name: Download ${{ matrix.server }} server
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ matrix.server }}_server_bin
+    - name: Download cert
+      uses: actions/download-artifact@v4
+      with:
+        name: cert
+    - name: Download files
+      uses: actions/download-artifact@v4
+      with:
+        name: files
+    - name: Download tquic_client
+      uses: actions/download-artifact@v4
+      with:
+        name: tquic_client_bin
+    - name: Start ${{ matrix.server }} server
+      run: |
+        sh start_${{ matrix.server }}.sh
+        pgrep ${{ matrix.server }}_server
+    - name: Benchmark ${{ matrix.server }}
+      run: |
+        chmod u+x ./tquic_client
+        ./tquic_client https://localhost:4433/file_1K --threads 10 --max-concurrent-conns 1 --max-concurrent-requests 1 --max-requests-per-conn 1 --total-requests-per-thread 0 -d 10 --disable-stateless-reset --send-batch-size 1 --recv-udp-payload-size 1350 --send-udp-payload-size 1350 --log-level OFF | grep "finished in" | awk '{prinit $4}' > benchmark_short_${{ matrix.server }}_1K_10_1
+    - name: Stop ${{ matrix.server }} server
+      run: |
+        killall ${{ matrix.server }}_server
+        sleep 1
+    - name: Upload benchmark result
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark_short_${{ matrix.server }}_1K_${{ matrix.conn }}_1
+        path: benchmark_short_*
+
+  result:
+    runs-on: ubuntu-latest
+    needs: [ run_long_conn, run_short_conn ]
+    steps:
+    - name: Download all benchmark results
+      uses: actions/download-artifact@v4
+
+    - name: Display structure of downloaded files
+      run: ls -R
+
+    - name: Download plot tools
+      uses: actions/checkout@v4
+      with:
+        path: tools
+
+    - name: Install dependencies
+      run: |
+        sudo apt install python3-matplotlib
+        pip3 install prettytable termcolor
+
+    - name: Plot and print all benchmark results
+      run: python3 tools/.github/workflows/plot-benchmark.py .
+
+    - name: Store all benchmark results
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark_all
+        path: benchmark_all.png
+

.github/workflows/tquic-fct.yml

@@ -37,7 +37,7 @@ jobs:
         cd quic-interop-runner
         pip3 install -r requirements.txt
 
-    - name: Install dependences
+    - name: Install dependencies
       run: |
         sudo modprobe ip6table_filter
         sudo add-apt-repository -y ppa:wireshark-dev/stable
@@ -76,7 +76,7 @@ jobs:
       with:
         path: tools
 
-    - name: Install dependences
+    - name: Install dependencies
       run: |
         sudo apt install python3-matplotlib