chore(ci): run benchmarks for each storage in separate workers

.github/scripts/changed-modules.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+# How to test this script, run it with the required environment variables:
+# 1. A file from the .github is modified:
+#    ALL_CHANGED_FILES=".github/a.txt .github/b/c/d/a.yaml" ./.github/scripts/changed-modules.sh
+#    The output should be: all modules.
+#
+# 2. A file from a module in the modules dir is modified:
+#    ALL_CHANGED_FILES="arangodb/go.mod" ./.github/scripts/changed-modules.sh
+#    The output should be: just the arangodb module.
+#
+# 3. A file from two modules in the modules dir are modified:
+#    ALL_CHANGED_FILES="arangodb/go.mod redis/go.mod" ./.github/scripts/changed-modules.sh
+#    The output should be: the arangodb and redis modules.
+
+# ROOT_DIR is the root directory of the repository.
+readonly ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+
+# define an array of modules that won't be included in the list
+readonly excluded_modules=("mockstorage")
+
+# modules is an array that will store the paths of all the modules in the repository.
+modules=()
+
+function is_excluded() {
+    for excluded_module in "${excluded_modules[@]}"; do
+        if [[ $1 == $excluded_module ]]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+# Find all go.mod files in the repository, building a list of all the available modules.
+# Do not include the root go.mod file.
+for modFile in $(find "${ROOT_DIR}" -name "go.mod" -not -path "${ROOT_DIR}/go.mod" -not -path "${ROOT_DIR}/**/testdata/*"); do
+    # do not include the excluded modules
+    if is_excluded "$(basename "$(dirname "${modFile}")")"; then
+        continue
+    fi
+    modules+=("\"$(basename "$(dirname "${modFile}")")\"")
+done
+
+# sort modules array
+IFS=$'\n' modules=($(sort <<<"${modules[*]}"))
+unset IFS
+
+# merge all modules and examples into a single array
+allModules=("${modules[@]}")
+
+# sort allModules array
+IFS=$'\n' allModules=($(sort <<<"${allModules[*]}"))
+unset IFS
+
+# Get the list of modified files, retrieved from the environment variable ALL_CHANGED_FILES.
+# On CI, this value will come from a Github Action retrieving the list of modified files from the pull request.
+readonly modified_files=${ALL_CHANGED_FILES[@]}
+
+# Initialize variables
+modified_modules=()
+
+# Check the modified files and determine which modules to build, following these rules:
+# - if the modified files only contain files in one of the modules, include that module in the list
+# - if the modified files contain any other file, include all modules in the list
+for file in $modified_files; do
+    if [[ $file == .github/* ]]; then
+        modified_modules=${allModules[@]}
+        break
+    fi
+
+    module_name=$(echo $file | cut -d'/' -f1)
+    if [[ ! " ${modified_modules[@]} " =~ " ${module_name} " ]]; then
+        if is_excluded "$module_name"; then
+            continue
+        fi
+        modified_modules+=("\"$module_name\"")
+    fi
+done
+
+# print all modules with this format:
+# each module will be enclosed in double quotes
+# each module will be separated by a comma
+# the entire list will be enclosed in square brackets
+echo "["$(IFS=,; echo "${modified_modules[*]}" | sed 's/ /,/g')"]"

.github/workflows/benchmark.yml

@@ -17,66 +17,42 @@ permissions:
 
 name: Benchmark
 jobs:
-  Compare:
+  detect-modules:
     runs-on: ubuntu-latest
-    services:
-      arangodb:
-        image: 'arangodb:latest'
-        env:
-          ARANGO_NO_AUTH: 1
-        ports:
-          - '8529:8529'
-      dynamodb:
-        image: 'amazon/dynamodb-local:latest'
-        ports:
-          - '8000:8000'
-      memcached:
-        image: 'memcached:latest'
-        ports:
-          - '11211:11211'
-      mongo:
-        image: 'mongo:latest'
-        ports:
-          - '27017:27017'
-      mssql:
-        image: 'mcmoe/mssqldocker:latest'
-        ports:
-          - '1433:1433'
-        env:
-          ACCEPT_EULA: Y
-          SA_PASSWORD: MsSql!1234
-          MSSQL_DB: master
-          MSSQL_USER: sa
-          MSSQL_PASSWORD: MsSql!1234
-        options: >-
-          --health-cmd "/opt/mssql-tools/bin/sqlcmd -U sa -P $SA_PASSWORD -Q 'select 1' -b -o /dev/null"
-          --health-interval 1s
-          --health-timeout 30s
-          --health-start-period 10s
-          --health-retries 20
-      mysql:
-        image: 'mysql:latest'
-        env:
-          MYSQL_DATABASE: fiber
-          MYSQL_USER: username
-          MYSQL_PASSWORD: password
-          MYSQL_ROOT_PASSWORD: password
-        ports:
-          - '3306:3306'
-        options: >-
-          --health-cmd "mysqladmin ping" --health-interval 10s --health-timeout
-          5s --health-retries 5
-      postgres:
-        image: 'postgres:latest'
-        ports:
-          - '5432:5432'
+    outputs:
+      modules: ${{ steps.set-modified-modules.outputs.modules }}
+      modules_count: ${{ steps.set-modified-modules-count.outputs.modules_count }}
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+
+      - id: changed-files
+        name: Get changed files
+        uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45.0.4
+
+      - id: set-modified-modules
+        name: Set all modified modules
         env:
-          POSTGRES_DB: fiber
-          POSTGRES_USER: username
-          POSTGRES_PASSWORD: "pass#w%rd"
-        options: >-
-          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
-          --health-retries 5
+          ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}"
+        run: echo "modules=$(./.github/scripts/changed-modules.sh)" >> $GITHUB_OUTPUT
+
+      - id: set-modified-modules-count
+        name: Set all modified modules count
+        run: echo "modules_count=$(echo ${{ toJSON(steps.set-modified-modules.outputs.modules) }} | jq '. | length')" >> $GITHUB_OUTPUT
+
+      - name: Print out the modules to be used
+        run: |
+          echo "${{ steps.set-modified-modules-count.outputs.modules_count }} modules in the build"
+          echo "${{ steps.set-modified-modules.outputs.modules }}"
+
+  Compare:
+    needs:
+      - detect-modules
+    strategy:
+      matrix:
+        module: ${{ fromJSON(needs.detect-modules.outputs.modules) }}
+      fail-fast: false
+    runs-on: ubuntu-latest
 
     steps:
       - name: Fetch Repository
@@ -93,22 +69,81 @@ jobs:
         with:
           node-version: '18'
 
+      - name: Install ArangoDB
+        if: ${{ matrix.module == 'arangodb' }}
+        run: |
+          docker run -d -p 8529:8529 -e "ARANGO_NO_AUTH=1" arangodb:latest
+
+      - name: Install DynamoDB
+        if: ${{ matrix.module == 'dynamodb' }}
+        run: |
+          docker run -d -p 8000:8000 amazon/dynamodb-local:latest
+
+      - name: Install Memcached
+        if: ${{ matrix.module == 'memcache' }}
+        run: |
+          docker run -d -p 11211:11211 memcached:latest
+
+      - name: Install MongoDB
+        if: ${{ matrix.module == 'mongodb' }}
+        run: |
+          docker run -d -p 27017:27017 mongo:latest
+
+      - name: Install MSSQL
+        if: ${{ matrix.module == 'mssql' }}
+        run: |
+          docker run -d -p 1433:1433 \
+            -e "ACCEPT_EULA=Y" -e "SA_PASSWORD=MsSql!1234" \
+            -e "MSSQL_DB=master" -e "MSSQL_USER=sa" \
+            -e "MSSQL_PASSWORD=MsSql!1234" \
+            --health-cmd "/opt/mssql-tools/bin/sqlcmd -U sa -P $SA_PASSWORD -Q 'select 1' -b -o /dev/null" \
+            --health-interval 1s \
+            --health-timeout 30s \
+            --health-start-period 10s \
+            --health-retries 20 \
+            mcmoe/mssqldocker:latest
+
+      - name: Install MySQL
+        if: ${{ matrix.module == 'mysql' }}
+        run: |
+          docker run -d -p 3306:3306 \
+            -e "MYSQL_DATABASE=fiber" \
+            -e "MYSQL_USER=username" \
+            -e "MYSQL_PASSWORD=password" \
+            -e "MYSQL_ROOT_PASSWORD=password" \
+            --health-cmd "mysqladmin ping" --health-interval 10s --health-timeout 5s --health-retries 5 \
+            mysql:latest
+
+      - name: Install Postgres
+        if: ${{ matrix.module == 'postgres' }}
+        run: |
+          docker run -d -p 5432:5432 \
+            -e "POSTGRES_DB=fiber" \
+            -e "POSTGRES_USER=username" \
+            -e "POSTGRES_PASSWORD=pass#w%rd" \
+            --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 \
+            postgres:latest
+
       - name: Install Azurite
+        if: ${{ matrix.module == 'azureblob' }}
         run: |
           docker run -d -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0 --blobPort 10000
 
       - name: Install Cloudflare Worker
+        if: ${{ matrix.module == 'cloudflarekv' }}
         run : |
           .github/scripts/initialize-wrangler.sh
           cd cloudflarekv && npx wrangler dev &
           npx wait-on tcp:8787
 
       - name: Install Coherence
+        if: ${{ matrix.module == 'coherence' }}
         run: |
           docker run -d -p 1408:1408 -p 30000:30000 ghcr.io/oracle/coherence-ce:22.06.5
           sleep 30
 
       - name: Install etcd
+        if: ${{ matrix.module == 'etcd' }}
         run: |
           docker run -d --name Etcd-server \
             --publish 2379:2379 \
@@ -118,38 +153,36 @@ jobs:
             bitnami/etcd:latest
 
       - name: Install ScyllaDb
+        if: ${{ matrix.module == 'scylladb' }}
         run: |
           docker run --name scylladb -p 9042:9042 -p 19042:19042 -p 9160:9160 -p 7000:7000 -p 7001:7001 -p 7199:7199 -p 9180:9180 -d scylladb/scylla:latest --broadcast-address 127.0.0.1 --listen-address 0.0.0.0 --broadcast-rpc-address 127.0.0.1
           sleep 15 # Wait for ScyllaDb to initialize
 
       - name: Setup Redis
+        if: ${{ matrix.module == 'redis' || matrix.module == 'rueidis' }}
         uses: shogo82148/actions-setup-redis@v1
         with:
           redis-version: '7.x'
           auto-start: 'false'
 
       - name: Run Redis
+        if: ${{ matrix.module == 'redis' || matrix.module == 'rueidis' }}
         run: |
           redis-server --port 6379 &
 
       - name: Run NATS
+        if: ${{ matrix.module == 'nats' }}
         run: |
           ./.github/scripts/gen-test-certs.sh
           docker run -d --name nats-jetstream -p 4443:4443 -v ./nats/testdata:/testdata -v ./tls:/tls nats:latest --jetstream -c /testdata/nats-tls.conf
           sleep 2
 
       - name: Run Benchmarks
+        working-directory: ${{ matrix.module }}
         run: |
           set -o pipefail
-          for d in */ ; do
-            [[ $d == "tls/" ]] && continue
-            [[ $d == "node_modules/" ]] && continue
-
-            cd "$d"
-            echo "Bench dir: $d"
-            go test ./... -benchmem -run=^$ -bench . | tee -a ../output.txt
-            cd ..
-          done
+          echo "Bench dir: ${{ matrix.module }}"
+          go test ./... -benchmem -run=^$ -bench . | tee -a output.txt
         shell: bash
         env:
           MSSQL_DATABASE: master
@@ -169,15 +202,15 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ./cache
-          key: ${{ runner.os }}-benchmark
+          key: ${{ runner.os }}-benchmark-${{ matrix.module }}
 
       - name: Save Benchmark Results
         uses: benchmark-action/[email protected]
         with:
           tool: "go"
-          output-file-path: output.txt
+          output-file-path: ${{ matrix.module }}/output.txt
           github-token: ${{ secrets.BENCHMARK_TOKEN }}
-          benchmark-data-dir-path: "benchmarks"
+          benchmark-data-dir-path: "benchmarks/${{ matrix.module }}"
           alert-threshold: "300%"
           fail-on-alert: true
           comment-on-alert: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}

.github/workflows/test-s3.yml

@@ -15,12 +15,10 @@ jobs:
         strategy:
             matrix:
                 go-version:
-                    - 1.19.x
-                    - 1.20.x
                     - 1.21.x
+                    - 1.22.x
+                    - 1.23.x
         steps:
-        -   name: Install MinIO
-            run:  docker run -d --restart always -p 9000:9000 --name storage-minio -e MINIO_ROOT_USER='minio-user' -e MINIO_ROOT_PASSWORD='minio-password' minio/minio server /data
         -   name: Fetch Repository
             uses: actions/checkout@v4
         -   name: Install Go