♻️ [CI] Improved chaching of dependencies for Kapua CI workflow

Signed-off-by: Alberto Codutti <[email protected]>
eclipse-kapua · Jan 23, 2025 · 58ef407 · 58ef407
1 parent fc1a7aa
commit 58ef407
Show file tree

Hide file tree

Showing 2 changed files with 88 additions and 22 deletions.
diff --git a/.github/actions/runTestsTaggedAs/action.yaml b/.github/actions/runTestsTaggedAs/action.yaml
@@ -20,47 +20,67 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Setup java
+    - name: Setup Java 11
       uses: actions/setup-java@v4
       with:
         distribution: 'zulu'
         java-version: 11
-    - name: Setup Node
+
+    - name: Setup Node 16
       uses: actions/setup-node@v4 # Installs Node and NPM
       with:
         node-version: 16
+
     - name: Install Swagger CLI # Installs Swagger CLI to bundle OpenAPI files
       run: 'npm install -g @apidevtools/swagger-cli'
       shell: bash
-    - name: Reuse cached maven artifacts dependencies
-      if: ${{ inputs.run-junit == 'false' }}
+
+    - name: Cache Maven repository - External dependencies # Cache of external Maven dependencies built from 'build' job
+      uses: actions/cache@v4
+      with:
+        path: ~/.m2/repository/
+        key: ${{ runner.os }}-maven-develop-dependencies
+
+    - name: Cache Maven repository - Kapua artifacts # Cache of Kapua artifacts built from 'build' job
       uses: actions/cache@v4
       with:
-        path: ~/.m2/repository
-        key: ${{ github.run_id }}-${{ github.run_number }}-maven-cache
+        path: ~/.m2/kapua-repository/org/eclipse/kapua
+        key: ${{ runner.os }}-maven-${{ github.run_number }}-kapua-artifacts
+        fail-on-cache-miss: true
+
+    - name: Build full cached Maven repository # This adds the built Kapua artifact of this run to the cached repository of external dependencies
+      run: mv ~/.m2/kapua-repository/org/eclipse/kapua ~/.m2/repository/org/eclipse/kapua
+      shell: bash
+
     - name: Docker images creation
       if: ${{ inputs.needs-docker-images == 'true' }}
       run: mvn clean install -pl ${APP_PROJECTS} && mvn clean install -Pdocker -f assembly/pom.xml -pl '!:kapua-assembly-api' #api container not used in the tests at all se we don't need to build it here
       shell: bash
+
     - name: Docker rest-api image creation
       if: ${{ inputs.needs-api-docker-image == 'true' }}
       run: mvn clean install -Pdocker -pl :kapua-assembly-api
       shell: bash
+
     - name: Dns look-up containers needed for tests - message-broker
       if: ${{ inputs.needs-docker-images == 'true' }}
       run: echo "127.0.0.1       message-broker" | sudo tee -a /etc/hosts
       shell: bash
+
     - name: Dns look-up containers needed for tests - job-engine
       if: ${{ inputs.needs-docker-images == 'true' }}
       run: echo "127.0.0.1       job-engine" | sudo tee -a /etc/hosts
       shell: bash
+
     - name: Cucumber tests execution step
       if: ${{ inputs.run-junit == 'false' }}
       run: mvn -B -Dgroups='!org.eclipse.kapua.qa.markers.junit.JUnitTests' -Dcucumber.filter.tags="${{ inputs.tag }}" -pl ${TEST_PROJECTS} verify
       shell: bash
+
     - name: Junit tests execution step
       if: ${{ inputs.run-junit == 'true' }}
       run: mvn -B -Dgroups='org.eclipse.kapua.qa.markers.junit.JUnitTests' verify
       shell: bash
+
     - name: Code coverage results upload
       uses: codecov/codecov-action@v4
diff --git a/.github/workflows/kapua-ci.yaml b/.github/workflows/kapua-ci.yaml
@@ -13,23 +13,57 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 45
     steps:
-      - uses: actions/checkout@v4 # Checks out a copy of the repository on the ubuntu-latest machine
-      - uses: actions/setup-java@v4
+      - name: Checkout repository # Checks out a copy of the repository on the worker
+        uses: actions/checkout@v4
+
+      - name: Setup Java 11
+        uses: actions/setup-java@v4
         with:
           distribution: 'zulu'
           java-version: 11
-      - uses: actions/setup-node@v4 # Installs Node and NPM
+
+      - name: Setup Node 16 # Installs Node and NPM
+        uses: actions/setup-node@v4
         with:
           node-version: 16
+
       - name: Install Swagger CLI # Installs Swagger CLI to bundle OpenAPI files
         run: 'npm install -g @apidevtools/swagger-cli'
-      - uses: actions/cache@v4 # Cache local Maven repository to reuse dependencies
+        shell: bash
+
+      - name: Cache Maven repository - External dependencies # Cache of external Maven dependencies to speed up build time
+        id: cache-maven-external-deps
+        uses: actions/cache@v4
         with:
-          path: ~/.m2/repository
-          key: ${{ github.run_id }}-${{ github.run_number }}-maven-cache
-      - run: mvn -v
-      - run: docker images -a  # used as log (should show only GitHub environment standard docker images; if kapua images are present, something is wrong)
-      - run: mvn -B -DskipTests clean install -T 1C
+          path: ~/.m2/repository/
+          key: ${{ runner.os }}-maven-develop-dependencies
+
+      - name: Cache Maven repository - Kapua artifacts # Cache of Kapua artifacts be reused in following jobs
+        id: cache-maven-kapua-artifacts
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/kapua-repository/org/eclipse/kapua
+          key: ${{ runner.os }}-maven-${{ github.run_number }}-kapua-artifacts
+
+      - name: Build full cached Maven repository # This adds the built Kapua artifact of this run to the cached repository of external dependencies. Used when re-running a job
+        if: steps.cache-maven-kapua-artifacts.outputs.cache-hit == 'true'
+        run: mv ~/.m2/kapua-repository/org/eclipse/kapua ~/.m2/repository/org/eclipse/kapua
+
+      - name: Maven version
+        run: mvn --version
+
+      - name: Build Kapua project
+        run: mvn -B -DskipTests clean install -T 1C
+
+      - name: Download all dependencies of the project # Downloading all referenced dependencies to build a complete cache if not present
+        if: steps.cache-maven-external-deps.outputs.cache-hit != 'true'
+        run: mvn dependency:go-offline
+
+      - name: Extract built Kapua artifacts # This splits the built Kapua artifact of this run from the cached repository of external dependencies for caching
+        run: |
+          mkdir --parents ~/.m2/kapua-repository/org/eclipse/
+          mv ~/.m2/repository/org/eclipse/kapua ~/.m2/kapua-repository/org/eclipse/kapua
+
   test-brokerAcl:
     needs: build
     runs-on: ubuntu-latest
@@ -291,18 +325,30 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 45
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-java@v4
+      - name: Clones Kapua repo inside the runner
+        uses: actions/checkout@v4
+
+      - name: Setup Java 11
+        uses: actions/setup-java@v4
         with:
           distribution: 'zulu'
           java-version: 11
-      - uses: actions/setup-node@v4 # Installs Node and NPM
+
+      - name: Setup Node 16
+        uses: actions/setup-node@v4 # Installs Node and NPM
         with:
           node-version: 16
+
       - name: Install Swagger CLI # Installs Swagger CLI to bundle OpenAPI files
         run: 'npm install -g @apidevtools/swagger-cli'
-      - uses: actions/cache@v4 # Cache local Maven repository to reuse dependencies
+        shell: bash
+
+      - name: Cache Maven repository - External dependencies # Cache of external Maven dependencies to speed up build time
+        id: cache-maven-external-deps
+        uses: actions/cache@v4
         with:
-          path: ~/.m2/repository
-          key: ${{ github.run_id }}-${{ github.run_number }}-maven-cache
-      - run: mvn -B -DskipTests install javadoc:jar
+          path: ~/.m2/repository/
+          key: ${{ runner.os }}-maven-develop-dependencies
+
+      - name: Build Kapua Javadoc
+        run: mvn -B -DskipTests install javadoc:jar