From 58ef407e96568454495d0f81a715c370ac6be242 Mon Sep 17 00:00:00 2001 From: Alberto Codutti Date: Wed, 22 Jan 2025 17:55:12 +0100 Subject: [PATCH] :recycle: [CI] Improved chaching of dependencies for Kapua CI workflow Signed-off-by: Alberto Codutti --- .github/actions/runTestsTaggedAs/action.yaml | 32 ++++++-- .github/workflows/kapua-ci.yaml | 78 ++++++++++++++++---- 2 files changed, 88 insertions(+), 22 deletions(-) diff --git a/.github/actions/runTestsTaggedAs/action.yaml b/.github/actions/runTestsTaggedAs/action.yaml index 0b449e091eb..c916b7a2f98 100644 --- a/.github/actions/runTestsTaggedAs/action.yaml +++ b/.github/actions/runTestsTaggedAs/action.yaml @@ -20,47 +20,67 @@ inputs: runs: using: "composite" steps: - - name: Setup java + - name: Setup Java 11 uses: actions/setup-java@v4 with: distribution: 'zulu' java-version: 11 - - name: Setup Node + + - name: Setup Node 16 uses: actions/setup-node@v4 # Installs Node and NPM with: node-version: 16 + - name: Install Swagger CLI # Installs Swagger CLI to bundle OpenAPI files run: 'npm install -g @apidevtools/swagger-cli' shell: bash - - name: Reuse cached maven artifacts dependencies - if: ${{ inputs.run-junit == 'false' }} + + - name: Cache Maven repository - External dependencies # Cache of external Maven dependencies built from 'build' job + uses: actions/cache@v4 + with: + path: ~/.m2/repository/ + key: ${{ runner.os }}-maven-develop-dependencies + + - name: Cache Maven repository - Kapua artifacts # Cache of Kapua artifacts built from 'build' job uses: actions/cache@v4 with: - path: ~/.m2/repository - key: ${{ github.run_id }}-${{ github.run_number }}-maven-cache + path: ~/.m2/kapua-repository/org/eclipse/kapua + key: ${{ runner.os }}-maven-${{ github.run_number }}-kapua-artifacts + fail-on-cache-miss: true + + - name: Build full cached Maven repository # This adds the built Kapua artifact of this run to the cached repository of external dependencies + run: mv ~/.m2/kapua-repository/org/eclipse/kapua ~/.m2/repository/org/eclipse/kapua + shell: bash + - name: Docker images creation if: ${{ inputs.needs-docker-images == 'true' }} run: mvn clean install -pl ${APP_PROJECTS} && mvn clean install -Pdocker -f assembly/pom.xml -pl '!:kapua-assembly-api' #api container not used in the tests at all se we don't need to build it here shell: bash + - name: Docker rest-api image creation if: ${{ inputs.needs-api-docker-image == 'true' }} run: mvn clean install -Pdocker -pl :kapua-assembly-api shell: bash + - name: Dns look-up containers needed for tests - message-broker if: ${{ inputs.needs-docker-images == 'true' }} run: echo "127.0.0.1 message-broker" | sudo tee -a /etc/hosts shell: bash + - name: Dns look-up containers needed for tests - job-engine if: ${{ inputs.needs-docker-images == 'true' }} run: echo "127.0.0.1 job-engine" | sudo tee -a /etc/hosts shell: bash + - name: Cucumber tests execution step if: ${{ inputs.run-junit == 'false' }} run: mvn -B -Dgroups='!org.eclipse.kapua.qa.markers.junit.JUnitTests' -Dcucumber.filter.tags="${{ inputs.tag }}" -pl ${TEST_PROJECTS} verify shell: bash + - name: Junit tests execution step if: ${{ inputs.run-junit == 'true' }} run: mvn -B -Dgroups='org.eclipse.kapua.qa.markers.junit.JUnitTests' verify shell: bash + - name: Code coverage results upload uses: codecov/codecov-action@v4 \ No newline at end of file diff --git a/.github/workflows/kapua-ci.yaml b/.github/workflows/kapua-ci.yaml index 670bdb0e063..aaf588b69b0 100755 --- a/.github/workflows/kapua-ci.yaml +++ b/.github/workflows/kapua-ci.yaml @@ -13,23 +13,57 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v4 # Checks out a copy of the repository on the ubuntu-latest machine - - uses: actions/setup-java@v4 + - name: Checkout repository # Checks out a copy of the repository on the worker + uses: actions/checkout@v4 + + - name: Setup Java 11 + uses: actions/setup-java@v4 with: distribution: 'zulu' java-version: 11 - - uses: actions/setup-node@v4 # Installs Node and NPM + + - name: Setup Node 16 # Installs Node and NPM + uses: actions/setup-node@v4 with: node-version: 16 + - name: Install Swagger CLI # Installs Swagger CLI to bundle OpenAPI files run: 'npm install -g @apidevtools/swagger-cli' - - uses: actions/cache@v4 # Cache local Maven repository to reuse dependencies + shell: bash + + - name: Cache Maven repository - External dependencies # Cache of external Maven dependencies to speed up build time + id: cache-maven-external-deps + uses: actions/cache@v4 with: - path: ~/.m2/repository - key: ${{ github.run_id }}-${{ github.run_number }}-maven-cache - - run: mvn -v - - run: docker images -a # used as log (should show only GitHub environment standard docker images; if kapua images are present, something is wrong) - - run: mvn -B -DskipTests clean install -T 1C + path: ~/.m2/repository/ + key: ${{ runner.os }}-maven-develop-dependencies + + - name: Cache Maven repository - Kapua artifacts # Cache of Kapua artifacts be reused in following jobs + id: cache-maven-kapua-artifacts + uses: actions/cache@v4 + with: + path: ~/.m2/kapua-repository/org/eclipse/kapua + key: ${{ runner.os }}-maven-${{ github.run_number }}-kapua-artifacts + + - name: Build full cached Maven repository # This adds the built Kapua artifact of this run to the cached repository of external dependencies. Used when re-running a job + if: steps.cache-maven-kapua-artifacts.outputs.cache-hit == 'true' + run: mv ~/.m2/kapua-repository/org/eclipse/kapua ~/.m2/repository/org/eclipse/kapua + + - name: Maven version + run: mvn --version + + - name: Build Kapua project + run: mvn -B -DskipTests clean install -T 1C + + - name: Download all dependencies of the project # Downloading all referenced dependencies to build a complete cache if not present + if: steps.cache-maven-external-deps.outputs.cache-hit != 'true' + run: mvn dependency:go-offline + + - name: Extract built Kapua artifacts # This splits the built Kapua artifact of this run from the cached repository of external dependencies for caching + run: | + mkdir --parents ~/.m2/kapua-repository/org/eclipse/ + mv ~/.m2/repository/org/eclipse/kapua ~/.m2/kapua-repository/org/eclipse/kapua + test-brokerAcl: needs: build runs-on: ubuntu-latest @@ -291,18 +325,30 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - name: Clones Kapua repo inside the runner + uses: actions/checkout@v4 + + - name: Setup Java 11 + uses: actions/setup-java@v4 with: distribution: 'zulu' java-version: 11 - - uses: actions/setup-node@v4 # Installs Node and NPM + + - name: Setup Node 16 + uses: actions/setup-node@v4 # Installs Node and NPM with: node-version: 16 + - name: Install Swagger CLI # Installs Swagger CLI to bundle OpenAPI files run: 'npm install -g @apidevtools/swagger-cli' - - uses: actions/cache@v4 # Cache local Maven repository to reuse dependencies + shell: bash + + - name: Cache Maven repository - External dependencies # Cache of external Maven dependencies to speed up build time + id: cache-maven-external-deps + uses: actions/cache@v4 with: - path: ~/.m2/repository - key: ${{ github.run_id }}-${{ github.run_number }}-maven-cache - - run: mvn -B -DskipTests install javadoc:jar + path: ~/.m2/repository/ + key: ${{ runner.os }}-maven-develop-dependencies + + - name: Build Kapua Javadoc + run: mvn -B -DskipTests install javadoc:jar