diff --git a/.github/actions/runTestsTaggedAs/action.yaml b/.github/actions/runTestsTaggedAs/action.yaml index 0b449e091eb..47d2f447241 100644 --- a/.github/actions/runTestsTaggedAs/action.yaml +++ b/.github/actions/runTestsTaggedAs/action.yaml @@ -1,4 +1,4 @@ -name: 'Execute tests tagged in a certain way' +name: 'Execute tests' description: 'Execute tests suite for tests tagged as specified' inputs: tag: @@ -20,47 +20,31 @@ inputs: runs: using: "composite" steps: - - name: Setup java - uses: actions/setup-java@v4 - with: - distribution: 'zulu' - java-version: 11 - - name: Setup Node - uses: actions/setup-node@v4 # Installs Node and NPM - with: - node-version: 16 - - name: Install Swagger CLI # Installs Swagger CLI to bundle OpenAPI files - run: 'npm install -g @apidevtools/swagger-cli' - shell: bash - - name: Reuse cached maven artifacts dependencies - if: ${{ inputs.run-junit == 'false' }} - uses: actions/cache@v4 - with: - path: ~/.m2/repository - key: ${{ github.run_id }}-${{ github.run_number }}-maven-cache + - name: Set up runner + uses: ./.github/actions/setUpRunner + + - name: Set up Maven caches + uses: ./.github/actions/setUpMavenCaches + - name: Docker images creation if: ${{ inputs.needs-docker-images == 'true' }} run: mvn clean install -pl ${APP_PROJECTS} && mvn clean install -Pdocker -f assembly/pom.xml -pl '!:kapua-assembly-api' #api container not used in the tests at all se we don't need to build it here shell: bash + - name: Docker rest-api image creation if: ${{ inputs.needs-api-docker-image == 'true' }} run: mvn clean install -Pdocker -pl :kapua-assembly-api shell: bash - - name: Dns look-up containers needed for tests - message-broker - if: ${{ inputs.needs-docker-images == 'true' }} - run: echo "127.0.0.1 message-broker" | sudo tee -a /etc/hosts - shell: bash - - name: Dns look-up containers needed for tests - job-engine - if: ${{ inputs.needs-docker-images == 'true' }} - run: echo "127.0.0.1 job-engine" | sudo tee -a /etc/hosts - shell: bash + - name: Cucumber tests execution step if: ${{ inputs.run-junit == 'false' }} run: mvn -B -Dgroups='!org.eclipse.kapua.qa.markers.junit.JUnitTests' -Dcucumber.filter.tags="${{ inputs.tag }}" -pl ${TEST_PROJECTS} verify shell: bash + - name: Junit tests execution step if: ${{ inputs.run-junit == 'true' }} run: mvn -B -Dgroups='org.eclipse.kapua.qa.markers.junit.JUnitTests' verify shell: bash + - name: Code coverage results upload uses: codecov/codecov-action@v4 \ No newline at end of file diff --git a/.github/actions/saveBuiltKapuaArtifacts/action.yaml b/.github/actions/saveBuiltKapuaArtifacts/action.yaml new file mode 100644 index 00000000000..5dbb51bb56f --- /dev/null +++ b/.github/actions/saveBuiltKapuaArtifacts/action.yaml @@ -0,0 +1,17 @@ +name: 'Save built Kapua Artifacts' +description: | + Saves the built Kapua artifacts for later usage +runs: + using: "composite" + steps: + - name: Extract built Kapua artifacts # This splits the built Kapua artifact of this run from the cached repository of external dependencies for caching + run: | + mkdir --parents ~/.m2/kapua-repository/org/eclipse/ + mv ~/.m2/repository/org/eclipse/kapua ~/.m2/kapua-repository/org/eclipse/kapua + shell: bash + + - name: Save built Kapua artifacts + uses: actions/cache/save@v4 + with: + path: ~/.m2/kapua-repository/org/eclipse/kapua + key: ${{ runner.os }}-maven-${{ github.run_number }}-kapua-artifacts \ No newline at end of file diff --git a/.github/actions/setUpMavenCaches/action.yaml b/.github/actions/setUpMavenCaches/action.yaml new file mode 100644 index 00000000000..7563468ba36 --- /dev/null +++ b/.github/actions/setUpMavenCaches/action.yaml @@ -0,0 +1,31 @@ +name: 'Set Up Maven caches' +description: | + Set up maven caches to speedup build time and reuse built artifacts +inputs: + kapua-artifact-cache-enabled: + description: Whether to enable Kapua artifacts cache or not. If not enable you'll be required to build Kapua Artifacts on the runner + default: 'true' +runs: + using: "composite" + steps: + - name: Cache Maven repository - External dependencies # Cache of external Maven dependencies to speed up build time + id: cache-maven-external-deps + uses: actions/cache@v4 + with: + path: ~/.m2/repository/ + key: ${{ runner.os }}-maven-develop-dependencies + + - name: Cache Maven repository - Kapua artifacts # Cache of built Kapua artifacts be reused in other jobs + if: ${{ inputs.kapua-artifact-cache-enabled == 'true' }} + id: cache-maven-kapua-artifacts + uses: actions/cache/restore@v4 + with: + path: ~/.m2/kapua-repository/org/eclipse/kapua + key: ${{ runner.os }}-maven-${{ github.run_number }}-kapua-artifacts + fail-on-cache-miss: 'true' + + - name: Build full cached Maven repository # This adds the built Kapua artifact of this run to the cached repository of external dependencies. Used when re-running a job + if: ${{ inputs.kapua-artifact-cache-enabled == 'true' }} + run: mv ~/.m2/kapua-repository/org/eclipse/kapua ~/.m2/repository/org/eclipse/kapua + shell: bash + diff --git a/.github/actions/setUpRunner/action.yaml b/.github/actions/setUpRunner/action.yaml new file mode 100644 index 00000000000..1d3f119f6f6 --- /dev/null +++ b/.github/actions/setUpRunner/action.yaml @@ -0,0 +1,30 @@ +name: 'Set Up Runner' +description: | + Set up runner with tools required for the build + - Setup Java 11 + - Setup Node 16 + - Install Swagger CLI + - Add entries to /etc/hosts for tests +runs: + using: "composite" + steps: + - name: Setup Java 11 + uses: actions/setup-java@v4 + with: + distribution: 'zulu' + java-version: 11 + + - name: Setup Node 16 + uses: actions/setup-node@v4 + with: + node-version: 16 + + - name: Install Swagger CLI # Required to bundle OpenAPI files + run: 'npm install -g @apidevtools/swagger-cli' + shell: bash + + - name: Dns look-up Docker containers needed for tests + run: | + echo "127.0.0.1 message-broker" | sudo tee -a /etc/hosts + echo "127.0.0.1 job-engine" | sudo tee -a /etc/hosts + shell: bash \ No newline at end of file diff --git a/.github/workflows/kapua-ci.yaml b/.github/workflows/kapua-ci.yaml index 670bdb0e063..bb4c8038388 100755 --- a/.github/workflows/kapua-ci.yaml +++ b/.github/workflows/kapua-ci.yaml @@ -13,296 +13,360 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v4 # Checks out a copy of the repository on the ubuntu-latest machine - - uses: actions/setup-java@v4 + - name: Checkout repository # Checks out a copy of the repository on the runner + uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner + - name: Set up Maven caches + uses: ./.github/actions/setUpMavenCaches with: - distribution: 'zulu' - java-version: 11 - - uses: actions/setup-node@v4 # Installs Node and NPM + kapua-artifact-cache-enabled: 'false' + - name: Maven version + run: mvn --version + - name: Build Kapua project + run: mvn -B -DskipTests clean install -T 1C + - name: Save built Kapua Artifacts + uses: ./.github/actions/saveBuiltKapuaArtifacts + + build-javadoc: + needs: build + name: Build Javadoc + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout repository # Checks out a copy of the repository on the runner + uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner + - name: Set up Maven caches + uses: ./.github/actions/setUpMavenCaches with: - node-version: 16 - - name: Install Swagger CLI # Installs Swagger CLI to bundle OpenAPI files - run: 'npm install -g @apidevtools/swagger-cli' - - uses: actions/cache@v4 # Cache local Maven repository to reuse dependencies + kapua-artifact-cache-enabled: 'false' + - name: Build Kapua Javadoc + run: mvn -B -DskipTests install javadoc:jar + + junit-tests: + needs: build + name: Run jUnit Tests + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout repository # Checks out a copy of the repository on the runner + uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner + - uses: ./.github/actions/runTestsTaggedAs with: - path: ~/.m2/repository - key: ${{ github.run_id }}-${{ github.run_number }}-maven-cache - - run: mvn -v - - run: docker images -a # used as log (should show only GitHub environment standard docker images; if kapua images are present, something is wrong) - - run: mvn -B -DskipTests clean install -T 1C + needs-docker-images: 'false' + run-junit: 'true' + test-brokerAcl: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@brokerAcl' needs-docker-images: 'true' + test-tag: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@tag' needs-docker-images: 'false' + test-broker: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@broker' needs-docker-images: 'true' + test-device: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@device' needs-docker-images: 'true' + test-device-management: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@deviceManagement' needs-docker-images: 'true' + test-connection: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@connection' needs-docker-images: 'true' + test-datastore: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@datastore' needs-docker-images: 'true' + test-user: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@user' needs-docker-images: 'false' + test-userIntegrationBase: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@userIntegrationBase' needs-docker-images: 'true' + test-userIntegration: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@userIntegration' needs-docker-images: 'true' + test-security: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@security' needs-docker-images: 'false' + test-jobsAndScheduler: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '(@job or @scheduler) and not @it' needs-docker-images: 'false' + test-job-IT: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@job and @it' needs-docker-images: 'true' + test-jobEngine-IT: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@jobEngine' needs-docker-images: 'true' + test-jobsIntegration: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@jobsIntegration' needs-docker-images: 'true' + test-accountAndTranslator: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@account or @translator' needs-docker-images: 'false' + test-RoleAndGroup: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@role or @group' needs-docker-images: 'false' + test-deviceRegistry: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@deviceRegistry' needs-docker-images: 'true' + test-endpoint: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@endpoint' needs-docker-images: 'true' + test-api-auth: needs: build runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@rest_auth' needs-docker-images: 'true' needs-api-docker-image: 'true' + test-api-corsfilter: needs: test-endpoint # test suite dependent on the endpoint service (if it has failings it's useless to perform these tests) runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@rest_cors' needs-docker-images: 'true' needs-api-docker-image: 'true' + test-api-parsing: needs: test-api-auth runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Clones Kapua repo inside the runner + - name: Checkout repository # Checks out a copy of the repository on the runner uses: actions/checkout@v4 + - name: Set up runner + uses: ./.github/actions/setUpRunner - uses: ./.github/actions/runTestsTaggedAs with: tag: '@rest_parsing' needs-docker-images: 'true' - needs-api-docker-image: 'true' - junit-tests: - needs: build - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - name: Clones Kapua repo inside the runner - uses: actions/checkout@v4 - - uses: ./.github/actions/runTestsTaggedAs - with: - needs-docker-images: 'false' - run-junit: 'true' - build-javadoc: - needs: build - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: 'zulu' - java-version: 11 - - uses: actions/setup-node@v4 # Installs Node and NPM - with: - node-version: 16 - - name: Install Swagger CLI # Installs Swagger CLI to bundle OpenAPI files - run: 'npm install -g @apidevtools/swagger-cli' - - uses: actions/cache@v4 # Cache local Maven repository to reuse dependencies - with: - path: ~/.m2/repository - key: ${{ github.run_id }}-${{ github.run_number }}-maven-cache - - run: mvn -B -DskipTests install javadoc:jar + needs-api-docker-image: 'true' \ No newline at end of file diff --git a/.github/workflows/security-scan.yaml b/.github/workflows/security-scan.yaml index 8f7c3eb4efb..5f038402956 100755 --- a/.github/workflows/security-scan.yaml +++ b/.github/workflows/security-scan.yaml @@ -3,7 +3,6 @@ on: push: branches: - 'develop' - - 'release-**' env: BUILD_OPTS: "" @@ -14,22 +13,18 @@ jobs: owasp-dependency-check: name: Owasp Dependency Check runs-on: ubuntu-latest + timeout-minutes: 45 steps: - - uses: actions/checkout@v4 # Checks out a copy of the repository on the ubuntu-latest machine - - uses: actions/setup-java@v4 - with: - distribution: 'zulu' - java-version: 11 - cache: 'maven' - - uses: actions/setup-node@v4 # Installs Node and NPM - with: - node-version: 16 - - name: Install Swagger CLI # Installs Swagger CLI to bundle OpenAPI files - run: 'npm install -g @apidevtools/swagger-cli' - - uses: actions/cache@v4 # Cache local Maven repository to reuse dependencies + - name: Checkout repository # Checks out a copy of the repository on the runner + uses: actions/checkout@v4 + + - name: Set up runner + uses: ./.github/actions/setUpRunner + + - name: Set up Maven caches + uses: ./.github/actions/setUpMavenCaches with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - run: mvn -B ${BUILD_OPTS} -DskipTests -Psecurity-scan verify \ No newline at end of file + kapua-artifact-cache-enabled: 'false' + + - name: Run Owasp Security Scan + run: mvn -B ${BUILD_OPTS} -DskipTests -Psecurity-scan verify \ No newline at end of file diff --git a/.github/workflows/sonarCloud-scan.yaml b/.github/workflows/sonarCloud-scan.yaml index 5c1104b2b76..27dd2c44235 100644 --- a/.github/workflows/sonarCloud-scan.yaml +++ b/.github/workflows/sonarCloud-scan.yaml @@ -4,43 +4,33 @@ on: push: branches: - 'develop' - - 'release-**' pull_request: branches: - 'develop' - - 'release-**' jobs: build: name: Analyze runs-on: ubuntu-latest + timeout-minutes: 45 steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - java-version: 11 - distribution: 'zulu' - - name: Set up Node 16 - uses: actions/setup-node@v4 + - name: Checkout repository # Checks out a copy of the repository on the runner + uses: actions/checkout@v4 + + - name: Set up runner + uses: ./.github/actions/setUpRunner + + - name: Set up Maven caches + uses: ./.github/actions/setUpMavenCaches with: - node-version: 16 - - name: Install Swagger CLI # Installs Swagger CLI to bundle OpenAPI files - run: 'npm install -g @apidevtools/swagger-cli' + kapua-artifact-cache-enabled: 'false' + - name: Cache SonarQube packages uses: actions/cache@v4 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar - restore-keys: ${{ runner.os }}-sonar - - name: Cache Maven packages - uses: actions/cache@v4 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + - name: Build and analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}