chore(ci): various optimizations for build processing, caching and concurrency

.github/workflows/blackduck_scan_scheduled.yaml

@@ -23,6 +23,24 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version-file: '${{ github.workspace }}/go.mod'
+          cache: false
+
+      # This step will only reuse the go mod and build cache from main made during the Build,
+      # see push_ocm.yaml => "ocm-cli-latest" Job
+      # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
+      # This is because we have huge storage requirements for our cache because of the mass of dependencies
+      - name: Restore / Reuse Cache from central build
+        id: cache-golang-restore
+        uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
+        with:
+          path: |
+            ${{ env.go_cache }}
+            ${{ env.go_modcache }}
+          key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
+          restore-keys: |
+            ${{ env.cache_name }}-${{ runner.os }}-go-
+        env:
+          cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
 
       - name: Blackduck Full Scan
         uses: mercedesbenzio/detect-action@v2

.github/workflows/buildcomponents.yaml

@@ -36,16 +36,14 @@ jobs:
           echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
       - name: Set up cache
         # https://github.com/actions/setup-go/issues/358 - cache is not working with setup-go for multiple jobs
-        uses: actions/cache@v4
+        uses: actions/cache/restore@v4
         with:
           path: |
             ${{ env.go_cache }}
             ${{ env.go_modcache }}
-          key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
-          restore-keys: |
-            ${{ env.cache_name }}-${{ runner.os }}-go-
+          key: ${{ env.cache_name }}-${{ runner.os }}-go-
         env:
-          cache_name: buildcomponents-go-cache
+          cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
 
       - name: Push OCM Components
         if: inputs.ocm_push == true

.github/workflows/check_diff_action.yaml

@@ -21,9 +21,14 @@ jobs:
       run: |
         echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
         echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-    - name: Set up cache
-      # https://github.com/actions/setup-go/issues/358 - cache is not working with setup-go for multiple jobs
-      uses: actions/cache@v4
+
+    # This step will only reuse the go mod and build cache from main made during the Build,
+    # see lint_and_test.yaml => "test" Job
+    # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
+    # This is because we have huge storage requirements for our cache because of the mass of dependencies
+    - name: Restore / Reuse Cache from central build
+      id: cache-golang-restore
+      uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
       with:
         path: |
           ${{ env.go_cache }}
@@ -32,7 +37,7 @@ jobs:
         restore-keys: |
           ${{ env.cache_name }}-${{ runner.os }}-go-
       env:
-        cache_name: diff-check-go-cache
+        cache_name: run-tests-go-cache # needs to be the same key in the end as in the build step
 
     - name: Make generate and deepcopy
       run: |

.github/workflows/codeql.yml

@@ -27,7 +27,8 @@ jobs:
     #   - https://gh.io/supported-runners-and-hardware-resources
     #   - https://gh.io/using-larger-runners (GitHub.com only)
     # Consider using larger runners or machines with greater resources for possible analysis time improvements.
-    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    # - ADDENDUM: We moved this to a larger runner for faster analysis
+    runs-on: large_runner
     timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
     permissions:
       # required for all workflows
@@ -45,7 +46,7 @@ jobs:
       matrix:
         include:
         - language: go
-          build-mode: autobuild
+          build-mode: manual
         # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
         # Use `c-cpp` to analyze code written in C, C++ or both
         # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
@@ -61,7 +62,39 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v4
 
-    # Initializes the CodeQL tools for scanning.
+
+    - name: Setup Go
+      uses: actions/setup-go@v5
+      if: matrix.build-mode == 'manual'
+      with:
+        go-version-file: '${{ github.workspace }}/go.mod'
+        cache: false
+
+    - name: Get go environment for use with cache
+      if: matrix.build-mode == 'manual'
+      run: |
+        echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
+        echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
+
+      # This step will only reuse the go mod and build cache from main made during the Build,
+      # see push_ocm.yaml => "ocm-cli-latest" Job
+      # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
+      # This is because we have huge storage requirements for our cache because of the mass of dependencies
+    - name: Restore / Reuse Cache from central build
+      if: matrix.build-mode == 'manual'
+      id: cache-golang-restore
+      uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
+      with:
+        path: |
+          ${{ env.go_cache }}
+          ${{ env.go_modcache }}
+        key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
+        restore-keys: |
+          ${{ env.cache_name }}-${{ runner.os }}-go-
+      env:
+        cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
+
+      # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v3
       with:
@@ -74,20 +107,9 @@ jobs:
         # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
         # queries: security-extended,security-and-quality
 
-    # If the analyze step fails for one of the languages you are analyzing with
-    # "We were unable to automatically build your code", modify the matrix above
-    # to set the build mode to "manual" for that language. Then modify this step
-    # to build your code.
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-    - if: matrix.build-mode == 'manual'
-      run: |
-        echo 'If you are using a "manual" build mode for one or more of the' \
-          'languages you are analyzing, replace this with the commands to build' \
-          'your code, for example:'
-        echo '  make bootstrap'
-        echo '  make release'
-        exit 1
+    - name: Build
+      if: matrix.build-mode == 'manual'
+      run: make build -j
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v3

.github/workflows/components.yaml

@@ -28,9 +28,13 @@ jobs:
         run: |
           echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
           echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-      - name: Set up cache
-        # https://github.com/actions/setup-go/issues/358 - cache is not working with setup-go for multiple jobs
-        uses: actions/cache@v4
+      # This step will only reuse the go mod and build cache from main made during the Build,
+      # see push_ocm.yaml => "ocm-cli-latest" Job
+      # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
+      # This is because we have huge storage requirements for our cache because of the mass of dependencies
+      - name: Restore / Reuse Cache from central build
+        id: cache-golang-restore
+        uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
         with:
           path: |
             ${{ env.go_cache }}
@@ -39,7 +43,7 @@ jobs:
           restore-keys: |
             ${{ env.cache_name }}-${{ runner.os }}-go-
         env:
-          cache_name: cli-go-cache
+          cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
 
       - name: CTF
         run: |
@@ -62,9 +66,14 @@ jobs:
         run: |
           echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
           echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-      - name: Set up cache
-        # https://github.com/actions/setup-go/issues/358 - cache is not working with setup-go for multiple jobs
-        uses: actions/cache@v4
+
+      # This step will only reuse the go mod and build cache from main made during the Build,
+      # see push_ocm.yaml => "ocm-cli-latest" Job
+      # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
+      # This is because we have huge storage requirements for our cache because of the mass of dependencies
+      - name: Restore / Reuse Cache from central build
+        id: cache-golang-restore
+        uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
         with:
           path: |
             ${{ env.go_cache }}
@@ -73,7 +82,7 @@ jobs:
           restore-keys: |
             ${{ env.cache_name }}-${{ runner.os }}-go-
         env:
-          cache_name: helminstaller-go-cache
+          cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
 
       - name: CTF
         run: |
@@ -96,9 +105,14 @@ jobs:
         run: |
           echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
           echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-      - name: Set up cache
-        # https://github.com/actions/setup-go/issues/358 - cache is not working with setup-go for multiple jobs
-        uses: actions/cache@v4
+
+      # This step will only reuse the go mod and build cache from main made during the Build,
+      # see push_ocm.yaml => "ocm-cli-latest" Job
+      # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
+      # This is because we have huge storage requirements for our cache because of the mass of dependencies
+      - name: Restore / Reuse Cache from central build
+        id: cache-golang-restore
+        uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
         with:
           path: |
             ${{ env.go_cache }}
@@ -107,7 +121,7 @@ jobs:
           restore-keys: |
             ${{ env.cache_name }}-${{ runner.os }}-go-
         env:
-          cache_name: helmdemo-go-cache
+          cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
 
       - name: CTF
         run: |
@@ -130,9 +144,14 @@ jobs:
         run: |
           echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
           echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-      - name: Set up cache
-        # https://github.com/actions/setup-go/issues/358 - cache is not working with setup-go for multiple jobs
-        uses: actions/cache@v4
+
+      # This step will only reuse the go mod and build cache from main made during the Build,
+      # see push_ocm.yaml => "ocm-cli-latest" Job
+      # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
+      # This is because we have huge storage requirements for our cache because of the mass of dependencies
+      - name: Restore / Reuse Cache from central build
+        id: cache-golang-restore
+        uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
         with:
           path: |
             ${{ env.go_cache }}
@@ -141,7 +160,7 @@ jobs:
           restore-keys: |
             ${{ env.cache_name }}-${{ runner.os }}-go-
         env:
-          cache_name: helm-subchart-go-cache
+          cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
 
       - name: CTF
         run: |
@@ -167,9 +186,14 @@ jobs:
         run: |
           echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
           echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-      - name: Set up cache
-        # https://github.com/actions/setup-go/issues/358 - cache is not working with setup-go for multiple jobs
-        uses: actions/cache@v4
+
+      # This step will only reuse the go mod and build cache from main made during the Build,
+      # see push_ocm.yaml => "ocm-cli-latest" Job
+      # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
+      # This is because we have huge storage requirements for our cache because of the mass of dependencies
+      - name: Restore / Reuse Cache from central build
+        id: cache-golang-restore
+        uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
         with:
           path: |
             ${{ env.go_cache }}
@@ -178,7 +202,7 @@ jobs:
           restore-keys: |
             ${{ env.cache_name }}-${{ runner.os }}-go-
         env:
-          cache_name: ecr-plugin-go-cache
+          cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
 
       - name: CTF
         run: |

.github/workflows/lint_and_test.yaml

@@ -30,9 +30,17 @@ jobs:
         run: |
           echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
           echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-      - name: Set up cache
-        # https://github.com/actions/setup-go/issues/358 - cache is not working with setup-go for multiple jobs
-        uses: actions/cache@v4
+
+      # This step will only reuse the go mod and build cache from main made during the Build,
+      # see lint_and_test.yaml => "test" Job
+      # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
+      # This is because we have huge storage requirements for our cache because of the mass of dependencies
+      #
+      # NOTE: This is different from our regular build cache (which contains all archs and is built in a different job)
+      # This is because it requires caching of test dependencies, which are compiled only for linux-amd64 for test runs in CI.
+      - name: Restore / Reuse Cache from central build
+        id: cache-golang-restore
+        uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
         with:
           path: |
             ${{ env.go_cache }}
@@ -41,10 +49,27 @@ jobs:
           restore-keys: |
             ${{ env.cache_name }}-${{ runner.os }}-go-
         env:
-          cache_name: run-tests-go-cache
+          cache_name: run-tests-go-cache # needs to be the same key in the end as in the build step
 
+      - name: Build
+        run: make build -j
       - name: Test
-        run: make build install-requirements test
+        run: make install-requirements test
+
+      # NOTE: This is different from our regular build cache (which contains all archs and is built in a different job)
+      # This is because it requires caching of test dependencies, which are compiled only for linux-amd64 for test runs in CI.
+      - name: Save Cache of Build (only on main)
+        id: cache-golang-save
+        if: github.ref == 'refs/heads/main' # Only run on main, never in PR
+        uses: actions/cache/save@v4 # Only saves cache build-test (linux-amd64)
+        with:
+          path: |
+            ${{ env.go_cache }}
+            ${{ env.go_modcache }}
+          key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
+          upload-chunk-size: 256000000 # default of 32MB is not really optimal for our large cache, choose 256MB instead
+        env:
+          cache_name: run-tests-go-cache # needs to be the same key in the end as in the build step
 
   go-lint:
     name: Lint Golang
@@ -64,9 +89,14 @@ jobs:
         run: |
           echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
           echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-      - name: Set up cache
-        # https://github.com/actions/setup-go/issues/358 - cache is not working with setup-go for multiple jobs
-        uses: actions/cache@v4
+
+      # This step will only reuse the go mod and build cache from main made during the Build,
+      # see push_ocm.yaml => "ocm-cli-latest" Job
+      # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
+      # This is because we have huge storage requirements for our cache because of the mass of dependencies
+      - name: Restore / Reuse Cache from central build
+        id: cache-golang-restore
+        uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
         with:
           path: |
             ${{ env.go_cache }}
@@ -75,7 +105,7 @@ jobs:
           restore-keys: |
             ${{ env.cache_name }}-${{ runner.os }}-go-
         env:
-          cache_name: golint-go-cache
+          cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
 
       - name: Install goimports
         run: go install golang.org/x/tools/cmd/goimports@latest