CI testing improvements (#2734)

* add CI files Signed-off-by: chipzoller <[email protected]> * add CI to dev doc Signed-off-by: chipzoller <[email protected]> * expand chart test Signed-off-by: chipzoller <[email protected]> * add new values for testing Signed-off-by: chipzoller <[email protected]> * make linter happy Signed-off-by: Chip Zoller <[email protected]> * use minimal aggregator value set Signed-off-by: chipzoller <[email protected]> * lint Signed-off-by: chipzoller <[email protected]> --------- Signed-off-by: chipzoller <[email protected]> Signed-off-by: Chip Zoller <[email protected]>
kubecost · Nov 27, 2023 · 2564343 · 2564343
1 parent 57befe7
commit 2564343
Show file tree

Hide file tree

Showing 9 changed files with 186 additions and 11 deletions.
diff --git a/.github/ci-files/README b/.github/ci-files/README
@@ -0,0 +1 @@
+Files in this directory are used as dependencies for CI tests.
diff --git a/.github/ci-files/cloudIntegrationSecret.yaml b/.github/ci-files/cloudIntegrationSecret.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+data:
+  cloud-integration.json: ewogICAgImF3cyI6IFsKICAgICAgICB7CiAgICAgICAgICAgICJhdGhlbmFCdWNrZXROYW1lIjogInMzOi8vQVdTX2Nsb3VkX2ludGVncmF0aW9uX2F0aGVuYUJ1Y2tldE5hbWUiLAogICAgICAgICAgICAiYXRoZW5hUmVnaW9uIjogIkFXU19jbG91ZF9pbnRlZ3JhdGlvbl9hdGhlbmFSZWdpb24iLAogICAgICAgICAgICAiYXRoZW5hRGF0YWJhc2UiOiAiQVdTX2Nsb3VkX2ludGVncmF0aW9uX2F0aGVuYURhdGFiYXNlIiwKICAgICAgICAgICAgImF0aGVuYVRhYmxlIjogIkFXU19jbG91ZF9pbnRlZ3JhdGlvbl9hdGhlbmFCdWNrZXROYW1lIiwKICAgICAgICAgICAgInByb2plY3RJRCI6ICJBV1NfY2xvdWRfaW50ZWdyYXRpb25fYXRoZW5hX3Byb2plY3RJRCIsCiAgICAgICAgICAgICJzZXJ2aWNlS2V5TmFtZSI6ICJBV1NfY2xvdWRfaW50ZWdyYXRpb25fYXRoZW5hX3NlcnZpY2VLZXlOYW1lIiwKICAgICAgICAgICAgInNlcnZpY2VLZXlTZWNyZXQiOiAiQVdTX2Nsb3VkX2ludGVncmF0aW9uX2F0aGVuYV9zZXJ2aWNlS2V5U2VjcmV0IgogICAgICAgIH0KICAgIF0KfQ==
+kind: Secret
+metadata:
+  name: cloud-integration
+  namespace: kubecost
+type: Opaque
diff --git a/.github/ci-files/federatedStorageConfigSecret.yaml b/.github/ci-files/federatedStorageConfigSecret.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+data:
+  federated-store.yaml: dHlwZTogUzMKY29uZmlnOgogIGJ1Y2tldDogImt1YmVjb3N0LVVOSVFVRV9OQU1FLW1ldHJpY3MiCiAgZW5kcG9pbnQ6ICJzMy5hbWF6b25hd3MuY29tIgogIHJlZ2lvbjogInVzLWVhc3QtMiIKICAjIHVzZSBpcnNhIHdoZW4gcG9zc2libGUtIG90aGVyd2lzZSBhY2Nlc3Mga2V5cyBjYW4gYmUgdXNlZDoKICAjIGFjY2Vzc19rZXk6ICJZT1VSX0tFWSIKICAjIHNlY3JldF9rZXk6ICJZT1VSX1NFQ1JFVCIKICBpbnNlY3VyZTogZmFsc2UKICBzaWduYXR1cmVfdmVyc2lvbjI6IGZhbHNlCiAgcHV0X3VzZXJfbWV0YWRhdGE6CiAgICAgICJYLUFtei1BY2wiOiAiYnVja2V0LW93bmVyLWZ1bGwtY29udHJvbCIKICBodHRwX2NvbmZpZzoKICAgIGlkbGVfY29ubl90aW1lb3V0OiA5MHMKICAgIHJlc3BvbnNlX2hlYWRlcl90aW1lb3V0OiAybQogICAgaW5zZWN1cmVfc2tpcF92ZXJpZnk6IGZhbHNlCiAgdHJhY2U6CiAgICBlbmFibGU6IHRydWUKICBwYXJ0X3NpemU6IDEzNDIxNzcyOA==
+kind: Secret
+metadata:
+  name: federated-store
+  namespace: kubecost
+type: Opaque
diff --git a/.github/ci-files/kubecost-thanos.yaml b/.github/ci-files/kubecost-thanos.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+data:
+  object-store.yaml: dHlwZTogUzMKY29uZmlnOgogIGJ1Y2tldDogIkFXU19vYmplY3Rfc3RvcmVfYnVja2V0IgogIGVuZHBvaW50OiAiczMuYW1hem9uYXdzLmNvbSIKICByZWdpb246IHVzLWVhc3QtMQogIGFjY2Vzc19rZXk6ICJBV1Nfb2JqZWN0X3N0b3JlX2FjY2Vzc19rZXkiCiAgaW5zZWN1cmU6IGZhbHNlCiAgc2lnbmF0dXJlX3ZlcnNpb24yOiBmYWxzZQogIHNlY3JldF9rZXk6ICJBV1Nfb2JqZWN0X3N0b3JlX3NlY3JldF9rZXkiCiAgcHV0X3VzZXJfbWV0YWRhdGE6CiAgICAgICJYLUFtei1BY2wiOiAiYnVja2V0LW93bmVyLWZ1bGwtY29udHJvbCIKICBodHRwX2NvbmZpZzoKICAgIGlkbGVfY29ubl90aW1lb3V0OiA5MHMKICAgIHJlc3BvbnNlX2hlYWRlcl90aW1lb3V0OiAybQogICAgaW5zZWN1cmVfc2tpcF92ZXJpZnk6IGZhbHNlCiAgdHJhY2U6CiAgICBlbmFibGU6IHRydWUKICBwYXJ0X3NpemU6IDEzNDIxNzcyOA==
+kind: Secret
+metadata:
+  name: kubecost-thanos
+  namespace: kubecost
+type: Opaque
diff --git a/.github/workflows/chart.yaml b/.github/workflows/chart.yaml
@@ -14,22 +14,34 @@ jobs:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
-      - name: Helm lint
-        working-directory: ./cost-analyzer
-        run: helm lint
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@b43128a8b25298e1e7b043b78ea6613844e079b1 # v2.6.0
+
+      # Lint all chart values including those in the ci directory.
+      - name: Run chart-testing (lint)
+        run: ct lint --chart-dirs=cost-analyzer/ --charts cost-analyzer/ --validate-maintainers=false
 
+      # Run `helm template` on the main values file plus all those in the ci directory.
       - name: Helm template
-        working-directory: ./cost-analyzer
-        run: helm template . --set global.prometheus.enabled=false --set global.grafana.enabled=false > full.yaml
+        run: |
+          helm template cost-analyzer/ -f cost-analyzer/values.yaml > full.yaml
+          directory="cost-analyzer/ci"
+          for file in "$directory"/*; do
+          if [ -f "$file" ]; then
+              helm template cost-analyzer/ -f "$file" >> full.yaml
+          fi
+          done
 
+      # Run Kubeconform on the combined, templatized output across all tested values stored in `full.yaml`.
+      # Ensure all the rendered resources are valid.
       - name: Kubeconform
         uses: docker://ghcr.io/yannh/kubeconform:latest
         with:
           entrypoint: /kubeconform
-          args: "-summary -output text ./cost-analyzer/full.yaml"
+          args: "-summary -output text full.yaml"
 
 
-  # Installs the chart across a matrix of modern Kubernetes versions
+  # Test cluster versions.
   deploy-chart:
     runs-on: ubuntu-latest
     strategy:
@@ -60,19 +72,65 @@ jobs:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@b43128a8b25298e1e7b043b78ea6613844e079b1 # v2.6.0
+
       - name: Create KinD cluster
         uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0
         with:
           version: v0.20.0
           node_image: kindest/node:${{ matrix.k8s-version.version }}
           kubectl_version: ${{ matrix.k8s-version.version }}
+
+      # Create necessary test files in the cluster. Some configurations of the chart
+      # require pre-existing files like Secrets in order for volume mounts to be valid. Without them,
+      # Pods won't come up and tests will fail.
+      - name: Create kubecost Namespace and test files
+        run: |
+          kubectl create ns kubecost
+          kubectl -n kubecost create -f .github/ci-files
   
+      # Install the chart with default values and check results.
       - name: Install Kubecost chart
         working-directory: ./cost-analyzer
-        run: helm install --wait --wait-for-jobs kubecost . -n kubecost --create-namespace
+        run: helm install --wait --wait-for-jobs kubecost . -n kubecost
+        # run: ct install --namespace kubecost --chart-dirs=cost-analyzer/ --charts cost-analyzer/
 
       - name: Wait for ready 
-        run: kubectl wait --namespace kubecost --for=condition=ready pod --selector app.kubernetes.io/name=cost-analyzer --timeout=120s
+        run: kubectl wait -n kubecost --for=condition=ready pod --selector app.kubernetes.io/name=cost-analyzer --timeout=120s
 
       - name: Run Helm tests 
-        run: helm test -n kubecost kubecost
+        run: helm test -n kubecost kubecost
+
+      - name: Uninstall chart
+        run: helm uninstall kubecost -n kubecost --no-hooks --wait
+
+      - name: Cleanup all Pods in Kubecost Namespace
+        run: kubectl -n kubecost delete deployments,daemonsets,statefulsets,pods --all --force
+
+      # Loops over all the other values files in the ci directory and installs the chart, runs Helm tests, and uninstalls.
+      # Additional sleeps seem necessary for the main Kubecost Pod to be completely available, both frontend and backend.
+      - name: Install, test, remove chart for other values
+        id: loopingtests
+        run: |
+          directory="cost-analyzer/ci"
+          for file in "$directory"/*; do
+          if [ -f "$file" ]; then
+            echo "### Sleeping for 30 seconds ###"
+            sleep 30
+            echo "### Performing installation with values from $file ###"
+            helm install --wait --wait-for-jobs kubecost cost-analyzer/ -n kubecost -f "$file"
+            echo "### Waiting for cost-analyzer Pod readiness. ###"
+            kubectl wait -n kubecost --for=condition=ready pod --selector app.kubernetes.io/name=cost-analyzer --timeout=120s
+            echo "### Sleeping for 60 seconds longer. ###"
+            sleep 60
+            echo "### [DEBUG] Listing Pods in Kubecost Namespace. ###"
+            kubectl -n kubecost get pods
+            echo "### Performing helm tests ###"
+            helm test -n kubecost $(helm ls -n kubecost --all --short)
+            echo "### Performing uninstallation ###"
+            helm uninstall $(helm ls -n kubecost --all --short) -n kubecost --no-hooks --wait
+            echo "### Cleaning up all Pods and Pod controllers in Kubecost Namespace ###"
+            kubectl -n kubecost delete deployments,daemonsets,statefulsets,pods --all --force
+          fi
+          done
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
@@ -22,3 +22,35 @@ minikube start --force
 ```
 
 For more information on GitHub Codespaces, see the reference documentation [here](https://docs.github.com/en/codespaces/overview).
+
+## CI Testing
+
+This repository employs CI checks designed to catch many common issues with Helm charts. These checks must all pass for a PR to be merged as they are designed to prevent regressions and other errors that may impact successful deployment and operation. The workflow `chart.yaml` is responsible for these checks and a graph of what is checked and the order is shown below.
+
+```mermaid
+flowchart LR
+    A(Lint) --> B(Template)
+    B --> C(Conformance)
+    C --> D{Test}
+    D ---> E[Version 1]
+    D ---> F[Version 2]
+    D ---> G[Version N]
+```
+
+In addition to the default `values.yaml` file required by every chart, this repository also allows testing of additional values files for other configurations of Kubecost. Any values files placed at `/cost-analyzer/ci` will be automatically picked up by this testing process. Values files placed here must conform to the pattern `*-values.yaml` in order to be linted. Changes to any templates will allow testing by all combined values files.
+
+### Linting
+
+Charts and chart values will be linted for YAML syntax and Helm best practices.
+
+### Templating
+
+The chart will be fully templated with each available values file to ensure, given the input values, the templates render correctly.
+
+### Conformance
+
+Once templating is successful, the combined results of the chart templated across all values will be examined for correctness against Kubernetes OpenAPI schemas to ensure the resources are compliant with the latest version.
+
+### Testing
+
+If all previous tests pass, the chart with each of the eligible values files will be deployed across a matrix of the last nine (9) versions of Kubernetes clusters to ensure all expected resources are available, and finally that a basic end-to-end test of the Kubecost deployment is successful. In order for some deployment configurations to succeed, there may be some dependent resources which are required. For example, in some configurations Kubecost requires Kubernetes Secrets to already exist so they may be consumed by Pods in the form of a volume mount. Any such prerequisite resources should be stored in `/.github/ci-files` as they will be automatically deployed as part of the test suite. Files in this directory must not clash and all will be deployed at the outset of testing.
diff --git a/cost-analyzer/ci/aggregator-values.yaml b/cost-analyzer/ci/aggregator-values.yaml
@@ -0,0 +1,12 @@
+kubecostAggregator:
+  enabled: true
+  cloudCost:
+    enabled: true
+  aggregatorStorage:
+    storageRequest: 5Gi
+  aggregatorDbStorage:
+    storageRequest: 10Gi
+kubecostModel:
+  federatedStorageConfigSecret: federated-store
+kubecostProductConfigs:
+  cloudIntegrationSecret: cloud-integration
diff --git a/cost-analyzer/ci/federatedetl-primary-netcosts-values.yaml b/cost-analyzer/ci/federatedetl-primary-netcosts-values.yaml
@@ -0,0 +1,48 @@
+kubecostProductConfigs:
+  clusterName: CLUSTER_NAME
+  # cloudIntegrationSecret: cloud-integration
+federatedETL:
+  useExistingS3Config: false
+  primaryCluster: true
+  federatedCluster: true
+  federator:
+    enabled: true
+    # primaryClusterID: CLUSTER_NAME # Add after initial setup. This will break the combined folder setup if included at deployment.
+kubecostModel:
+  containerStatsEnabled: true
+  cloudCost:
+    enabled: true  # Set to true to enable CloudCost view that gives you visibility of your Cloud provider resources cost
+  etlCloudAsset: false  # Set etlCloudAsset to false when cloudCost.enabled=true
+  federatedStorageConfigSecret: federated-store
+serviceAccount:  # this example uses AWS IRSA, which creates a service account with rights to the s3 bucket. If using keys+secrets in the federated-store, set create: true
+  create: true
+kubecostDeployment:
+  queryServiceReplicas: 0  # to improve performance, increase replica count. see: https://docs.kubecost.com/install-and-configure/install/etl-backup/query-service-replicas
+global:
+  prometheus:
+    enabled: true
+    # fqdn: http://prometheus-operated.monitoring:9090
+  grafana:  # prometheus metrics will be local cluster only, disable grafana to save resources
+    enabled: false
+    proxy: false
+prometheus:
+  kubeStateMetrics:
+    enabled: false
+  kube-state-metrics:
+    disabled: true
+  nodeExporter:
+    enabled: false
+  server:
+    global:
+      external_labels:
+        # cluster_id should be unique for all clusters and the same value as .kubecostProductConfigs.clusterName
+        cluster_id: CLUSTER_NAME
+networkCosts:
+  # optional, see: https://docs.kubecost.com/install-and-configure/advanced-configuration/network-costs-configuration
+  enabled: true
+  config:
+    services:
+      # set the appropriate cloud provider to true
+      amazon-web-services: true
+      # google-cloud-services: true
+      # azure-cloud-services: true
diff --git a/cost-analyzer/values.yaml b/cost-analyzer/values.yaml
@@ -646,7 +646,7 @@ service:
   labels: {}
   annotations: {}
   sessionAffinity:
-    enabled: false # Makes sure that connections from a client are passed to the same Pod each time, when set to `true`. You should set it when you enabled authentication through OIDC or SAML integration.
+    enabled: false  # Makes sure that connections from a client are passed to the same Pod each time, when set to `true`. You should set it when you enabled authentication through OIDC or SAML integration.
     timeoutSeconds: 10800
 
 # Enabling long-term durable storage with Postgres requires an enterprise license
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Files in this directory are used as dependencies for CI tests.