Merge commit 'b675a8cacaa5b6c64cf00d3583df026cfc2ec507' into atm/add-…

…etl-utils-pod * commit 'b675a8cacaa5b6c64cf00d3583df026cfc2ec507': (28 commits) exec (#2701) swap jimmidyson/configmap-reload for prometheus-operator/prometheus-config-reloader (#2698) update savedReports and advancedReports in values.yml to reflect current filter schema add systemProxy env vars (#2687) feat(cost-analyzer): add StatefulSet as option (#2188) [Feature] Development guide and devcontainers (#2680) Bump actions/checkout from 4.1.0 to 4.1.1 (#2683) remove replicasets from core (#2678) networkCosts service discovery (#2677) Begin Helm testing (#2674) update securityContexts (#2669) consistent image name for aggregator (#2676) Add version matrix and more tests (#2664) label consistency (#2673) setting to 50h to match etl retention time (#2667) pv sizing proxy for wf add missing bracket add ability to override cc sa name Added /savings/localLowDisks proxy for Aggregator. update perms (#2662) ...
kubecost · Oct 30, 2023 · a876f66 · a876f66
2 parents 6cddb71 + b675a8c
commit a876f66
Show file tree

Hide file tree

Showing 31 changed files with 682 additions and 723 deletions.
diff --git a/.devcontainer/cluster/Dockerfile b/.devcontainer/cluster/Dockerfile
@@ -0,0 +1,18 @@
+FROM ubuntu:22.04@sha256:2b7412e6465c3c7fc5bb21d3e6f1917c167358449fecac8176c6e496e5c1f05f
+
+RUN apt-get update && apt-get install -y sudo git curl wget apt-transport-https ca-certificates gnupg-agent software-properties-common
+ARG USERNAME=root
+RUN echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
+    && chmod 0440 /etc/sudoers.d/$USERNAME
+
+# Install Docker
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+RUN echo \
+    "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
+    $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+RUN apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io
+
+# Expose ports for Minikube and Docker
+EXPOSE 22 80 2375 8443
+
+CMD ["/bin/bash"]
diff --git a/.devcontainer/cluster/devcontainer.json b/.devcontainer/cluster/devcontainer.json
@@ -0,0 +1,13 @@
+{
+    "name": "Cluster",
+    "build": {
+        "dockerfile": "Dockerfile"
+    },
+    "remoteUser": "root",
+    "mounts": ["source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"],
+    "runArgs": ["--privileged", "--network=host", "-p", "22:22", "-p", "80:80", "-p", "2375:2375", "-p", "8443:8443"],
+    "features": {
+        "ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {}
+    },
+    "postCreateCommand": "./.devcontainer/tools.sh"
+}
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -0,0 +1,4 @@
+{
+    "name": "Default",
+    "postCreateCommand": "./.devcontainer/tools.sh"
+}
diff --git a/.devcontainer/tools.sh b/.devcontainer/tools.sh
@@ -0,0 +1,4 @@
+#! /bin/bash
+
+## Install yq
+sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
diff --git a/.github/workflows/chart.yaml b/.github/workflows/chart.yaml
@@ -12,7 +12,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Helm lint
         working-directory: ./cost-analyzer
@@ -36,6 +36,12 @@ jobs:
       fail-fast: false
       matrix:
         k8s-version:
+          - name: v1.20
+            version: v1.20.15
+          - name: v1.21
+            version: v1.21.14
+          - name: v1.22
+            version: v1.22.17
           - name: v1.23
             version: v1.23.17
           - name: v1.24
@@ -52,7 +58,7 @@ jobs:
     name: ${{ matrix.k8s-version.name }} test
     steps:
       - name: Checkout
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Create KinD cluster
         uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0
@@ -63,5 +69,10 @@ jobs:
 
       - name: Install Kubecost chart
         working-directory: ./cost-analyzer
-        run: |
-          helm install --wait --wait-for-jobs kubecost . --namespace kubecost --create-namespace --set global.prometheus.enabled=false --set global.grafana.enabled=false
+        run: helm install --wait --wait-for-jobs kubecost . -n kubecost --create-namespace
+
+      - name: Wait for ready 
+        run: kubectl wait --namespace kubecost --for=condition=ready pod --selector app.kubernetes.io/name=cost-analyzer --timeout=120s
+
+      - name: Run Helm tests 
+        run: helm test -n kubecost kubecost
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
@@ -0,0 +1,24 @@
+# Kubecost Helm Chart Development Guide
+
+This guide contains tips on setting up a development environment for the Kubecost Helm chart.
+
+> [!IMPORTANT]  
+> Following some of these steps may involve billing charges by GitHub for either an individual account or organization.
+
+## Developing with Codespaces and Devcontainers
+
+GitHub includes a feature called [Codespaces](https://github.com/features/codespaces) which allows you to set up an instant, fully-provisioned development environment in the cloud in seconds. This is a containerized environment powered by [Development Containers](https://containers.dev/) ("devcontainers") which have all the necessary project-specific tools to get started.
+
+This repository contains two such devcontainers to aid in easy development, testing, and contribution. The first, which is the default, contains basic tools such as `helm` and `kubectl` along with some other commonly-used tools for Chart development. This default devcontainer will be the one used if no other selection is chosen. Follow the process [here](https://docs.github.com/en/codespaces/developing-in-a-codespace/creating-a-codespace-for-a-repository#creating-a-codespace-for-a-repository) to create a Codespaces environment using the default devcontainer.
+
+The second devcontainer provides a Docker-in-Docker experience allowing you to test/develop your Helm chart changes as well as deploy them to a running cluster all inside the Codespaces environment. In order to create this more advanced Codespaces environment, follow the guide [here](https://docs.github.com/en/codespaces/developing-in-a-codespace/creating-a-codespace-for-a-repository#creating-a-codespace-for-a-repository) at step four and then select the "Cluster" configuration as shown below. You may also wish to use a larger machine type such as the 4-core option if you intend on actually deploying Kubecost.
+
+![Custom devcontainer profile](/docs/images/custom-devcontainer.png)
+
+This Cluster profile includes Docker and Minikube allowing you to not only develop against the Helm chart but also fully deploy, as opposed to just rendering, the Chart to inspect changes. When running Minikube in this devcontainer, pass the `--force` flag to permit Minikube to run as root.
+
+```sh
+minikube start --force
+```
+
+For more information on GitHub Codespaces, see the reference documentation [here](https://docs.github.com/en/codespaces/overview).
diff --git a/README.md b/README.md
@@ -2,6 +2,16 @@
 
 This is the official Helm chart for [Kubecost](https://www.kubecost.com/), an enterprise-grade application to monitor and manage Kubernetes spend. Please see the [website](https://www.kubecost.com/) for more details on what Kubecost can do for you and the official documentation [here](https://docs.kubecost.com/), or contact [[email protected]](mailto:[email protected]) for assistance.
 
+## Version Support
+
+Kubecost strives to support as many versions of Kubernetes as possible. Below is the version support matrix which has been tested. Versions outside of the stated range may still work but are untested.
+
+| Chart Version                | Kubernetes Min | Kubernetes Max |
+|--------------------------------|----------------|----------------|
+| 1.106                          | 1.20           | 1.28           |
+
+## Installation
+
 To install via Helm, run the following command.
 
 ```sh

diff --git a/cost-analyzer/Chart.yaml b/cost-analyzer/Chart.yaml
@@ -1,9 +1,9 @@
 apiVersion: v2
-appVersion: "1.106.0"
+appVersion: "1.106.3"
 description: A Helm chart that sets up Kubecost, Prometheus, and Grafana to monitor
   cloud costs.
 name: cost-analyzer
-version: "1.106.0"
+version: "1.106.3"
 annotations:
   "artifacthub.io/links": |
     - name: Homepage

diff --git a/cost-analyzer/README.md b/cost-analyzer/README.md
@@ -99,3 +99,57 @@ Adjusting the log format changes the format in which the logs are output making
 |--------|----------------------------------------------------------------------------------------------------------------------------|
 | `JSON`   | `{"level":"info","time":"2006-01-02T15:04:05.999999999Z07:00","message":"Starting cost-model (git commit \"1.91.0-rc.0\")"}` |
 | `pretty` | `2006-01-02T15:04:05.999999999Z07:00 INF Starting cost-model (git commit "1.91.0-rc.0")`                                     |
+
+## Testing
+To perform local testing do next:
+- install locally [kind](https://github.com/kubernetes-sigs/kind) according to documentation.
+- install locally [ct](https://github.com/helm/chart-testing) according to documentation.
+- create local cluster using `kind` \
+use image version from https://github.com/kubernetes-sigs/kind/releases e.g. `kindest/node:v1.25.11@sha256:227fa11ce74ea76a0474eeefb84cb75d8dad1b08638371ecf0e86259b35be0c8`
+```shell
+kind create cluster --image kindest/node:v1.25.11@sha256:227fa11ce74ea76a0474eeefb84cb75d8dad1b08638371ecf0e86259b35be0c8
+```
+- perform ct execution
+```shell
+ct install  --chart-dirs="." --charts="."
+```
+
+- perform ct StatefulSet execution
+
+```shell
+# create multiple nodes kind config
+cat > kind-config.yaml <<EOF
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+- role: worker
+EOF
+# creaet kind cluster with kind config
+kind create cluster --name kubecost-statefulset --config kind-config.yaml --image kindest/node:v1.25.11@sha256:227fa11ce74ea76a0474eeefb84cb75d8dad1b08638371ecf0e86259b35be0c8
+# deploy an object storage for our testing purpose (https://min.io/docs/minio/kubernetes/upstream/index.html)
+curl --silent https://raw.githubusercontent.com/minio/docs/master/source/extra/examples/minio-dev.yaml  | sed -e "s/kubealpha.local/kubecost-statefulset-worker/" -e "s%minio server /data%mkdir -p /data/kubecost; minio server /data%" | kubectl apply -f -
+# create a headless service to the minio S3 API port
+kubectl create service clusterip -n minio-dev minio --tcp=9000:9000 --clusterip="None"
+# create our testing namespace
+kubectl create namespace kubecost-statefulset
+# create the bucket config 
+cat > etlBucketConfigSecret.yaml <<EOF
+type: s3
+config:
+  bucket: kubecost
+  endpoint: minio.minio-dev:9000
+  insecure: true
+  access_key: minioadmin
+  secret_key: minioadmin
+EOF
+# create the secret with the object-store.yaml
+kubectl create secret generic -n kubecost-statefulset object-store --from-file=object-store.yaml=etlBucketConfigSecret.yaml
+# start our chart-testing
+ct install --namespace kubecost-statefulset --chart-dirs="." --charts="." --helm-extra-set-args="--set=global.prometheus.enabled=true --set=global.grafana.enabled=true --set=kubecostDeployment.leaderFollower.enabled=true --set=kubecostDeployment.statefulSet.enabled=true --set=kubecostDeployment.replicas=2 --set=kubecostModel.etlBucketConfigSecret=object-store"
+# cleanup
+kind delete cluster --name kubecost-statefulset 
+```
+
+
diff --git a/cost-analyzer/charts/prometheus/README.md b/cost-analyzer/charts/prometheus/README.md
@@ -183,17 +183,17 @@ Parameter | Description | Default
 `configmapReload.prometheus.enabled` | If false, the configmap-reload container for Prometheus will not be deployed | `true`
 `configmapReload.prometheus.containerSecurityContext` | securityContext for container | `{}`
 `configmapReload.prometheus.name` | configmap-reload container name | `configmap-reload`
-`configmapReload.prometheus.image.repository` | configmap-reload container image repository | `jimmidyson/configmap-reload`
-`configmapReload.prometheus.image.tag` | configmap-reload container image tag | `v0.5.0`
+`configmapReload.prometheus.image.repository` | configmap-reload container image repository | `quay.io/prometheus-operator/prometheus-config-reloader`
+`configmapReload.prometheus.image.tag` | configmap-reload container image tag | `v0.68.0`
 `configmapReload.prometheus.image.pullPolicy` | configmap-reload container image pull policy | `IfNotPresent`
 `configmapReload.prometheus.extraArgs` | Additional configmap-reload container arguments | `{}`
 `configmapReload.prometheus.extraVolumeDirs` | Additional configmap-reload volume directories | `{}`
 `configmapReload.prometheus.extraConfigmapMounts` | Additional configmap-reload configMap mounts | `[]`
 `configmapReload.prometheus.resources` | configmap-reload pod resource requests & limits | `{}`
 `configmapReload.alertmanager.enabled` | If false, the configmap-reload container for AlertManager will not be deployed | `true`
 `configmapReload.alertmanager.name` | configmap-reload container name | `configmap-reload`
-`configmapReload.alertmanager.image.repository` | configmap-reload container image repository | `jimmidyson/configmap-reload`
-`configmapReload.alertmanager.image.tag` | configmap-reload container image tag | `v0.5.0`
+`configmapReload.alertmanager.image.repository` | configmap-reload container image repository | `quay.io/prometheus-operator/prometheus-config-reloader`
+`configmapReload.alertmanager.image.tag` | configmap-reload container image tag | `v0.68.0`
 `configmapReload.alertmanager.image.pullPolicy` | configmap-reload container image pull policy | `IfNotPresent`
 `configmapReload.alertmanager.extraArgs` | Additional configmap-reload container arguments | `{}`
 `configmapReload.alertmanager.extraVolumeDirs` | Additional configmap-reload volume directories | `{}`

diff --git a/cost-analyzer/charts/prometheus/templates/server-deployment.yaml b/cost-analyzer/charts/prometheus/templates/server-deployment.yaml
@@ -50,8 +50,8 @@ spec:
           image: "{{ .Values.configmapReload.prometheus.image.repository }}:{{ .Values.configmapReload.prometheus.image.tag }}"
           imagePullPolicy: "{{ .Values.configmapReload.prometheus.image.pullPolicy }}"
           args:
-            - --volume-dir=/etc/config
-            - --webhook-url=http://127.0.0.1:9090{{ .Values.server.prefixURL }}/-/reload
+            - --watched-dir=/etc/config
+            - --reload-url=http://127.0.0.1:9090{{ .Values.server.prefixURL }}/-/reload
           {{- range $key, $value := .Values.configmapReload.prometheus.extraArgs }}
             - --{{ $key }}={{ $value }}
           {{- end }}

diff --git a/cost-analyzer/charts/prometheus/templates/server-statefulset.yaml b/cost-analyzer/charts/prometheus/templates/server-statefulset.yaml
@@ -51,8 +51,8 @@ spec:
           image: "{{ .Values.configmapReload.prometheus.image.repository }}:{{ .Values.configmapReload.prometheus.image.tag }}"
           imagePullPolicy: "{{ .Values.configmapReload.prometheus.image.pullPolicy }}"
           args:
-            - --volume-dir=/etc/config
-            - --webhook-url=http://127.0.0.1:9090{{ .Values.server.prefixURL }}/-/reload
+            - --watched-dir=/etc/config
+            - --reload-url=http://127.0.0.1:9090{{ .Values.server.prefixURL }}/-/reload
           {{- range $key, $value := .Values.configmapReload.prometheus.extraArgs }}
             - --{{ $key }}={{ $value }}
           {{- end }}

diff --git a/cost-analyzer/charts/prometheus/values.yaml b/cost-analyzer/charts/prometheus/values.yaml
@@ -307,8 +307,6 @@ alertmanager:
     type: ClusterIP
 
 ## Monitors ConfigMap changes and POSTs to a URL
-## Ref: https://github.com/jimmidyson/configmap-reload
-##
 configmapReload:
   prometheus:
     ## If false, the configmap-reload container will not be deployed
@@ -322,8 +320,8 @@ configmapReload:
     ## configmap-reload container image
     ##
     image:
-      repository: jimmidyson/configmap-reload
-      tag: v0.9.0
+      repository: quay.io/prometheus-operator/prometheus-config-reloader
+      tag: v0.68.0
       pullPolicy: IfNotPresent
 
     ## Additional configmap-reload container arguments
@@ -362,8 +360,8 @@ configmapReload:
     ## configmap-reload container image
     ##
     image:
-      repository: jimmidyson/configmap-reload
-      tag: v0.9.0
+      repository: quay.io/prometheus-operator/prometheus-config-reloader
+      tag: v0.68.0
       pullPolicy: IfNotPresent
 
     ## Additional configmap-reload container arguments

diff --git a/cost-analyzer/scripts/create-admission-controller-tls.sh b/cost-analyzer/scripts/create-admission-controller-tls.sh
@@ -19,6 +19,11 @@ kubectl create secret tls webhook-server-tls \
     --cert "certs/tls.crt" \
     --key "certs/tls.key" -n "${namespace}"
 
-echo -e "\nUpdating values.yaml ..."
 ENCODED_CA=$(base64 < certs/tls.crt | tr -d '\n')
-sed -i '' 's@${CA_BUNDLE}@'"${ENCODED_CA}"'@g' ../values.yaml
+
+if [ -f "../values.yaml" ]; then
+  echo -e "\nUpdating values.yaml ..."
+  sed -i '' 's@${CA_BUNDLE}@'"${ENCODED_CA}"'@g' ../values.yaml
+else
+  echo -e "\nThe CA bundle to use in your values file is: \n${ENCODED_CA}"
+fi
diff --git a/cost-analyzer/templates/_helpers.tpl b/cost-analyzer/templates/_helpers.tpl
@@ -137,16 +137,19 @@ Create the name of the service account
 {{- end -}}
 {{- end -}}
 {{- define "aggregator.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create -}}
-    {{ default (include "aggregator.fullname" .) .Values.serviceAccount.name }}
+{{- if .Values.kubecostAggregator.serviceAccountName -}}
+    {{ .Values.kubecostAggregator.serviceAccountName }}
 {{- else -}}
-    {{ default "default" .Values.serviceAccount.name }}
+    {{ template "cost-analyzer.serviceAccountName" . }}
 {{- end -}}
 {{- end -}}
 {{- define "cloudCost.serviceAccountName" -}}
-    {{ (include "cloudCost.fullname" .) }}
+{{- if .Values.kubecostAggregator.cloudCost.serviceAccountName -}}
+    {{ .Values.kubecostAggregator.cloudCost.serviceAccountName }}
+{{- else -}}
+    {{ template "cost-analyzer.serviceAccountName" . }}
+{{- end -}}
 {{- end -}}
-
 {{/*
 Network Costs name used to tie autodiscovery of metrics to daemon set pods
 */}}
@@ -228,7 +231,7 @@ app: federator
 {{- end -}}
 {{- define "aggregator.commonLabels" -}}
 {{ include "cost-analyzer.chartLabels" . }}
-app: kubecost-aggregator
+app: aggregator
 {{- end -}}
 {{- define "cloudCost.commonLabels" -}}
 {{ include "cost-analyzer.chartLabels" . }}
@@ -260,7 +263,7 @@ app: federator
 {{- define "aggregator.selectorLabels" -}}
 app.kubernetes.io/name: {{ include "aggregator.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
-app: kubecost-aggregator
+app: aggregator
 {{- end -}}
 {{- define "cloudCost.selectorLabels" -}}
 app.kubernetes.io/name: {{ include "cloudCost.name" . }}
@@ -319,9 +322,9 @@ Return the appropriate apiVersion for podsecuritypolicy.
 
 {{/*
 Recursive filter which accepts a map containing an input map (.v) and an output map (.r). The template
-will traverse all values inside .v recursively writing non-map values to the output .r. If a nested map 
-is discovered, we look for an 'enabled' key. If it doesn't exist, we continue traversing the 
-map. If it does exist, we omit the inner map traversal iff enabled is false. This filter writes the 
+will traverse all values inside .v recursively writing non-map values to the output .r. If a nested map
+is discovered, we look for an 'enabled' key. If it doesn't exist, we continue traversing the
+map. If it does exist, we omit the inner map traversal iff enabled is false. This filter writes the
 enabled only version to the output .r
 */}}
 {{- define "cost-analyzer.filter" -}}
@@ -359,8 +362,8 @@ The implied use case is {{ template "cost-analyzer.filterEnabled" .Values }}
 
 {{/*
 This template runs the full check for leader/follower requirements in order to determine
-whether it should be configured. This template will return true if it's enabled and all 
-requirements are met. 
+whether it should be configured. This template will return true if it's enabled and all
+requirements are met.
 */}}
 {{- define "cost-analyzer.leaderFollowerEnabled" }}
     {{- if .Values.kubecostDeployment }}