Add kubeRBACProxy property to support querying in cluster prometheus …

…in openshift (#3700) * Add kubeRBACProxy property to support querying in cluster prometheus in openshift
kubecost · Oct 24, 2024 · b01e76c · b01e76c
1 parent fe03983
commit b01e76c
Show file tree

Hide file tree

Showing 9 changed files with 79 additions and 0 deletions.
diff --git a/.lycheeignore b/.lycheeignore
@@ -7,6 +7,7 @@ prometheus.monitor.azure.com
 
 # Prometheus
 cost-analyzer-prometheus-server
+prometheus-k8s
 kubecost-prometheus-server
 kubecost-cost-analyzer-mimir-proxy
 prometheus-operated

diff --git a/cost-analyzer/templates/NOTES.txt b/cost-analyzer/templates/NOTES.txt
@@ -9,6 +9,7 @@
 {{- include "federatedStorageSourceCheck" . -}}
 {{- include "prometheusRetentionCheck" . -}}
 {{- include "clusterIDCheck" . -}}
+{{- include "kubeRBACProxyBearerTokenCheck" . -}}
 
 {{- $servicePort := .Values.service.port | default 9090 }}
 Kubecost {{ .Chart.Version }} has been successfully installed.

diff --git a/cost-analyzer/templates/_helpers.tpl b/cost-analyzer/templates/_helpers.tpl
@@ -203,6 +203,15 @@ Verify a cluster_id is set in the Prometheus global config
   {{- end -}}
 {{- end -}}
 
+{{/*
+  Verify if both kube-rbac-proxy and bearer token are set
+*/}}
+{{- define "kubeRBACProxyBearerTokenCheck" -}}
+{{- if and (.Values.global.prometheus.kubeRBACProxy) (.Values.global.prometheus.queryServiceBearerTokenSecretName) }}
+  {{- fail "\n\nBoth kubeRBACProxy and queryServiceBearerTokenSecretName are set. Please specify only one." -}}
+{{- end -}}
+{{- end -}}
+
 
 {{/*
 Verify the cloud integration secret exists with the expected key when cloud integration is enabled.

diff --git a/cost-analyzer/templates/cost-analyzer-cluster-role-binding-template.yaml b/cost-analyzer/templates/cost-analyzer-cluster-role-binding-template.yaml
@@ -33,4 +33,21 @@ subjects:
   - kind: ServiceAccount
     name: {{ template "cost-analyzer.serviceAccountName" . }}
     namespace: {{ .Release.Namespace }} 
+---
 {{- end }} 
+{{- if and .Values.global.platforms.openshift.enabled .Values.global.platforms.openshift.createMonitoringClusterRoleBinding }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ template "cost-analyzer.serviceAccountName" . }}-openshift-monitoring
+  labels:
+    {{ include "cost-analyzer.commonLabels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-monitoring-view
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "cost-analyzer.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }} 
+{{- end }}
diff --git a/cost-analyzer/templates/cost-analyzer-deployment-template.yaml b/cost-analyzer/templates/cost-analyzer-deployment-template.yaml
@@ -861,6 +861,10 @@ spec:
             - name: INSECURE_SKIP_VERIFY
               value: {{ (quote .Values.global.prometheus.insecureSkipVerify) }}
             {{- end }}
+            {{- if .Values.global.prometheus.kubeRBACProxy }}
+            - name: KUBE_RBAC_PROXY_ENABLED
+              value: {{ (quote .Values.global.prometheus.kubeRBACProxy) }}
+            {{- end }}
             {{- if .Values.pricingCsv }}
             {{- if .Values.pricingCsv.enabled }}
             - name: USE_CSV_PROVIDER

diff --git a/cost-analyzer/templates/monitoring-role-binding-template.yaml b/cost-analyzer/templates/monitoring-role-binding-template.yaml
@@ -0,0 +1,17 @@
+{{- if (.Values.global.platforms.openshift.enabled) }}
+{{- if (.Values.global.platforms.openshift.createMonitoringResourceReaderRoleBinding) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ template "cost-analyzer.fullname" . }}-reader
+subjects:
+- kind: ServiceAccount
+  name: {{ .Values.global.platforms.openshift.monitoringServiceAccountName | quote }}
+  namespace: openshift-monitoring
+roleRef:
+  kind: Role
+  name: {{ template "cost-analyzer.fullname" . }}-reader
+  apiGroup: rbac.authorization.k8s.io
+{{- end -}}
+{{- end -}}
diff --git a/cost-analyzer/templates/monitoring-role-template.yaml b/cost-analyzer/templates/monitoring-role-template.yaml
@@ -0,0 +1,19 @@
+{{- if (.Values.global.platforms.openshift.enabled) }}
+{{- if (.Values.global.platforms.openshift.createMonitoringResourceReaderRoleBinding) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ template "cost-analyzer.fullname" . }}-reader
+rules:
+  - apiGroups: 
+      - ''
+    resources:
+      - "pods"
+      - "services"
+      - "endpoints"
+    verbs:
+      - list
+      - watch
+{{- end -}}
+{{- end -}}
diff --git a/cost-analyzer/values-openshift.yaml b/cost-analyzer/values-openshift.yaml
@@ -1,9 +1,19 @@
 global:
+  prometheus:
+    enabled: true  # Kubecost depends on Prometheus data, it is not optional. When enabled: false, Prometheus will not be installed and you must configure your own Prometheus to scrape kubecost as well as provide the fqdn below. -- Warning: Before changing this setting, please read to understand the risks https://docs.kubecost.com/install-and-configure/install/custom-prom
+    fqdn: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091  # example address of a prometheus to connect to. Include protocol (http:// or https://) Ignored if enabled: true
+    # insecureSkipVerify: false  # If true, kubecost will not check the TLS cert of prometheus
+    # queryServiceBearerTokenSecretName: mcdbsecret  # kubectl create secret generic mcdbsecret -n kubecost --from-file=TOKEN
+    # kubeRBACProxy: false # If true, kubecost will use kube-rbac-proxy to authenticate with in cluster Prometheus for openshift
+
   # Platforms is a higher-level abstraction for platform-specific values and settings.
   platforms:
     # Deploying to OpenShift (OCP) requires enabling this option.
     openshift:
       enabled: true  # Deploy Kubecost to OpenShift.
+      # createMonitoringClusterRoleBinding: false  # Create a Cluster Role Binding to allow using in-cluster prometheus or thanos.
+      # createMonitoringResourceReaderRoleBinding: false  # Create a Role and Role Binding to allow in-cluster prometheus or thanos to list and watch resources. This will be necessary if you are not using bundled prometheus and need to add scrape config for resources.
+      # monitoringServiceAccountName: prometheus-k8s  # Name of the service account to bind to the Resource Reader Role Binding.
       route:
         enabled: false  # Create an OpenShift Route.
         annotations: {}  # Add annotations to the Route.

diff --git a/cost-analyzer/values.yaml b/cost-analyzer/values.yaml
@@ -6,6 +6,7 @@ global:
     # insecureSkipVerify: false  # If true, kubecost will not check the TLS cert of prometheus
     # queryServiceBasicAuthSecretName: dbsecret # kubectl create secret generic dbsecret -n kubecost --from-file=USERNAME --from-file=PASSWORD
     # queryServiceBearerTokenSecretName: mcdbsecret  # kubectl create secret generic mcdbsecret -n kubecost --from-file=TOKEN
+    # kubeRBACProxy: false # If true, kubecost will use kube-rbac-proxy to authenticate with in cluster Prometheus for openshift
 
   grafana:
     enabled: true  # If false, Grafana will not be installed