use TLS certificates for metrics scraping from Grafana

openshift · Oct 1, 2021 · 8b8c08a · 8b8c08a
1 parent 191860e
commit 8b8c08a
Show file tree

Hide file tree

Showing 8 changed files with 175 additions and 20 deletions.
diff --git a/assets/grafana/deployment.yaml b/assets/grafana/deployment.yaml
@@ -137,6 +137,36 @@ spec:
         - mountPath: /etc/proxy/secrets
           name: secret-grafana-proxy
           readOnly: false
+      - args:
+        - --secure-listen-address=0.0.0.0:9097
+        - --upstream=http://127.0.0.1:3001
+        - --config-file=/etc/kube-rbac-proxy/config.yaml
+        - --tls-cert-file=/etc/tls/private/tls.crt
+        - --tls-private-key-file=/etc/tls/private/tls.key
+        - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+        - --client-ca-file=/etc/tls/client/client-ca.crt
+        - --logtostderr=true
+        - --v=10
+        image: quay.io/brancz/kube-rbac-proxy:v0.11.0
+        name: kube-rbac-proxy-metric
+        ports:
+        - containerPort: 9097
+          name: metrics
+        resources:
+          requests:
+            cpu: 1m
+            memory: 15Mi
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /etc/kube-rbac-proxy
+          name: secret-grafana-kube-rbac-proxy-metric
+          readOnly: true
+        - mountPath: /etc/tls/private
+          name: secret-alertmanager-main-tls
+          readOnly: true
+        - mountPath: /etc/tls/client
+          name: metrics-client-ca
+          readOnly: true
       nodeSelector:
         kubernetes.io/os: linux
       priorityClassName: system-cluster-critical
@@ -199,3 +229,6 @@ spec:
       - name: secret-grafana-proxy
         secret:
           secretName: grafana-proxy
+      - configMap:
+          name: metrics-client-ca
+        name: metrics-client-ca
diff --git a/assets/grafana/kube-rbac-proxy-metric-secret.yaml b/assets/grafana/kube-rbac-proxy-metric-secret.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+data: {}
+kind: Secret
+metadata:
+  labels:
+    app.kubernetes.io/name: grafana
+  name: grafana-kube-rbac-proxy-metric
+  namespace: openshift-monitoring
+stringData:
+  config.yaml: |-
+    "authorization":
+      "static":
+      - "path": "/metrics"
+        "resourceRequest": false
+        "user":
+          "name": "system:serviceaccount:openshift-monitoring:prometheus-k8s"
+        "verb": "get"
+type: Opaque
diff --git a/assets/grafana/service-monitor.yaml b/assets/grafana/service-monitor.yaml
@@ -12,7 +12,7 @@ spec:
   endpoints:
   - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
     interval: 30s
-    port: https
+    port: metrics
     scheme: https
     tlsConfig:
       caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt

diff --git a/assets/grafana/service.yaml b/assets/grafana/service.yaml
@@ -15,6 +15,9 @@ spec:
   - name: https
     port: 3000
     targetPort: https
+  - name: metrics
+    port: 9097
+    targetPort: metrics
   selector:
     app.kubernetes.io/component: grafana
     app.kubernetes.io/name: grafana

diff --git a/jsonnet/components/grafana.libsonnet b/jsonnet/components/grafana.libsonnet
@@ -1,3 +1,4 @@
+local generateSecret = import '../utils/generate-secret.libsonnet';
 local grafana = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/grafana.libsonnet';
 
 function(params)
@@ -77,7 +78,7 @@ function(params)
             {
               bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
               interval: '30s',
-              port: 'https',
+              port: 'metrics',
               scheme: 'https',
               tlsConfig: {
                 caFile: '/etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt',
@@ -116,11 +117,19 @@ function(params)
         },
       },
       spec+: {
-        ports: [{
-          name: 'https',
-          port: 3000,
-          targetPort: 'https',
-        }],
+        ports: [
+          {
+            name: 'https',
+            port: 3000,
+            targetPort: 'https',
+          },
+          {
+            name: 'metrics',
+            port: 9097,
+            targetPort: 'metrics',
+          },
+
+        ],
         type: 'ClusterIP',
       },
     },
@@ -139,6 +148,12 @@ function(params)
       data: {},
     },
 
+    kubeRbacProxyMetricSecret: generateSecret.staticAuthSecret(cfg.namespace, cfg.commonLabels, 'grafana-kube-rbac-proxy-metric') + {
+      metadata+: {
+        labels: { 'app.kubernetes.io/name': 'grafana' },
+      },
+    },
+
     // In order for the oauth proxy to perform a TokenReview and
     // SubjectAccessReview for authN and authZ the Grafana ServiceAccount
     // requires the `create` action on both of these.
@@ -261,6 +276,51 @@ function(params)
                   },
                 ],
               },
+              {
+                name: 'kube-rbac-proxy-metric',
+                image: cfg.kubeRbacProxyImage,
+                resources: {
+                  requests: {
+                    cpu: '1m',
+                    memory: '15Mi',
+                  },
+                },
+                ports: [
+                  {
+                    containerPort: 9097,
+                    name: 'metrics',
+                  },
+                ],
+                args: [
+                  '--secure-listen-address=0.0.0.0:9097',
+                  '--upstream=http://127.0.0.1:3001',
+                  '--config-file=/etc/kube-rbac-proxy/config.yaml',
+                  '--tls-cert-file=/etc/tls/private/tls.crt',
+                  '--tls-private-key-file=/etc/tls/private/tls.key',
+                  '--tls-cipher-suites=' + cfg.tlsCipherSuites,
+                  '--client-ca-file=/etc/tls/client/client-ca.crt',
+                  '--logtostderr=true',
+                  '--v=10',
+                ],
+                terminationMessagePolicy: 'FallbackToLogsOnError',
+                volumeMounts: [
+                  {
+                    mountPath: '/etc/kube-rbac-proxy',
+                    name: 'secret-' + $.kubeRbacProxyMetricSecret.metadata.name,
+                    readOnly: true,
+                  },
+                  {
+                    mountPath: '/etc/tls/private',
+                    name: 'secret-alertmanager-main-tls',
+                    readOnly: true,
+                  },
+                  {
+                    mountPath: '/etc/tls/client',
+                    name: 'metrics-client-ca',
+                    readOnly: true,
+                  },
+                ],
+              },
             ],
             volumes+: [
               {
@@ -275,6 +335,12 @@ function(params)
                   secretName: 'grafana-proxy',
                 },
               },
+              {
+                name: 'metrics-client-ca',
+                configMap: {
+                  name: 'metrics-client-ca',
+                },
+              },
             ],
             securityContext: {},
             priorityClassName: 'system-cluster-critical',

diff --git a/jsonnet/main.jsonnet b/jsonnet/main.jsonnet
@@ -195,6 +195,8 @@ local inCluster =
             },
           },
         },
+        tlsCipherSuites: $.values.common.tlsCipherSuites,
+        kubeRbacProxyImage: $.values.common.images.kubeRbacProxy,
       },
       kubeStateMetrics: {
         namespace: $.values.common.namespace,

diff --git a/pkg/manifests/manifests.go b/pkg/manifests/manifests.go
@@ -166,19 +166,20 @@ var (
 	PrometheusOperatorUserWorkloadDeployment         = "prometheus-operator-user-workload/deployment.yaml"
 	PrometheusOperatorUserWorkloadServiceMonitor     = "prometheus-operator-user-workload/service-monitor.yaml"
 
-	GrafanaClusterRoleBinding   = "grafana/cluster-role-binding.yaml"
-	GrafanaClusterRole          = "grafana/cluster-role.yaml"
-	GrafanaConfigSecret         = "grafana/config.yaml"
-	GrafanaDatasourcesSecret    = "grafana/dashboard-datasources.yaml"
-	GrafanaDashboardDefinitions = "grafana/dashboard-definitions.yaml"
-	GrafanaDashboardSources     = "grafana/dashboard-sources.yaml"
-	GrafanaDeployment           = "grafana/deployment.yaml"
-	GrafanaProxySecret          = "grafana/proxy-secret.yaml"
-	GrafanaRoute                = "grafana/route.yaml"
-	GrafanaServiceAccount       = "grafana/service-account.yaml"
-	GrafanaService              = "grafana/service.yaml"
-	GrafanaServiceMonitor       = "grafana/service-monitor.yaml"
-	GrafanaTrustedCABundle      = "grafana/trusted-ca-bundle.yaml"
+	GrafanaClusterRoleBinding    = "grafana/cluster-role-binding.yaml"
+	GrafanaClusterRole           = "grafana/cluster-role.yaml"
+	GrafanaConfigSecret          = "grafana/config.yaml"
+	GrafanaDatasourcesSecret     = "grafana/dashboard-datasources.yaml"
+	GrafanaDashboardDefinitions  = "grafana/dashboard-definitions.yaml"
+	GrafanaDashboardSources      = "grafana/dashboard-sources.yaml"
+	GrafanaDeployment            = "grafana/deployment.yaml"
+	GrafanaRBACProxyMetricSecret = "grafana/kube-rbac-proxy-metric-secret.yaml"
+	GrafanaProxySecret           = "grafana/proxy-secret.yaml"
+	GrafanaRoute                 = "grafana/route.yaml"
+	GrafanaServiceAccount        = "grafana/service-account.yaml"
+	GrafanaService               = "grafana/service.yaml"
+	GrafanaServiceMonitor        = "grafana/service-monitor.yaml"
+	GrafanaTrustedCABundle       = "grafana/trusted-ca-bundle.yaml"
 
 	ClusterMonitoringOperatorService            = "cluster-monitoring-operator/service.yaml"
 	ClusterMonitoringOperatorServiceMonitor     = "cluster-monitoring-operator/service-monitor.yaml"
@@ -2370,6 +2371,17 @@ func (f *Factory) GrafanaDeployment(proxyCABundleCM *v1.ConfigMap) (*appsv1.Depl
 	return d, nil
 }
 
+func (f *Factory) GrafanaRBACProxyMetricSecret() (*v1.Secret, error) {
+	s, err := f.NewSecret(f.assets.MustNewAssetReader(GrafanaRBACProxyMetricSecret))
+	if err != nil {
+		return nil, err
+	}
+
+	s.Namespace = f.namespace
+
+	return s, nil
+}
+
 func (f *Factory) GrafanaProxySecret() (*v1.Secret, error) {
 	s, err := f.NewSecret(f.assets.MustNewAssetReader(GrafanaProxySecret))
 	if err != nil {

diff --git a/pkg/tasks/grafana.go b/pkg/tasks/grafana.go
@@ -16,6 +16,7 @@ package tasks
 
 import (
 	"context"
+
 	"github.com/openshift/cluster-monitoring-operator/pkg/client"
 	"github.com/openshift/cluster-monitoring-operator/pkg/manifests"
 	"github.com/pkg/errors"
@@ -79,6 +80,16 @@ func (t *GrafanaTask) create(ctx context.Context) error {
 		return errors.Wrap(err, "waiting for Grafana Route to become ready failed")
 	}
 
+	rs, err := t.factory.GrafanaRBACProxyMetricSecret()
+	if err != nil {
+		return errors.Wrap(err, "initializing Grafana RBAC proxy metric Secret failed")
+	}
+
+	err = t.client.CreateIfNotExistSecret(ctx, rs)
+	if err != nil {
+		return errors.Wrap(err, "creating Grafana RBAC proxy metric Secret failed")
+	}
+
 	ps, err := t.factory.GrafanaProxySecret()
 	if err != nil {
 		return errors.Wrap(err, "initializing Grafana proxy Secret failed")
@@ -283,6 +294,16 @@ func (t *GrafanaTask) destroy(ctx context.Context) error {
 		return errors.Wrap(err, "deleting Grafana Config Secret failed")
 	}
 
+	rs, err := t.factory.GrafanaRBACProxyMetricSecret()
+	if err != nil {
+		return errors.Wrap(err, "initializing Grafana RBAC proxy metric Secret failed")
+	}
+
+	err = t.client.DeleteSecret(ctx, rs)
+	if err != nil {
+		return errors.Wrap(err, "deleting Grafana RBAC proxy metric Secret failed")
+	}
+
 	ps, err := t.factory.GrafanaProxySecret()
 	if err != nil {
 		return errors.Wrap(err, "initializing Grafana proxy Secret failed")