remove bearer token from service monitors

openshift · Sep 27, 2021 · 7856030 · 7856030
1 parent 02fed16
commit 7856030
Show file tree

Hide file tree

Showing 7 changed files with 193 additions and 17 deletions.
diff --git a/assets/alertmanager/alertmanager.yaml b/assets/alertmanager/alertmanager.yaml
@@ -72,6 +72,7 @@ spec:
     - --tls-cert-file=/etc/tls/private/tls.crt
     - --tls-private-key-file=/etc/tls/private/tls.key
     - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+    - --client-ca-file=/etc/tls/client/client-ca.crt
     - --logtostderr=true
     - --v=10
     image: quay.io/brancz/kube-rbac-proxy:v0.11.0
@@ -89,10 +90,44 @@ spec:
       name: secret-alertmanager-kube-rbac-proxy
     - mountPath: /etc/tls/private
       name: secret-alertmanager-main-tls
+    - mountPath: /etc/tls/client
+      name: metrics-client-ca
+      readOnly: false
+  - args:
+    - --secure-listen-address=0.0.0.0:9097
+    - --upstream=http://127.0.0.1:9096
+    - --config-file=/etc/kube-rbac-proxy/config.yaml
+    - --tls-cert-file=/etc/tls/private/tls.crt
+    - --tls-private-key-file=/etc/tls/private/tls.key
+    - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+    - --client-ca-file=/etc/tls/client/client-ca.crt
+    - --logtostderr=true
+    - --v=10
+    image: quay.io/brancz/kube-rbac-proxy:v0.11.0
+    name: kube-rbac-proxy-metric
+    ports:
+    - containerPort: 9097
+      name: metric
+    resources:
+      requests:
+        cpu: 1m
+        memory: 15Mi
+    terminationMessagePolicy: FallbackToLogsOnError
+    volumeMounts:
+    - mountPath: /etc/kube-rbac-proxy
+      name: secret-alertmanager-kube-rbac-proxy-metric
+      readOnly: true
+    - mountPath: /etc/tls/private
+      name: secret-alertmanager-main-tls
+      readOnly: true
+    - mountPath: /etc/tls/client
+      name: metrics-client-ca
+      readOnly: true
   - args:
     - --insecure-listen-address=127.0.0.1:9096
     - --upstream=http://127.0.0.1:9093
     - --label=namespace
+    - --unsafe-passthrough-paths=/metrics
     image: quay.io/prometheuscommunity/prom-label-proxy:v0.3.0
     name: prom-label-proxy
     resources:
@@ -127,9 +162,17 @@ spec:
   - alertmanager-main-tls
   - alertmanager-main-proxy
   - alertmanager-kube-rbac-proxy
+  - alertmanager-kube-rbac-proxy-metric
   securityContext:
     fsGroup: 65534
     runAsNonRoot: true
     runAsUser: 65534
   serviceAccountName: alertmanager-main
   version: 0.22.2
+  volumes:
+  - configMap:
+      name: metrics-client-ca
+    name: metrics-client-ca
+  - name: secret-alertmanager-kube-rbac-proxy-metric
+    secret:
+      secretName: alertmanager-kube-rbac-proxy-metric
diff --git a/assets/alertmanager/kube-rbac-proxy-metric-secret.yaml b/assets/alertmanager/kube-rbac-proxy-metric-secret.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+data: {}
+kind: Secret
+metadata:
+  labels:
+    app.kubernetes.io/name: alertmanager-main
+  name: alertmanager-kube-rbac-proxy-metric
+  namespace: openshift-monitoring
+stringData:
+  config.yaml: |-
+    "authorization":
+      "static":
+      - "path": "/metrics"
+        "resourceRequest": false
+        "user":
+          "name": "system:serviceaccount:openshift-monitoring:prometheus-k8s"
+        "verb": "get"
+type: Opaque
diff --git a/assets/alertmanager/service-monitor.yaml b/assets/alertmanager/service-monitor.yaml
@@ -10,9 +10,8 @@ metadata:
   namespace: openshift-monitoring
 spec:
   endpoints:
-  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-    interval: 30s
-    port: web
+  - interval: 30s
+    port: metric
     scheme: https
     tlsConfig:
       caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt

diff --git a/assets/alertmanager/service.yaml b/assets/alertmanager/service.yaml
@@ -19,6 +19,9 @@ spec:
   - name: tenancy
     port: 9092
     targetPort: tenancy
+  - name: metric
+    port: 9097
+    targetPort: metric
   selector:
     alertmanager: main
     app: alertmanager

diff --git a/jsonnet/components/alertmanager.libsonnet b/jsonnet/components/alertmanager.libsonnet
@@ -1,6 +1,7 @@
 local alertmanager = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/alertmanager.libsonnet';
 // TODO: replace current addition of kube-rbac-proxy with upstream lib
 // local krp = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/kube-rbac-proxy.libsonnet';
+local generateSecret = import '../utils/generate-secret.libsonnet';
 
 function(params)
   local cfg = params;
@@ -86,6 +87,11 @@ function(params)
             port: 9092,
             targetPort: 'tenancy',
           },
+          {
+            name: 'metric',
+            port: 9097,
+            targetPort: 'metric',
+          },
         ],
         type: 'ClusterIP',
       },
@@ -183,13 +189,19 @@ function(params)
       },
     },
 
+    kubeRbacProxyMetricSecret: generateSecret.staticAuthSecret(cfg.namespace, cfg.commonLabels, 'alertmanager-kube-rbac-proxy-metric') + {
+      metadata+: {
+        labels: { 'app.kubernetes.io/name': 'alertmanager-main' },
+      },
+    },
+
     // This changes the alertmanager to be scraped with TLS, authN and authZ,
     // which are not present in kube-prometheus.
     serviceMonitor+: {
       spec+: {
         endpoints: [
           {
-            port: 'web',
+            port: 'metric',
             interval: '30s',
             scheme: 'https',
             tlsConfig: {
@@ -198,7 +210,6 @@ function(params)
               certFile: '/etc/prometheus/secrets/metrics-client-certs/tls.crt',
               keyFile: '/etc/prometheus/secrets/metrics-client-certs/tls.key',
             },
-            bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
           },
         ],
       },
@@ -218,6 +229,7 @@ function(params)
           'alertmanager-main-tls',
           'alertmanager-main-proxy',
           $.kubeRbacProxySecret.metadata.name,
+          $.kubeRbacProxyMetricSecret.metadata.name,
         ],
         listenLocal: true,
         resources: {
@@ -306,6 +318,7 @@ function(params)
               '--tls-cert-file=/etc/tls/private/tls.crt',
               '--tls-private-key-file=/etc/tls/private/tls.key',
               '--tls-cipher-suites=' + cfg.tlsCipherSuites,
+              '--client-ca-file=/etc/tls/client/client-ca.crt',
               '--logtostderr=true',
               '--v=10',
             ],
@@ -319,6 +332,56 @@ function(params)
                 mountPath: '/etc/tls/private',
                 name: 'secret-alertmanager-main-tls',
               },
+              {
+                mountPath: '/etc/tls/client',
+                name: 'metrics-client-ca',
+                readOnly: false,
+              },
+            ],
+          },
+          {
+            name: 'kube-rbac-proxy-metric',
+            image: cfg.kubeRbacProxyImage,
+            resources: {
+              requests: {
+                cpu: '1m',
+                memory: '15Mi',
+              },
+            },
+            ports: [
+              {
+                containerPort: 9097,
+                name: 'metric',
+              },
+            ],
+            args: [
+              '--secure-listen-address=0.0.0.0:9097',
+              '--upstream=http://127.0.0.1:9096',
+              '--config-file=/etc/kube-rbac-proxy/config.yaml',
+              '--tls-cert-file=/etc/tls/private/tls.crt',
+              '--tls-private-key-file=/etc/tls/private/tls.key',
+              '--tls-cipher-suites=' + cfg.tlsCipherSuites,
+              '--client-ca-file=/etc/tls/client/client-ca.crt',
+              '--logtostderr=true',
+              '--v=10',
+            ],
+            terminationMessagePolicy: 'FallbackToLogsOnError',
+            volumeMounts: [
+              {
+                mountPath: '/etc/kube-rbac-proxy',
+                name: 'secret-' + $.kubeRbacProxyMetricSecret.metadata.name,
+                readOnly: true,
+              },
+              {
+                mountPath: '/etc/tls/private',
+                name: 'secret-alertmanager-main-tls',
+                readOnly: true,
+              },
+              {
+                mountPath: '/etc/tls/client',
+                name: 'metrics-client-ca',
+                readOnly: true,
+              },
             ],
           },
           {
@@ -328,6 +391,7 @@ function(params)
               '--insecure-listen-address=127.0.0.1:9096',
               '--upstream=http://127.0.0.1:9093',
               '--label=namespace',
+              '--unsafe-passthrough-paths=/metrics',
             ],
             resources: {
               requests: {
@@ -347,6 +411,20 @@ function(params)
             },
           },
         ],
+        volumes+: [
+          {
+            name: 'metrics-client-ca',
+            configMap: {
+              name: 'metrics-client-ca',
+            },
+          },
+          {
+            name: 'secret-' + $.kubeRbacProxyMetricSecret.metadata.name,
+            secret: {
+              secretName: $.kubeRbacProxyMetricSecret.metadata.name,
+            },
+          },
+        ],
       },
     },
     // Removing PDB since it doesn't allow cluster upgrade when hard pod anti affinity is not set https://github.com/openshift/cluster-monitoring-operator/pull/1198

diff --git a/pkg/manifests/manifests.go b/pkg/manifests/manifests.go
@@ -53,18 +53,19 @@ const (
 )
 
 var (
-	AlertmanagerConfig             = "alertmanager/secret.yaml"
-	AlertmanagerService            = "alertmanager/service.yaml"
-	AlertmanagerProxySecret        = "alertmanager/proxy-secret.yaml"
-	AlertmanagerMain               = "alertmanager/alertmanager.yaml"
-	AlertmanagerServiceAccount     = "alertmanager/service-account.yaml"
-	AlertmanagerClusterRoleBinding = "alertmanager/cluster-role-binding.yaml"
-	AlertmanagerClusterRole        = "alertmanager/cluster-role.yaml"
-	AlertmanagerRBACProxySecret    = "alertmanager/kube-rbac-proxy-secret.yaml"
-	AlertmanagerRoute              = "alertmanager/route.yaml"
-	AlertmanagerServiceMonitor     = "alertmanager/service-monitor.yaml"
-	AlertmanagerTrustedCABundle    = "alertmanager/trusted-ca-bundle.yaml"
-	AlertmanagerPrometheusRule     = "alertmanager/prometheus-rule.yaml"
+	AlertmanagerConfig                = "alertmanager/secret.yaml"
+	AlertmanagerService               = "alertmanager/service.yaml"
+	AlertmanagerProxySecret           = "alertmanager/proxy-secret.yaml"
+	AlertmanagerMain                  = "alertmanager/alertmanager.yaml"
+	AlertmanagerServiceAccount        = "alertmanager/service-account.yaml"
+	AlertmanagerClusterRoleBinding    = "alertmanager/cluster-role-binding.yaml"
+	AlertmanagerClusterRole           = "alertmanager/cluster-role.yaml"
+	AlertmanagerRBACProxySecret       = "alertmanager/kube-rbac-proxy-secret.yaml"
+	AlertmanagerRBACProxyMetricSecret = "alertmanager/kube-rbac-proxy-metric-secret.yaml"
+	AlertmanagerRoute                 = "alertmanager/route.yaml"
+	AlertmanagerServiceMonitor        = "alertmanager/service-monitor.yaml"
+	AlertmanagerTrustedCABundle       = "alertmanager/trusted-ca-bundle.yaml"
+	AlertmanagerPrometheusRule        = "alertmanager/prometheus-rule.yaml"
 
 	KubeStateMetricsClusterRoleBinding  = "kube-state-metrics/cluster-role-binding.yaml"
 	KubeStateMetricsClusterRole         = "kube-state-metrics/cluster-role.yaml"
@@ -468,6 +469,8 @@ func (f *Factory) AlertmanagerMain(host string, trustedCABundleCM *v1.ConfigMap)
 			}
 		case "kube-rbac-proxy":
 			a.Spec.Containers[i].Image = f.config.Images.KubeRbacProxy
+		case "kube-rbac-proxy-mertic":
+			a.Spec.Containers[i].Image = f.config.Images.KubeRbacProxy
 		case "prom-label-proxy":
 			a.Spec.Containers[i].Image = f.config.Images.PromLabelProxy
 		}
@@ -489,6 +492,17 @@ func (f *Factory) AlertmanagerRBACProxySecret() (*v1.Secret, error) {
 	return s, nil
 }
 
+func (f *Factory) AlertmanagerRBACProxyMetricSecret() (*v1.Secret, error) {
+	s, err := f.NewSecret(f.assets.MustNewAssetReader(AlertmanagerRBACProxyMetricSecret))
+	if err != nil {
+		return nil, err
+	}
+
+	s.Namespace = f.namespace
+
+	return s, nil
+}
+
 func (f *Factory) AlertmanagerRoute() (*routev1.Route, error) {
 	r, err := f.NewRoute(f.assets.MustNewAssetReader(AlertmanagerRoute))
 	if err != nil {

diff --git a/pkg/tasks/alertmanager.go b/pkg/tasks/alertmanager.go
@@ -16,6 +16,7 @@ package tasks
 
 import (
 	"context"
+
 	"github.com/openshift/cluster-monitoring-operator/pkg/client"
 	"github.com/openshift/cluster-monitoring-operator/pkg/manifests"
 	"github.com/pkg/errors"
@@ -83,6 +84,16 @@ func (t *AlertmanagerTask) create(ctx context.Context) error {
 		return errors.Wrap(err, "creating Alertmanager RBAC proxy Secret failed")
 	}
 
+	rsm, err := t.factory.AlertmanagerRBACProxyMetricSecret()
+	if err != nil {
+		return errors.Wrap(err, "initializing Alertmanager RBAC proxy metric Secret failed")
+	}
+
+	err = t.client.CreateIfNotExistSecret(ctx, rsm)
+	if err != nil {
+		return errors.Wrap(err, "creating Alertmanager RBAC proxy metric Secret failed")
+	}
+
 	cr, err := t.factory.AlertmanagerClusterRole()
 	if err != nil {
 		return errors.Wrap(err, "initializing Alertmanager ClusterRole failed")
@@ -213,6 +224,16 @@ func (t *AlertmanagerTask) destroy(ctx context.Context) error {
 		return errors.Wrap(err, "deleting Alertmanager RBAC proxy Secret failed")
 	}
 
+	rsm, err := t.factory.AlertmanagerRBACProxyMetricSecret()
+	if err != nil {
+		return errors.Wrap(err, "initializing Alertmanager RBAC proxy metric Secret failed")
+	}
+
+	err = t.client.DeleteSecret(ctx, rsm)
+	if err != nil {
+		return errors.Wrap(err, "deleting Alertmanager RBAC proxy metric Secret failed")
+	}
+
 	cr, err := t.factory.AlertmanagerClusterRole()
 	if err != nil {
 		return errors.Wrap(err, "initializing Alertmanager ClusterRole failed")