WIP: remove kube-rbac-proxy sidecar for metrics

assets/alertmanager-user-workload/alertmanager.yaml

@@ -29,6 +29,7 @@ spec:
     - --upstream=http://127.0.0.1:9093
     - --tls-cert-file=/etc/tls/private/tls.crt
     - --tls-private-key-file=/etc/tls/private/tls.key
+    - --client-ca-file=/etc/tls/client/client-ca.crt
     - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
     - --config-file=/etc/kube-rbac-proxy/config.yaml
     - --logtostderr=true
@@ -54,6 +55,9 @@ spec:
     - mountPath: /etc/kube-rbac-proxy
       name: secret-alertmanager-kube-rbac-proxy
       readOnly: true
+    - mountPath: /etc/tls/client
+      name: metrics-client-ca
+      readOnly: true
   - args:
     - --secure-listen-address=0.0.0.0:9092
     - --upstream=http://127.0.0.1:9096
@@ -81,41 +85,6 @@ spec:
       name: secret-alertmanager-kube-rbac-proxy-tenancy
     - mountPath: /etc/tls/private
       name: secret-alertmanager-user-workload-tls
-  - args:
-    - --secure-listen-address=0.0.0.0:9097
-    - --upstream=http://127.0.0.1:9093
-    - --config-file=/etc/kube-rbac-proxy/config.yaml
-    - --tls-cert-file=/etc/tls/private/tls.crt
-    - --tls-private-key-file=/etc/tls/private/tls.key
-    - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
-    - --client-ca-file=/etc/tls/client/client-ca.crt
-    - --logtostderr=true
-    - --allow-paths=/metrics
-    image: quay.io/brancz/kube-rbac-proxy:v0.15.0
-    name: kube-rbac-proxy-metric
-    ports:
-    - containerPort: 9097
-      name: metrics
-    resources:
-      requests:
-        cpu: 1m
-        memory: 15Mi
-    securityContext:
-      allowPrivilegeEscalation: false
-      capabilities:
-        drop:
-        - ALL
-    terminationMessagePolicy: FallbackToLogsOnError
-    volumeMounts:
-    - mountPath: /etc/kube-rbac-proxy
-      name: secret-alertmanager-kube-rbac-proxy-metric
-      readOnly: true
-    - mountPath: /etc/tls/private
-      name: secret-alertmanager-user-workload-tls
-      readOnly: true
-    - mountPath: /etc/tls/client
-      name: metrics-client-ca
-      readOnly: true
   - args:
     - --insecure-listen-address=127.0.0.1:9096
     - --upstream=http://127.0.0.1:9093
@@ -156,7 +125,6 @@ spec:
   - alertmanager-user-workload-tls
   - alertmanager-kube-rbac-proxy
   - alertmanager-kube-rbac-proxy-tenancy
-  - alertmanager-kube-rbac-proxy-metric
   securityContext:
     fsGroup: 65534
     runAsNonRoot: true

assets/alertmanager-user-workload/service-monitor.yaml

@@ -14,7 +14,7 @@ spec:
   endpoints:
   - bearerTokenFile: ""
     interval: 30s
-    port: metrics
+    port: web
     scheme: https
     tlsConfig:
       caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt

assets/alertmanager-user-workload/service.yaml

@@ -20,9 +20,6 @@ spec:
   - name: tenancy
     port: 9092
     targetPort: tenancy
-  - name: metrics
-    port: 9097
-    targetPort: metrics
   selector:
     app.kubernetes.io/component: alert-router
     app.kubernetes.io/instance: user-workload

jsonnet/components/alertmanager-user-workload.libsonnet

@@ -47,11 +47,6 @@ function(params)
             port: 9092,
             targetPort: 'tenancy',
           },
-          {
-            name: 'metrics',
-            port: 9097,
-            targetPort: 'metrics',
-          },
         ],
         type: 'ClusterIP',
       },
@@ -63,7 +58,7 @@ function(params)
       spec+: {
         endpoints: [
           {
-            port: 'metrics',
+            port: 'web',
             interval: '30s',
             scheme: 'https',
             tlsConfig: {
@@ -80,7 +75,6 @@ function(params)
     // In order for kube-rbac-proxy to perform a TokenReview and
     // SubjectAccessReview for authN and authZ the alertmanager ServiceAccount
     // requires the `create` action on both of these.
-
     clusterRole: {
       apiVersion: 'rbac.authorization.k8s.io/v1',
       kind: 'ClusterRole',
@@ -180,12 +174,6 @@ function(params)
       },
     },
 
-    kubeRbacProxyMetricSecret: generateSecret.staticAuthSecret(cfg.namespace, cfg.commonLabels, 'alertmanager-kube-rbac-proxy-metric') + {
-      metadata+: {
-        labels: { 'app.kubernetes.io/name': 'alertmanager-' + cfg.name },
-      },
-    },
-
     alertmanager+: {
       spec+: {
         securityContext: {
@@ -201,7 +189,6 @@ function(params)
           'alertmanager-user-workload-tls',
           $.kubeRbacProxySecret.metadata.name,
           $.kubeRbacProxyTenancySecret.metadata.name,
-          $.kubeRbacProxyMetricSecret.metadata.name,
         ],
         listenLocal: true,
         resources: {
@@ -231,6 +218,7 @@ function(params)
               '--upstream=http://127.0.0.1:9093',
               '--tls-cert-file=/etc/tls/private/tls.crt',
               '--tls-private-key-file=/etc/tls/private/tls.key',
+              '--client-ca-file=/etc/tls/client/client-ca.crt',
               '--tls-cipher-suites=' + cfg.tlsCipherSuites,
               '--config-file=/etc/kube-rbac-proxy/config.yaml',
               '--logtostderr=true',
@@ -247,6 +235,11 @@ function(params)
                 name: 'secret-' + $.kubeRbacProxySecret.metadata.name,
                 readOnly: true,
               },
+              {
+                mountPath: '/etc/tls/client',
+                name: 'metrics-client-ca',
+                readOnly: true,
+              },
             ],
             securityContext: {
               allowPrivilegeEscalation: false,
@@ -296,59 +289,6 @@ function(params)
               },
             },
           },
-          {
-            // TODO: merge this metric proxy with tenancy proxy when the issue below is fixed:
-            // https://github.com/brancz/kube-rbac-proxy/issues/146
-            name: 'kube-rbac-proxy-metric',
-            image: cfg.kubeRbacProxyImage,
-            resources: {
-              requests: {
-                cpu: '1m',
-                memory: '15Mi',
-              },
-            },
-            ports: [
-              {
-                containerPort: 9097,
-                name: 'metrics',
-              },
-            ],
-            args: [
-              '--secure-listen-address=0.0.0.0:9097',
-              '--upstream=http://127.0.0.1:9093',
-              '--config-file=/etc/kube-rbac-proxy/config.yaml',
-              '--tls-cert-file=/etc/tls/private/tls.crt',
-              '--tls-private-key-file=/etc/tls/private/tls.key',
-              '--tls-cipher-suites=' + cfg.tlsCipherSuites,
-              '--client-ca-file=/etc/tls/client/client-ca.crt',
-              '--logtostderr=true',
-              '--allow-paths=/metrics',
-            ],
-            terminationMessagePolicy: 'FallbackToLogsOnError',
-            volumeMounts: [
-              {
-                mountPath: '/etc/kube-rbac-proxy',
-                name: 'secret-' + $.kubeRbacProxyMetricSecret.metadata.name,
-                readOnly: true,
-              },
-              {
-                mountPath: '/etc/tls/private',
-                name: 'secret-alertmanager-user-workload-tls',
-                readOnly: true,
-              },
-              {
-                mountPath: '/etc/tls/client',
-                name: 'metrics-client-ca',
-                readOnly: true,
-              },
-            ],
-            securityContext: {
-              allowPrivilegeEscalation: false,
-              capabilities: {
-                drop: ['ALL'],
-              },
-            },
-          },
           {
             name: 'prom-label-proxy',
             image: cfg.promLabelProxyImage,

pkg/client/client.go

@@ -849,12 +849,16 @@ func (c *Client) DeletePrometheusRuleByNamespaceAndName(ctx context.Context, nam
 }
 
 func (c *Client) DeleteSecret(ctx context.Context, s *v1.Secret) error {
-	err := c.kclient.CoreV1().Secrets(s.Namespace).Delete(ctx, s.GetName(), metav1.DeleteOptions{})
-	if apierrors.IsNotFound(err) {
-		return nil
+	return c.DeleteSecretByNamespaceAndName(ctx, s.GetNamespace(), s.GetName())
+}
+
+func (c *Client) DeleteSecretByNamespaceAndName(ctx context.Context, namespace, name string) error {
+	err := c.kclient.CoreV1().Secrets(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+	if err != nil && !apierrors.IsNotFound(err) {
+		return fmt.Errorf("deleting Secret object failed: %w", err)
 	}
 
-	return err
+	return nil
 }
 
 // validatePrometheusResource is a helper method for ValidatePrometheus.

pkg/manifests/manifests.go

@@ -95,7 +95,6 @@ var (
 	AlertmanagerUserWorkloadClusterRole            = "alertmanager-user-workload/cluster-role.yaml"
 	AlertmanagerUserWorkloadRBACProxySecret        = "alertmanager-user-workload/kube-rbac-proxy-secret.yaml"
 	AlertmanagerUserWorkloadRBACProxyTenancySecret = "alertmanager-user-workload/kube-rbac-proxy-tenancy-secret.yaml"
-	AlertmanagerUserWorkloadRBACProxyMetricSecret  = "alertmanager-user-workload/kube-rbac-proxy-metric-secret.yaml"
 	AlertmanagerUserWorkloadTrustedCABundle        = "alertmanager-user-workload/trusted-ca-bundle.yaml"
 	AlertmanagerUserWorkloadPodDisruptionBudget    = "alertmanager-user-workload/pod-disruption-budget.yaml"
 	AlertmanagerUserWorkloadServiceMonitor         = "alertmanager-user-workload/service-monitor.yaml"
@@ -717,10 +716,6 @@ func (f *Factory) AlertmanagerRBACProxyMetricSecret() (*v1.Secret, error) {
 	return f.NewSecret(f.assets.MustNewAssetReader(AlertmanagerRBACProxyMetricSecret))
 }
 
-func (f *Factory) AlertmanagerUserWorkloadRBACProxyMetricSecret() (*v1.Secret, error) {
-	return f.NewSecret(f.assets.MustNewAssetReader(AlertmanagerUserWorkloadRBACProxyMetricSecret))
-}
-
 func (f *Factory) AlertmanagerRoute() (*routev1.Route, error) {
 	return f.NewRoute(f.assets.MustNewAssetReader(AlertmanagerRoute))
 }

pkg/tasks/alertmanager_user_workload.go

@@ -92,14 +92,9 @@ func (t *AlertmanagerUserWorkloadTask) create(ctx context.Context) error {
 		return errors.Wrap(err, "creating Alertmanager User Workload RBAC proxy tenancy Secret failed")
 	}
 
-	rsm, err := t.factory.AlertmanagerUserWorkloadRBACProxyMetricSecret()
+	err = t.client.DeleteSecretByNamespaceAndName(ctx, "openshift-user-workload-monitoring", "alertmanager-kube-rbac-proxy-metric")
 	if err != nil {
-		return errors.Wrap(err, "initializing Alertmanager User Workload RBAC proxy metric Secret failed")
-	}
-
-	err = t.client.CreateIfNotExistSecret(ctx, rsm)
-	if err != nil {
-		return errors.Wrap(err, "creating Alertmanager User Workload RBAC proxy metric Secret failed")
+		return errors.Wrap(err, "deleting Alertmanager User Workload RBAC proxy metric Secret failed")
 	}
 
 	if t.config.UserWorkloadConfiguration.Alertmanager.Secrets != nil {
@@ -226,16 +221,6 @@ func (t *AlertmanagerUserWorkloadTask) destroy(ctx context.Context) error {
 		return errors.Wrap(err, "deleting Alertmanager User Workload RBAC proxy tenancy Secret failed")
 	}
 
-	rsm, err := t.factory.AlertmanagerUserWorkloadRBACProxyMetricSecret()
-	if err != nil {
-		return errors.Wrap(err, "initializing Alertmanager User Workload RBAC proxy metric Secret failed")
-	}
-
-	err = t.client.DeleteSecret(ctx, rsm)
-	if err != nil {
-		return errors.Wrap(err, "deleting Alertmanager User Workload RBAC proxy metric Secret failed")
-	}
-
 	cr, err := t.factory.AlertmanagerUserWorkloadClusterRole()
 	if err != nil {
 		return errors.Wrap(err, "initializing Alertmanager User Workload ClusterRole failed")