Do the relabeling before the scrape (#734)

Signed-off-by: Pete Wall <[email protected]>

charts/k8s-monitoring/README.md

@@ -667,6 +667,7 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | metrics.kubernetesMonitoring.enabled | bool | `true` | Report telemetry about this Kubernetes Monitoring chart as a metric. |
+| metrics.kubernetesMonitoring.scrapeInterval | string | 60s | How frequently to scrape the telemetry report metric. Overrides metrics.scrapeInterval |
 
 ### Metrics Job: Node Exporter
 

charts/k8s-monitoring/templates/alloy_config/_kubernetes_monitoring_telemetry.alloy.txt

@@ -7,35 +7,29 @@ prometheus.exporter.unix "kubernetes_monitoring_telemetry" {
   }
 }
 
-prometheus.scrape "kubernetes_monitoring_telemetry" {
-  job_name   = "integrations/kubernetes/kubernetes_monitoring_telemetry"
-  targets    = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets
-  scrape_interval = {{ .Values.metrics.scrapeInterval | quote }}
-{{- if .Values.alloy.alloy.clustering.enabled }}
-  clustering {
-    enabled = true
-  }
-{{- end }}
-  forward_to = [prometheus.relabel.kubernetes_monitoring_telemetry.receiver]
-}
-
-prometheus.relabel "kubernetes_monitoring_telemetry" {
-  max_cache_size = {{ .Values.metrics.maxCacheSize | int }}
-  rule {
-    target_label = "job"
-    action = "replace"
-    replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry"
-  }
+discovery.relabel "kubernetes_monitoring_telemetry" {
+  targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets
   rule {
     target_label = "instance"
     action = "replace"
     replacement = "{{ .Release.Name }}"
   }
   rule {
-    source_labels = ["__name__"]
-    regex = "up|grafana_kubernetes_monitoring_.*"
-    action = "keep"
+    target_label = "job"
+    action = "replace"
+    replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry"
   }
+}
+
+prometheus.scrape "kubernetes_monitoring_telemetry" {
+  job_name   = "integrations/kubernetes/kubernetes_monitoring_telemetry"
+  targets    = discovery.relabel.kubernetes_monitoring_telemetry.output
+  scrape_interval = {{ .Values.metrics.kubernetesMonitoring.scrapeInterval | default .Values.metrics.scrapeInterval | quote }}
+{{- if .Values.alloy.alloy.clustering.enabled }}
+  clustering {
+    enabled = true
+  }
+{{- end }}
   forward_to = [prometheus.relabel.metrics_service.receiver]
 }
 {{ end }}

charts/k8s-monitoring/values.yaml

@@ -1458,6 +1458,12 @@ metrics:
     # @section -- Metrics Job: Kubernetes Monitoring Telemetry
     enabled: true
 
+    # -- How frequently to scrape the telemetry report metric.
+    # Overrides metrics.scrapeInterval
+    # @section -- Metrics Job: Kubernetes Monitoring Telemetry
+    # @default -- 60s
+    scrapeInterval:
+
   # Alloy Modules
   # Modules can be invoked using metrics.extraConfig, this block is consuming opinionated modules from the grafana/alloy-modules repository
   # or any other repository that follows the same module structure.  Each module is expected to have a "kubernetes" module and a "scrape" module.