From 427fac7f78863e3f86af0a9098416ea3a88aaf44 Mon Sep 17 00:00:00 2001 From: Aaron Date: Fri, 25 Oct 2024 11:31:04 -0400 Subject: [PATCH] Added Profiles extraConfig Support (#819) --- charts/k8s-monitoring-v1/README.md | 6 + .../docs/examples/custom-config/output.yaml | 2150 +++++++++++++++++ .../examples/custom-config/profiles.alloy | 932 +++++++ .../docs/examples/custom-config/values.yaml | 19 + .../k8s-monitoring-v1/templates/_configs.tpl | 3 + charts/k8s-monitoring-v1/values.schema.json | 3 + charts/k8s-monitoring-v1/values.yaml | 7 + 7 files changed, 3120 insertions(+) diff --git a/charts/k8s-monitoring-v1/README.md b/charts/k8s-monitoring-v1/README.md index 723b92b89..b1b34727b 100644 --- a/charts/k8s-monitoring-v1/README.md +++ b/charts/k8s-monitoring-v1/README.md @@ -799,6 +799,12 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely |-----|------|---------|-------------| | profiles.enabled | bool | `false` | Receive and forward profiles. | +### Profiles Global + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| profiles.extraConfig | string | `""` | Extra configuration that will be added to the Grafana Alloy for Logs configuration file. This value is templated so that you can refer to other values from this file. This cannot be used to modify the generated configuration values, only append new components. See [Adding custom Flow configuration](#adding-custom-flow-configuration) for an example. | + ### Profiles (java) | Key | Type | Default | Description | diff --git a/charts/k8s-monitoring-v1/docs/examples/custom-config/output.yaml b/charts/k8s-monitoring-v1/docs/examples/custom-config/output.yaml index 6f6511e22..8f240834e 100644 --- a/charts/k8s-monitoring-v1/docs/examples/custom-config/output.yaml +++ b/charts/k8s-monitoring-v1/docs/examples/custom-config/output.yaml @@ -31,6 +31,22 @@ metadata: app.kubernetes.io/part-of: alloy app.kubernetes.io/component: rbac --- +# Source: k8s-monitoring/charts/alloy-profiles/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k8smon-alloy-profiles + namespace: default + labels: + helm.sh/chart: alloy-profiles-0.9.1 + app.kubernetes.io/name: alloy-profiles + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.4.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: rbac +--- # Source: k8s-monitoring/charts/alloy/templates/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount @@ -122,6 +138,19 @@ data: username: "MTIzNDU=" password: "SXQncyBhIHNlY3JldCB0byBldmVyeW9uZQ==" --- +# Source: k8s-monitoring/templates/profiles-service-credentials.yaml +apiVersion: v1 +kind: Secret +metadata: + name: pyroscope-k8s-monitoring + namespace: default +type: Opaque +data: + + host: "aHR0cHM6Ly9weXJvc2NvcGUuZXhhbXBsZS5jb20=" + username: "MTIzNDU=" + password: "SXQncyBhIHNlY3JldCB0byBldmVyeW9uZQ==" +--- # Source: k8s-monitoring/templates/alloy-config.yaml apiVersion: v1 kind: ConfigMap @@ -1314,6 +1343,947 @@ data: namespace = "default" } --- +# Source: k8s-monitoring/templates/alloy-profiles-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: k8smon-alloy-profiles + namespace: default +data: + config.alloy: |- + // Profiles: eBPF + discovery.kubernetes "ebpf_pods" { + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + role = "pod" + } + + discovery.relabel "ebpf_pods" { + targets = discovery.kubernetes.ebpf_pods.targets + rule { + action = "drop" + regex = "Succeeded|Failed|Completed" + source_labels = ["__meta_kubernetes_pod_phase"] + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + // provide arbitrary service_name label, otherwise it will be set to {__meta_kubernetes_namespace}/{__meta_kubernetes_pod_container_name} + rule { + action = "replace" + regex = "(.*)@(.*)" + replacement = "ebpf/${1}/${2}" + separator = "@" + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + target_label = "service_name" + } + } + + pyroscope.ebpf "ebpf_pods" { + targets = discovery.relabel.ebpf_pods.output + + demangle = "none" + + forward_to = [pyroscope.write.profiles_service.receiver] + } + // Profiles: Java + discovery.kubernetes "java_pods" { + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + role = "pod" + } + + discovery.process "java_pods" { + join = discovery.kubernetes.java_pods.targets + } + + discovery.relabel "java_pods" { + targets = discovery.process.java_pods.targets + rule { + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "Succeeded|Failed|Completed" + action = "drop" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + regex = "^$" + action = "drop" + } + rule { + source_labels = ["__meta_process_exe"] + action = "keep" + regex = ".*/java$" + } + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + action = "replace" + target_label = "node" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + } + + pyroscope.java "java_pods" { + targets = discovery.relabel.java_pods.output + profiling_config { + interval = "60s" + alloc = "512k" + cpu = true + sample_rate = 100 + lock = "10ms" + } + forward_to = [pyroscope.write.profiles_service.receiver] + } + // Profiles: pprof + discovery.kubernetes "pprof_pods" { + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + role = "pod" + } + + discovery.relabel "pprof_pods" { + targets = concat(discovery.kubernetes.pprof_pods.targets) + + rule { + action = "drop" + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "Pending|Succeeded|Failed|Completed" + } + + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_label_(.+)" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + } + + discovery.relabel "pprof_pods_memory_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + discovery.relabel "pprof_pods_memory_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + pyroscope.scrape "pyroscope_scrape_memory" { + targets = concat(discovery.relabel.pprof_pods_memory_default_name.output, discovery.relabel.pprof_pods_memory_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = true + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] + } + discovery.relabel "pprof_pods_cpu_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + discovery.relabel "pprof_pods_cpu_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + pyroscope.scrape "pyroscope_scrape_cpu" { + targets = concat(discovery.relabel.pprof_pods_cpu_default_name.output, discovery.relabel.pprof_pods_cpu_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = true + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] + } + discovery.relabel "pprof_pods_goroutine_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + discovery.relabel "pprof_pods_goroutine_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + pyroscope.scrape "pyroscope_scrape_goroutine" { + targets = concat(discovery.relabel.pprof_pods_goroutine_default_name.output, discovery.relabel.pprof_pods_goroutine_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = true + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] + } + discovery.relabel "pprof_pods_block_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + discovery.relabel "pprof_pods_block_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + pyroscope.scrape "pyroscope_scrape_block" { + targets = concat(discovery.relabel.pprof_pods_block_default_name.output, discovery.relabel.pprof_pods_block_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = true + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] + } + discovery.relabel "pprof_pods_mutex_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + discovery.relabel "pprof_pods_mutex_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + pyroscope.scrape "pyroscope_scrape_mutex" { + targets = concat(discovery.relabel.pprof_pods_mutex_default_name.output, discovery.relabel.pprof_pods_mutex_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = true + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] + } + discovery.relabel "pprof_pods_fgprof_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + discovery.relabel "pprof_pods_fgprof_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + pyroscope.scrape "pyroscope_scrape_fgprof" { + targets = concat(discovery.relabel.pprof_pods_fgprof_default_name.output, discovery.relabel.pprof_pods_fgprof_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = true + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] + } + // Pyroscope + remote.kubernetes.secret "profiles_service" { + name = "pyroscope-k8s-monitoring" + namespace = "default" + } + pyroscope.write "profiles_service" { + endpoint { + url = nonsensitive(remote.kubernetes.secret.profiles_service.data["host"]) + headers = { + "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.profiles_service.data["tenantId"]), + } + + basic_auth { + username = nonsensitive(remote.kubernetes.secret.profiles_service.data["username"]) + password = remote.kubernetes.secret.profiles_service.data["password"] + } + } + external_labels = { + region = "southwest", + tenant = "widgetco", + env = remote.kubernetes.configmap.cluster_info.data["env"], + region = remote.kubernetes.configmap.cluster_info.data["region"], + cluster = "custom-config-test", + } + } + + logging { + level = "info" + format = "logfmt" + } + + remote.kubernetes.configmap "cluster_info" { + name = "cluster_info" + namespace = "default" + } +--- # Source: k8s-monitoring/charts/prometheus-operator-crds/charts/crds/templates/crd-alertmanagerconfigs.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -66261,6 +67231,106 @@ rules: resources: ["replicasets"] verbs: ["get", "list", "watch"] --- +# Source: k8s-monitoring/charts/alloy-profiles/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: k8smon-alloy-profiles + labels: + helm.sh/chart: alloy-profiles-0.9.1 + app.kubernetes.io/name: alloy-profiles + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.4.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: rbac +rules: + # Rules which allow discovery.kubernetes to function. + - apiGroups: + - "" + - "discovery.k8s.io" + - "networking.k8s.io" + resources: + - endpoints + - endpointslices + - ingresses + - nodes + - nodes/proxy + - nodes/metrics + - pods + - services + verbs: + - get + - list + - watch + # Rules which allow loki.source.kubernetes and loki.source.podlogs to work. + - apiGroups: + - "" + resources: + - pods + - pods/log + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - "monitoring.grafana.com" + resources: + - podlogs + verbs: + - get + - list + - watch + # Rules which allow mimir.rules.kubernetes to work. + - apiGroups: ["monitoring.coreos.com"] + resources: + - prometheusrules + verbs: + - get + - list + - watch + - nonResourceURLs: + - /metrics + verbs: + - get + # Rules for prometheus.kubernetes.* + - apiGroups: ["monitoring.coreos.com"] + resources: + - podmonitors + - servicemonitors + - probes + verbs: + - get + - list + - watch + # Rules which allow eventhandler to work. + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + # needed for remote.kubernetes.* + - apiGroups: [""] + resources: + - "configmaps" + - "secrets" + verbs: + - get + - list + - watch + # needed for otelcol.processor.k8sattributes + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["extensions"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] +--- # Source: k8s-monitoring/charts/alloy/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -66650,6 +67720,29 @@ subjects: name: k8smon-alloy-logs namespace: default --- +# Source: k8s-monitoring/charts/alloy-profiles/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: k8smon-alloy-profiles + labels: + helm.sh/chart: alloy-profiles-0.9.1 + app.kubernetes.io/name: alloy-profiles + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.4.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: rbac +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k8smon-alloy-profiles +subjects: + - kind: ServiceAccount + name: k8smon-alloy-profiles + namespace: default +--- # Source: k8s-monitoring/charts/alloy/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -66769,6 +67862,32 @@ spec: targetPort: 12345 protocol: "TCP" --- +# Source: k8s-monitoring/charts/alloy-profiles/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: k8smon-alloy-profiles + labels: + helm.sh/chart: alloy-profiles-0.9.1 + app.kubernetes.io/name: alloy-profiles + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.4.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: networking +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: alloy-profiles + app.kubernetes.io/instance: k8smon + internalTrafficPolicy: Cluster + ports: + - name: http-metrics + port: 12345 + targetPort: 12345 + protocol: "TCP" +--- # Source: k8s-monitoring/charts/alloy/templates/cluster_service.yaml apiVersion: v1 kind: Service @@ -67115,6 +68234,94 @@ spec: hostPath: path: /var/log --- +# Source: k8s-monitoring/charts/alloy-profiles/templates/controllers/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: k8smon-alloy-profiles + labels: + helm.sh/chart: alloy-profiles-0.9.1 + app.kubernetes.io/name: alloy-profiles + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.4.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy +spec: + minReadySeconds: 10 + selector: + matchLabels: + app.kubernetes.io/name: alloy-profiles + app.kubernetes.io/instance: k8smon + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: alloy + k8s.grafana.com/logs.job: integrations/alloy + labels: + app.kubernetes.io/name: alloy-profiles + app.kubernetes.io/instance: k8smon + spec: + serviceAccountName: k8smon-alloy-profiles + containers: + - name: alloy + image: docker.io/grafana/alloy:v1.4.2 + imagePullPolicy: IfNotPresent + args: + - run + - /etc/alloy/config.alloy + - --storage.path=/tmp/alloy + - --server.http.listen-addr=0.0.0.0:12345 + - --server.http.ui-path-prefix=/ + - --stability.level=public-preview + env: + - name: ALLOY_DEPLOY_MODE + value: "helm" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - containerPort: 12345 + name: http-metrics + readinessProbe: + httpGet: + path: /-/ready + port: 12345 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 1 + securityContext: + privileged: true + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - name: config + mountPath: /etc/alloy + - name: config-reloader + image: ghcr.io/jimmidyson/configmap-reload:v0.12.0 + args: + - --volume-dir=/etc/alloy + - --webhook-url=http://localhost:12345/-/reload + volumeMounts: + - name: config + mountPath: /etc/alloy + resources: + requests: + cpu: 1m + memory: 5Mi + hostPID: true + dnsPolicy: ClusterFirst + nodeSelector: + kubernetes.io/os: linux + tolerations: + - effect: NoSchedule + operator: Exists + volumes: + - name: config + configMap: + name: k8smon-alloy-profiles +--- # Source: k8s-monitoring/charts/prometheus-node-exporter/templates/daemonset.yaml apiVersion: apps/v1 kind: DaemonSet @@ -68865,6 +70072,939 @@ data: forward_to = [loki.process.logs_service.receiver] } + remote.kubernetes.configmap "cluster_info" { + name = "cluster_info" + namespace = "default" + } + profiles.alloy: |- + // Profiles: eBPF + discovery.kubernetes "ebpf_pods" { + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + role = "pod" + } + + discovery.relabel "ebpf_pods" { + targets = discovery.kubernetes.ebpf_pods.targets + rule { + action = "drop" + regex = "Succeeded|Failed|Completed" + source_labels = ["__meta_kubernetes_pod_phase"] + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + // provide arbitrary service_name label, otherwise it will be set to {__meta_kubernetes_namespace}/{__meta_kubernetes_pod_container_name} + rule { + action = "replace" + regex = "(.*)@(.*)" + replacement = "ebpf/${1}/${2}" + separator = "@" + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + target_label = "service_name" + } + } + + pyroscope.ebpf "ebpf_pods" { + targets = discovery.relabel.ebpf_pods.output + + demangle = "none" + + forward_to = [pyroscope.write.profiles_service.receiver] + } + // Profiles: Java + discovery.kubernetes "java_pods" { + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + role = "pod" + } + + discovery.process "java_pods" { + join = discovery.kubernetes.java_pods.targets + } + + discovery.relabel "java_pods" { + targets = discovery.process.java_pods.targets + rule { + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "Succeeded|Failed|Completed" + action = "drop" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + regex = "^$" + action = "drop" + } + rule { + source_labels = ["__meta_process_exe"] + action = "keep" + regex = ".*/java$" + } + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + action = "replace" + target_label = "node" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + } + + pyroscope.java "java_pods" { + targets = discovery.relabel.java_pods.output + profiling_config { + interval = "60s" + alloc = "512k" + cpu = true + sample_rate = 100 + lock = "10ms" + } + forward_to = [pyroscope.write.profiles_service.receiver] + } + // Profiles: pprof + discovery.kubernetes "pprof_pods" { + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + role = "pod" + } + + discovery.relabel "pprof_pods" { + targets = concat(discovery.kubernetes.pprof_pods.targets) + + rule { + action = "drop" + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "Pending|Succeeded|Failed|Completed" + } + + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_label_(.+)" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + } + + discovery.relabel "pprof_pods_memory_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + discovery.relabel "pprof_pods_memory_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + pyroscope.scrape "pyroscope_scrape_memory" { + targets = concat(discovery.relabel.pprof_pods_memory_default_name.output, discovery.relabel.pprof_pods_memory_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = true + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] + } + discovery.relabel "pprof_pods_cpu_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + discovery.relabel "pprof_pods_cpu_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + pyroscope.scrape "pyroscope_scrape_cpu" { + targets = concat(discovery.relabel.pprof_pods_cpu_default_name.output, discovery.relabel.pprof_pods_cpu_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = true + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] + } + discovery.relabel "pprof_pods_goroutine_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + discovery.relabel "pprof_pods_goroutine_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + pyroscope.scrape "pyroscope_scrape_goroutine" { + targets = concat(discovery.relabel.pprof_pods_goroutine_default_name.output, discovery.relabel.pprof_pods_goroutine_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = true + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] + } + discovery.relabel "pprof_pods_block_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + discovery.relabel "pprof_pods_block_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + pyroscope.scrape "pyroscope_scrape_block" { + targets = concat(discovery.relabel.pprof_pods_block_default_name.output, discovery.relabel.pprof_pods_block_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = true + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] + } + discovery.relabel "pprof_pods_mutex_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + discovery.relabel "pprof_pods_mutex_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + pyroscope.scrape "pyroscope_scrape_mutex" { + targets = concat(discovery.relabel.pprof_pods_mutex_default_name.output, discovery.relabel.pprof_pods_mutex_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = true + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] + } + discovery.relabel "pprof_pods_fgprof_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + discovery.relabel "pprof_pods_fgprof_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } + } + + pyroscope.scrape "pyroscope_scrape_fgprof" { + targets = concat(discovery.relabel.pprof_pods_fgprof_default_name.output, discovery.relabel.pprof_pods_fgprof_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = true + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] + } + // Pyroscope + remote.kubernetes.secret "profiles_service" { + name = "pyroscope-k8s-monitoring" + namespace = "default" + } + pyroscope.write "profiles_service" { + endpoint { + url = nonsensitive(remote.kubernetes.secret.profiles_service.data["host"]) + headers = { + "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.profiles_service.data["tenantId"]), + } + + basic_auth { + username = nonsensitive(remote.kubernetes.secret.profiles_service.data["username"]) + password = remote.kubernetes.secret.profiles_service.data["password"] + } + } + external_labels = { + region = "southwest", + tenant = "widgetco", + env = remote.kubernetes.configmap.cluster_info.data["env"], + region = remote.kubernetes.configmap.cluster_info.data["region"], + cluster = "custom-config-test", + } + } + + logging { + level = "info" + format = "logfmt" + } + remote.kubernetes.configmap "cluster_info" { name = "cluster_info" namespace = "default" @@ -68989,6 +71129,16 @@ spec: exit 1 fi echo "Grafana Alloy for Logs config file is valid" + echo Validating Grafana Alloy for Profiles config file + if ! alloy fmt /etc/alloy/profiles.alloy > /dev/null; then + exit 1 + fi + output=$(alloy run --stability.level public-preview "/etc/alloy/profiles.alloy" 2>&1) + if ! echo "${output}" | grep "KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be defined" >/dev/null; then + echo "${output}" + exit 1 + fi + echo "Grafana Alloy for Profiles config file is valid" env: - name: KUBERNETES_SERVICE_HOST # Intentionally disable its connection to Kubernetes to make it fail in a known way value: "" diff --git a/charts/k8s-monitoring-v1/docs/examples/custom-config/profiles.alloy b/charts/k8s-monitoring-v1/docs/examples/custom-config/profiles.alloy index e69de29bb..ed28ca40c 100644 --- a/charts/k8s-monitoring-v1/docs/examples/custom-config/profiles.alloy +++ b/charts/k8s-monitoring-v1/docs/examples/custom-config/profiles.alloy @@ -0,0 +1,932 @@ +// Profiles: eBPF +discovery.kubernetes "ebpf_pods" { + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + role = "pod" +} + +discovery.relabel "ebpf_pods" { + targets = discovery.kubernetes.ebpf_pods.targets + rule { + action = "drop" + regex = "Succeeded|Failed|Completed" + source_labels = ["__meta_kubernetes_pod_phase"] + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + // provide arbitrary service_name label, otherwise it will be set to {__meta_kubernetes_namespace}/{__meta_kubernetes_pod_container_name} + rule { + action = "replace" + regex = "(.*)@(.*)" + replacement = "ebpf/${1}/${2}" + separator = "@" + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + target_label = "service_name" + } +} + +pyroscope.ebpf "ebpf_pods" { + targets = discovery.relabel.ebpf_pods.output + + demangle = "none" + + forward_to = [pyroscope.write.profiles_service.receiver] +} +// Profiles: Java +discovery.kubernetes "java_pods" { + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + role = "pod" +} + +discovery.process "java_pods" { + join = discovery.kubernetes.java_pods.targets +} + +discovery.relabel "java_pods" { + targets = discovery.process.java_pods.targets + rule { + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "Succeeded|Failed|Completed" + action = "drop" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + regex = "^$" + action = "drop" + } + rule { + source_labels = ["__meta_process_exe"] + action = "keep" + regex = ".*/java$" + } + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + action = "replace" + target_label = "node" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } +} + +pyroscope.java "java_pods" { + targets = discovery.relabel.java_pods.output + profiling_config { + interval = "60s" + alloc = "512k" + cpu = true + sample_rate = 100 + lock = "10ms" + } + forward_to = [pyroscope.write.profiles_service.receiver] +} +// Profiles: pprof +discovery.kubernetes "pprof_pods" { + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } + role = "pod" +} + +discovery.relabel "pprof_pods" { + targets = concat(discovery.kubernetes.pprof_pods.targets) + + rule { + action = "drop" + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "Pending|Succeeded|Failed|Completed" + } + + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_label_(.+)" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } +} + +discovery.relabel "pprof_pods_memory_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +discovery.relabel "pprof_pods_memory_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +pyroscope.scrape "pyroscope_scrape_memory" { + targets = concat(discovery.relabel.pprof_pods_memory_default_name.output, discovery.relabel.pprof_pods_memory_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = true + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] +} +discovery.relabel "pprof_pods_cpu_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +discovery.relabel "pprof_pods_cpu_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +pyroscope.scrape "pyroscope_scrape_cpu" { + targets = concat(discovery.relabel.pprof_pods_cpu_default_name.output, discovery.relabel.pprof_pods_cpu_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = true + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] +} +discovery.relabel "pprof_pods_goroutine_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +discovery.relabel "pprof_pods_goroutine_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +pyroscope.scrape "pyroscope_scrape_goroutine" { + targets = concat(discovery.relabel.pprof_pods_goroutine_default_name.output, discovery.relabel.pprof_pods_goroutine_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = true + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] +} +discovery.relabel "pprof_pods_block_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +discovery.relabel "pprof_pods_block_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_block_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +pyroscope.scrape "pyroscope_scrape_block" { + targets = concat(discovery.relabel.pprof_pods_block_default_name.output, discovery.relabel.pprof_pods_block_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = true + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] +} +discovery.relabel "pprof_pods_mutex_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +discovery.relabel "pprof_pods_mutex_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +pyroscope.scrape "pyroscope_scrape_mutex" { + targets = concat(discovery.relabel.pprof_pods_mutex_default_name.output, discovery.relabel.pprof_pods_mutex_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = true + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = false + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] +} +discovery.relabel "pprof_pods_fgprof_default_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port_name"] + action = "keep" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +discovery.relabel "pprof_pods_fgprof_custom_name" { + targets = concat(discovery.relabel.pprof_pods.output) + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_scrape"] + action = "keep" + regex = "true" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port_name"] + action = "drop" + regex = "" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port_name" + action = "keepequal" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_scheme"] + action = "replace" + regex = "(https?)" + target_label = "__scheme__" + replacement = "$1" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_path"] + action = "replace" + regex = "(.+)" + target_label = "__profile_path__" + replacement = "$1" + } + + rule { + source_labels = ["__address__", "__meta_kubernetes_pod_annotation_profiles_grafana_com_fgprof_port"] + action = "replace" + regex = "(.+?)(?::\\d+)?;(\\d+)" + target_label = "__address__" + replacement = "$1:$2" + } +} + +pyroscope.scrape "pyroscope_scrape_fgprof" { + targets = concat(discovery.relabel.pprof_pods_fgprof_default_name.output, discovery.relabel.pprof_pods_fgprof_custom_name.output) + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + profiling_config { + profile.memory { + enabled = false + } + profile.block { + enabled = false + } + profile.goroutine { + enabled = false + } + profile.mutex { + enabled = false + } + profile.process_cpu { + enabled = false + } + profile.fgprof { + enabled = true + } + profile.godeltaprof_memory { + enabled = false + } + profile.godeltaprof_mutex { + enabled = false + } + profile.godeltaprof_block { + enabled = false + } + } + + forward_to = [pyroscope.write.profiles_service.receiver] +} +// Pyroscope +remote.kubernetes.secret "profiles_service" { + name = "pyroscope-k8s-monitoring" + namespace = "default" +} +pyroscope.write "profiles_service" { + endpoint { + url = nonsensitive(remote.kubernetes.secret.profiles_service.data["host"]) + headers = { + "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.profiles_service.data["tenantId"]), + } + + basic_auth { + username = nonsensitive(remote.kubernetes.secret.profiles_service.data["username"]) + password = remote.kubernetes.secret.profiles_service.data["password"] + } + } + external_labels = { + region = "southwest", + tenant = "widgetco", + env = remote.kubernetes.configmap.cluster_info.data["env"], + region = remote.kubernetes.configmap.cluster_info.data["region"], + cluster = "custom-config-test", + } +} + +logging { + level = "info" + format = "logfmt" +} + +remote.kubernetes.configmap "cluster_info" { + name = "cluster_info" + namespace = "default" +} diff --git a/charts/k8s-monitoring-v1/docs/examples/custom-config/values.yaml b/charts/k8s-monitoring-v1/docs/examples/custom-config/values.yaml index d9ee0404d..1ab5d0389 100644 --- a/charts/k8s-monitoring-v1/docs/examples/custom-config/values.yaml +++ b/charts/k8s-monitoring-v1/docs/examples/custom-config/values.yaml @@ -25,6 +25,17 @@ externalServices: externalLabelsFrom: env: remote.kubernetes.configmap.cluster_info.data["env"] region: remote.kubernetes.configmap.cluster_info.data["region"] + pyroscope: + host: https://pyroscope.example.com + basicAuth: + username: 12345 + password: "It's a secret to everyone" + externalLabels: + region: southwest + tenant: widgetco + externalLabelsFrom: + env: remote.kubernetes.configmap.cluster_info.data["env"] + region: remote.kubernetes.configmap.cluster_info.data["region"] extraConfig: |- discovery.relabel "animal_service" { @@ -101,3 +112,11 @@ logs: name = "cluster_info" namespace = {{ .Release.Namespace | quote }} } + +profiles: + enabled: true + extraConfig: |- + remote.kubernetes.configmap "cluster_info" { + name = "cluster_info" + namespace = {{ .Release.Namespace | quote }} + } diff --git a/charts/k8s-monitoring-v1/templates/_configs.tpl b/charts/k8s-monitoring-v1/templates/_configs.tpl index 0a29d845b..2a96eb71e 100644 --- a/charts/k8s-monitoring-v1/templates/_configs.tpl +++ b/charts/k8s-monitoring-v1/templates/_configs.tpl @@ -160,4 +160,7 @@ {{- include "alloy.config.profilesService" . }} {{- include "alloy.config.logging" (index .Values "alloy-profiles").logging }} {{- include "alloy.config.liveDebugging" (index .Values "alloy-profiles").liveDebugging}} + {{- if .Values.profiles.extraConfig }} + {{- tpl .Values.profiles.extraConfig $ | indent 0 }} + {{- end }} {{- end -}} diff --git a/charts/k8s-monitoring-v1/values.schema.json b/charts/k8s-monitoring-v1/values.schema.json index c93b7739e..764d65f68 100644 --- a/charts/k8s-monitoring-v1/values.schema.json +++ b/charts/k8s-monitoring-v1/values.schema.json @@ -2209,6 +2209,9 @@ "enabled": { "type": "boolean" }, + "extraConfig": { + "type": "string" + }, "java": { "type": "object", "properties": { diff --git a/charts/k8s-monitoring-v1/values.yaml b/charts/k8s-monitoring-v1/values.yaml index fbb6ab8ad..e4c886fc8 100644 --- a/charts/k8s-monitoring-v1/values.yaml +++ b/charts/k8s-monitoring-v1/values.yaml @@ -1911,6 +1911,13 @@ profiles: - mutex - fgprof + # -- Extra configuration that will be added to the Grafana Alloy for Logs configuration file. + # This value is templated so that you can refer to other values from this file. + # This cannot be used to modify the generated configuration values, only append new components. + # See [Adding custom Flow configuration](#adding-custom-flow-configuration) for an example. + # @section -- Profiles Global + extraConfig: "" + # Telemetry data receiver settings receivers: grpc: