deploy: add options for prometheus monitoring

config/extras/monitoring/alerts.yaml

@@ -0,0 +1,95 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: piraeus-datastore
+spec:
+  groups:
+    - name: linstor.rules
+      rules:
+        - alert: linstorSatelliteErrorRate
+          annotations:
+            description: |
+              LINSTOR Satellite "{{ $labels.hostname }}" reports {{ $value }} errors in the last 15 minutes.
+              Use "linstor error-reports list --nodes {{ $labels.hostname }} --since 15minutes" to see them.
+          expr: increase(linstor_error_reports_count{module="SATELLITE"}[15m]) > 0
+          labels:
+            severity: warning
+        - alert: linstorControllerErrorRate
+          annotations:
+            description: |
+              LINSTOR Controller reports {{ $value }} errors in the last 15 minutes.
+              Use "linstor error-reports list --since 15minutes" to see them.
+          expr: increase(linstor_error_reports_count{module="CONTROLLER"}[15m]) > 0
+          labels:
+            severity: warning
+        - alert: linstorSatelliteNotOnline
+          annotations:
+            description: |
+              LINSTOR Satellite "{{ $labels.hostname }}" is not ONLINE.
+              Check that the Satellite is running and reachable from the LINSTOR Controller.
+          expr: linstor_node_state{nodetype="SATELLITE"} != 2
+          labels:
+            severity: critical
+        - alert: linstorStoragePoolErrors
+          annotations:
+            description: |
+              Storage pool "{{ $labels.storage_pool }}" on node "{{ $labels.node }}" ({{ $labels.driver }}={{ $labels.backing_pool }}) is reporting errors.
+          expr: linstor_storage_pool_error_count > 0
+          labels:
+            severity: critical
+        - alert: linstorStoragePoolAtCapacity
+          annotations:
+            description: |
+              Storage pool "{{ $labels.storage_pool }}" on node "{{ $labels.node }}" ({{ $labels.driver }}={{ $labels.backing_pool }}) has less than 20% free space available.
+          expr: ( linstor_storage_pool_capacity_free_bytes / linstor_storage_pool_capacity_total_bytes ) < 0.20
+          labels:
+            severity: warn
+    - name: drbd.rules
+      rules:
+        - alert: drbdConnectionNotConnected
+          annotations:
+            description: |
+              DRBD Resource "{{ $labels.name }}" on "{{ $labels.pod }}" is not connected to "{{ $labels.conn_name }}": {{ $labels.drbd_connection_state }}.
+          expr: drbd_connection_state{drbd_connection_state!="Connected"} > 0
+          labels:
+            severity: warn
+        - alert: drbdDeviceNotUpToDate
+          annotations:
+            description: |
+              DRBD device "{{ $labels.name }}" on "{{ $labels.pod }}" has unexpected device state "{{ $labels.drbd_device_state }}".
+          expr: drbd_device_state{drbd_device_state!~"UpToDate|Diskless"} > 0
+          labels:
+            severity: warn
+        - alert: drbdDeviceUnintentionalDiskless
+          annotations:
+            description: |
+              DRBD device "{{ $labels.name }}" on "{{ $labels.pod }}" is unintenionally diskless.
+              This usually indicates IO errors reported on the backing device. Check the kernel log.
+          expr: drbd_device_unintentionaldiskless > 0
+          labels:
+            severity: warn
+        - alert: drbdDeviceWithoutQuorum
+          annotations:
+            description: |
+              DRBD device "{{ $labels.name }}" on "{{ $labels.pod }}" has no quorum.
+              This usually indicates connectivity issues.
+          expr: drbd_device_quorum == 0
+          labels:
+            severity: warn
+        - alert: drbdResourceSuspended
+          annotations:
+            description: |
+              DRBD resource "{{ $labels.name }}" on "{{ $labels.pod }}" has been suspended for 1m.
+          for: 1m
+          expr: drbd_resource_suspended > 0
+          labels:
+            severity: warn
+        - alert: drbdResourceResyncWithoutProgress
+          annotations:
+            description: |
+              DRBD resource "{{ $labels.name }}" on "{{ $labels.pod }}" has been in Inconsistent without resync progress for 5 minutes.
+              This may indicate there is no connection to UpToDate data, or a stuck resync.
+          expr: drbd_device_state{drbd_device_state="Inconsistent"} and delta(drbd_peerdevice_outofsync_bytes[5m]) >= 0
+          labels:
+            severity: warn

config/extras/monitoring/kustomization.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - linstor-controller-monitor.yaml
+  - linstor-satellite-monitor.yaml
+  - alerts.yaml
+
+generatorOptions:
+  disableNameSuffixHash: true
+configMapGenerator:
+  - name: piraeus-datastore-dashboard
+    namespace: prometheus
+    options:
+      labels:
+        grafana_dashboard: "1"
+    files:
+      - piraeus-dashboard.json

config/prometheus/controllermonitor.yaml → config/extras/monitoring/linstor-controller-monitor.yaml

@@ -3,7 +3,6 @@ apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   name: linstor-controller
-  namespace: system
   labels:
     app.kubernetes.io/component: linstor-controller
 spec:

config/prometheus/satellitemonitor.yaml → config/extras/monitoring/linstor-satellite-monitor.yaml

@@ -3,7 +3,6 @@ apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
   name: linstor-satellite
-  namespace: system
   labels:
     app.kubernetes.io/component: linstor-satellite
 spec: