Merge pull request kubevirt#10982 from machadovilaca/refactor-monitor…

…ing-metrics

Refactor monitoring metrics

docs/metrics.md

@@ -21,12 +21,21 @@ The total number of requests to deprecated KubeVirt APIs. Type: Counter.
 ### kubevirt_configuration_emulation_enabled
 Indicates whether the Software Emulation is enabled in the configuration. Type: Gauge.
 
+### kubevirt_console_active_connections
+Amount of active Console connections, broken down by namespace and vmi name. Type: Gauge.
+
 ### kubevirt_nodes_with_kvm
 The number of nodes in the cluster that have the devices.kubevirt.io/kvm resource available. Type: Gauge.
 
 ### kubevirt_number_of_vms
 The number of VMs in the cluster by namespace. Type: Gauge.
 
+### kubevirt_portforward_active_tunnels
+Amount of active portforward tunnels, broken down by namespace and vmi name. Type: Gauge.
+
+### kubevirt_usbredir_active_connections
+Amount of active USB redirection connections, broken down by namespace and vmi name. Type: Gauge.
+
 ### kubevirt_virt_api_up
 The number of virt-api pods that are up. Type: Gauge.
 
@@ -255,6 +264,9 @@ Returns the amount of space in bytes restored from the source virtual machine. T
 ### kubevirt_vmsnapshot_persistentvolumeclaim_labels
 Returns the labels of the persistent volume claims that are used for restoring virtual machines. Type: Gauge.
 
+### kubevirt_vnc_active_connections
+Amount of active VNC connections, broken down by namespace and vmi name. Type: Gauge.
+
 ## Developing new metrics
 After developing new metrics or changing old ones, please run `make generate` to regenerate this document.
 

pkg/monitoring/metrics/virt-api/BUILD.bazel

@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 go_library(
     name = "go_default_library",
     srcs = [
+        "connection_metrics.go",
         "metrics.go",
         "vm_metrics.go",
     ],

pkg/monitoring/api/prometheus.go → pkg/monitoring/metrics/virt-api/connection_metrics.go

@@ -1,48 +1,68 @@
-package api
+/*
+ * This file is part of the KubeVirt project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright the KubeVirt Authors.
+ */
 
-import (
-	"github.com/prometheus/client_golang/prometheus"
-)
+package virt_api
+
+import "github.com/machadovilaca/operator-observability/pkg/operatormetrics"
 
 var (
-	namespaceAndVMILabels    = []string{"namespace", "vmi"}
-	activePortForwardTunnels = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
+	connectionMetrics = []operatormetrics.Metric{
+		activePortForwardTunnels,
+		activeVNCConnections,
+		activeConsoleConnections,
+		activeUSBRedirConnections,
+	}
+
+	namespaceAndVMILabels = []string{"namespace", "vmi"}
+
+	activePortForwardTunnels = operatormetrics.NewGaugeVec(
+		operatormetrics.MetricOpts{
 			Name: "kubevirt_portforward_active_tunnels",
-			Help: "Amount of active portforward tunnels, broken down by namespace and vmi name",
+			Help: "Amount of active portforward tunnels, broken down by namespace and vmi name.",
 		},
 		namespaceAndVMILabels,
 	)
-	activeVNCConnections = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
+
+	activeVNCConnections = operatormetrics.NewGaugeVec(
+		operatormetrics.MetricOpts{
 			Name: "kubevirt_vnc_active_connections",
-			Help: "Amount of active VNC connections, broken down by namespace and vmi name",
+			Help: "Amount of active VNC connections, broken down by namespace and vmi name.",
 		},
 		namespaceAndVMILabels,
 	)
-	activeConsoleConnections = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
+
+	activeConsoleConnections = operatormetrics.NewGaugeVec(
+		operatormetrics.MetricOpts{
 			Name: "kubevirt_console_active_connections",
-			Help: "Amount of active Console connections, broken down by namespace and vmi name",
+			Help: "Amount of active Console connections, broken down by namespace and vmi name.",
 		},
 		namespaceAndVMILabels,
 	)
-	activeUSBRedirConnections = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
+
+	activeUSBRedirConnections = operatormetrics.NewGaugeVec(
+		operatormetrics.MetricOpts{
 			Name: "kubevirt_usbredir_active_connections",
-			Help: "Amount of active USB redirection connections, broken down by namespace and vmi name",
+			Help: "Amount of active USB redirection connections, broken down by namespace and vmi name.",
 		},
 		namespaceAndVMILabels,
 	)
 )
 
-func init() {
-	prometheus.MustRegister(activePortForwardTunnels)
-	prometheus.MustRegister(activeVNCConnections)
-	prometheus.MustRegister(activeConsoleConnections)
-	prometheus.MustRegister(activeUSBRedirConnections)
-}
-
 type Decrementer interface {
 	Dec()
 }

pkg/monitoring/metrics/virt-api/metrics.go

@@ -21,14 +21,11 @@ package virt_api
 
 import "github.com/machadovilaca/operator-observability/pkg/operatormetrics"
 
-var (
-	metrics = [][]operatormetrics.Metric{
-		vmMetrics,
-	}
-)
-
 func SetupMetrics() error {
-	return operatormetrics.RegisterMetrics(metrics...)
+	return operatormetrics.RegisterMetrics(
+		connectionMetrics,
+		vmMetrics,
+	)
 }
 
 func ListMetrics() []operatormetrics.Metric {

pkg/monitoring/metrics/virt-controller/BUILD.bazel

@@ -5,7 +5,9 @@ go_library(
     srcs = [
         "component_metrics.go",
         "metrics.go",
+        "migration_metrics.go",
         "migrationstats_collector.go",
+        "perfscale_metrics.go",
         "vmistats_collector.go",
         "vmstats_collector.go",
     ],
@@ -21,15 +23,18 @@ go_library(
         "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library",
         "//vendor/github.com/prometheus/client_model/go:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/client-go/tools/cache:go_default_library",
     ],
 )
 
 go_test(
     name = "go_default_test",
     srcs = [
-        "metrics_suite_test.go",
+        "migration_metrics_test.go",
         "migrationstats_collector_test.go",
+        "perfscale_metrics_test.go",
+        "virt_controller_suite_test.go",
         "vmistats_collector_test.go",
         "vmstats_collector_test.go",
     ],

pkg/monitoring/metrics/virt-controller/component_metrics.go

@@ -25,7 +25,7 @@ import (
 )
 
 var (
-	operatorMetrics = []operatormetrics.Metric{
+	componentMetrics = []operatormetrics.Metric{
 		virtControllerLeading,
 		virtControllerReady,
 	}

pkg/monitoring/metrics/virt-controller/metrics.go

@@ -13,22 +13,28 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *
- * Copyright 2023 Red Hat, Inc.
+ * Copyright the KubeVirt Authors.
  *
  */
 
 package virt_controller
 
 import (
+	"fmt"
+	"time"
+
 	"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/cache"
 
 	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
 )
 
 var (
 	metrics = [][]operatormetrics.Metric{
-		operatorMetrics,
+		componentMetrics,
+		migrationMetrics,
+		perfscaleMetrics,
 	}
 
 	vmInformer                  cache.SharedIndexInformer
@@ -78,3 +84,42 @@ func UpdateVMIMigrationInformer(informer cache.SharedIndexInformer) {
 func ListMetrics() []operatormetrics.Metric {
 	return operatormetrics.ListMetrics()
 }
+
+func phaseTransitionTimeBuckets() []float64 {
+	return []float64{
+		(0.5 * time.Second.Seconds()),
+		(1 * time.Second.Seconds()),
+		(2 * time.Second.Seconds()),
+		(5 * time.Second.Seconds()),
+		(10 * time.Second.Seconds()),
+		(20 * time.Second.Seconds()),
+		(30 * time.Second.Seconds()),
+		(40 * time.Second.Seconds()),
+		(50 * time.Second.Seconds()),
+		(60 * time.Second).Seconds(),
+		(90 * time.Second).Seconds(),
+		(2 * time.Minute).Seconds(),
+		(3 * time.Minute).Seconds(),
+		(5 * time.Minute).Seconds(),
+		(10 * time.Minute).Seconds(),
+		(20 * time.Minute).Seconds(),
+		(30 * time.Minute).Seconds(),
+		(1 * time.Hour).Seconds(),
+	}
+}
+
+func getTransitionTimeSeconds(oldTime *metav1.Time, newTime *metav1.Time) (float64, error) {
+	if newTime == nil || oldTime == nil {
+		// no phase transition timestamp found
+		return 0.0, fmt.Errorf("missing phase transition timestamp, newTime: %v, oldTime: %v", newTime, oldTime)
+	}
+
+	diffSeconds := newTime.Time.Sub(oldTime.Time).Seconds()
+
+	// when transitions are very fast, we can encounter time skew. Make 0 the floor
+	if diffSeconds < 0 {
+		diffSeconds = 0.0
+	}
+
+	return diffSeconds, nil
+}

pkg/monitoring/metrics/virt-controller/migration_metrics.go

@@ -0,0 +1,100 @@
+/*
+ * This file is part of the KubeVirt project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright the KubeVirt Authors.
+ */
+
+package virt_controller
+
+import (
+	"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/cache"
+
+	v1 "kubevirt.io/api/core/v1"
+	"kubevirt.io/client-go/log"
+)
+
+const (
+	migrationTransTimeErrFmt = "Error encountered during VMI migration transition time histogram calculation: %v"
+	migrationTransTimeFail   = "Failed to get a histogram for a VMI migration lifecycle transition times"
+)
+
+var (
+	migrationMetrics = []operatormetrics.Metric{
+		vmiMigrationPhaseTransitionTimeFromCreation,
+	}
+
+	vmiMigrationPhaseTransitionTimeFromCreation = operatormetrics.NewHistogramVec(
+		operatormetrics.MetricOpts{
+			Name: "kubevirt_vmi_migration_phase_transition_time_from_creation_seconds",
+			Help: "Histogram of VM migration phase transitions duration from creation time in seconds.",
+		},
+		operatormetrics.HistogramOpts{
+			Buckets: phaseTransitionTimeBuckets(),
+		},
+		[]string{
+			// phase of the vmi migration
+			"phase",
+		},
+	)
+)
+
+func CreateVMIMigrationHandler(informer cache.SharedIndexInformer) error {
+	_, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		UpdateFunc: func(oldVMIMigration, newVMIMigration interface{}) {
+			updateVMIMigrationPhaseTransitionTimeFromCreationTime(oldVMIMigration.(*v1.VirtualMachineInstanceMigration), newVMIMigration.(*v1.VirtualMachineInstanceMigration))
+		},
+	})
+
+	return err
+}
+
+func updateVMIMigrationPhaseTransitionTimeFromCreationTime(oldVMIMigration *v1.VirtualMachineInstanceMigration, newVMIMigration *v1.VirtualMachineInstanceMigration) {
+	if oldVMIMigration == nil || oldVMIMigration.Status.Phase == newVMIMigration.Status.Phase {
+		return
+	}
+
+	diffSeconds, err := getVMIMigrationTransitionTimeSeconds(newVMIMigration)
+	if err != nil {
+		log.Log.V(4).Infof(migrationTransTimeErrFmt, err)
+		return
+	}
+
+	labels := []string{string(newVMIMigration.Status.Phase)}
+	histogram, err := vmiMigrationPhaseTransitionTimeFromCreation.GetMetricWithLabelValues(labels...)
+	if err != nil {
+		log.Log.Reason(err).Error(migrationTransTimeFail)
+		return
+	}
+
+	histogram.Observe(diffSeconds)
+}
+
+func getVMIMigrationTransitionTimeSeconds(newVMIMigration *v1.VirtualMachineInstanceMigration) (float64, error) {
+	var oldTime *metav1.Time
+	var newTime *metav1.Time
+
+	oldTime = newVMIMigration.CreationTimestamp.DeepCopy()
+	for _, transitionTimestamp := range newVMIMigration.Status.PhaseTransitionTimestamps {
+		if transitionTimestamp.Phase == newVMIMigration.Status.Phase {
+			newTime = transitionTimestamp.PhaseTransitionTimestamp.DeepCopy()
+		} else if newTime != nil {
+			break
+		}
+	}
+
+	return getTransitionTimeSeconds(oldTime, newTime)
+}