add backup creation time metrics

Signed-off-by: Shinya Hayashi <[email protected]>

internal/controller/mantlebackup_controller.go

@@ -11,9 +11,12 @@ import (
 	_ "embed"
 
 	mantlev1 "github.com/cybozu-go/mantle/api/v1"
+	"github.com/cybozu-go/mantle/cmd/backup"
 	"github.com/cybozu-go/mantle/internal/ceph"
 	"github.com/cybozu-go/mantle/internal/controller/internal/objectstorage"
+	"github.com/cybozu-go/mantle/internal/controller/metrics"
 	"github.com/cybozu-go/mantle/pkg/controller/proto"
+	"github.com/prometheus/client_golang/prometheus"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	aerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -2068,6 +2071,19 @@ func (r *MantleBackupReconciler) primaryCleanup(
 		return ctrl.Result{}, fmt.Errorf("failed to update SyncedToRemote to True: %w", err)
 	}
 
+	duration := time.Since(target.GetCreationTimestamp().Time).Seconds()
+	source := ""
+	if _, ok := target.GetLabels()[backup.MantleBackupConfigUID]; ok {
+		source = "mantle-backup-config"
+	}
+	metrics.BackupCreationDuration.
+		With(prometheus.Labels{
+			"namespace": r.managedCephClusterID,
+			"pvc":       target.Spec.PVC,
+			"source":    source,
+		}).
+		Observe(duration)
+
 	return ctrl.Result{}, nil
 }
 

internal/controller/metrics/metrics.go

@@ -0,0 +1,24 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	runtimemetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+const subsystem = "mantle"
+
+var (
+	BackupCreationDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Subsystem: subsystem,
+			Name:      "backup_creation_duration_seconds",
+			Help:      "Duration in seconds of backup creation.",
+			Buckets:   []float64{100, 250, 500, 750, 1_000, 2_500, 5_000, 7_500, 10_000, 25_000, 50_000, 75_000, 100_000, 250_000},
+		},
+		[]string{"namespace", "pvc", "source"},
+	)
+)
+
+func init() {
+	runtimemetrics.Registry.MustRegister(BackupCreationDuration)
+}

test/e2e/multik8s/suite_test.go

@@ -48,11 +48,11 @@ var _ = Describe("Mantle", func() {
 func waitControllerToBeReady() {
 	It("wait for mantle-controller to be ready", func() {
 		Eventually(func() error {
-			return checkDeploymentReady(primaryK8sCluster, "rook-ceph", "mantle-controller")
+			return checkDeploymentReady(primaryK8sCluster, cephClusterNamespace, "mantle-controller")
 		}).Should(Succeed())
 
 		Eventually(func() error {
-			return checkDeploymentReady(primaryK8sCluster, "rook-ceph", "mantle-controller")
+			return checkDeploymentReady(primaryK8sCluster, cephClusterNamespace, "mantle-controller")
 		}).Should(Succeed())
 	})
 }
@@ -481,9 +481,36 @@ func replicationTestSuite() {
 			ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
 			ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0)
 		})
+
+		It("should get metrics from the controller pod in the primary cluster", func(ctx SpecContext) {
+			metrics := []string{
+				`mantle_backup_creation_duration_seconds_count`,
+				`mantle_backup_creation_duration_seconds_sum`,
+			}
+			ensureMetricsAreExposed(metrics)
+		})
 	})
 }
 
+func getControllerPodName(ns string) (string, error) {
+	stdout, _, err := kubectl(primaryK8sCluster, nil, "get", "pod", "-n", ns,
+		"-l", "app.kubernetes.io/name=mantle", "-o", "jsonpath={.items[0].metadata.name}")
+	return string(stdout), err
+}
+
+func ensureMetricsAreExposed(metrics []string) {
+	GinkgoHelper()
+	controllerPod, err := getControllerPodName(cephClusterNamespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	stdout, _, err := kubectl(primaryK8sCluster, nil, "exec", "-n", cephClusterNamespace, controllerPod, "--",
+		"curl", "-s", "http://localhost:8080/metrics")
+	Expect(err).NotTo(HaveOccurred())
+	for _, metric := range metrics {
+		Expect(strings.Contains(string(stdout), metric)).To(BeTrue())
+	}
+}
+
 func changeToStandalone() {
 	Describe("change to standalone", func() {
 		var namespace, pvcName, backupName string