diff --git a/internal/controller/mantlebackup_controller.go b/internal/controller/mantlebackup_controller.go index c1dde3b8..5fdb9bfd 100644 --- a/internal/controller/mantlebackup_controller.go +++ b/internal/controller/mantlebackup_controller.go @@ -11,9 +11,12 @@ import ( _ "embed" mantlev1 "github.com/cybozu-go/mantle/api/v1" + "github.com/cybozu-go/mantle/cmd/backup" "github.com/cybozu-go/mantle/internal/ceph" "github.com/cybozu-go/mantle/internal/controller/internal/objectstorage" + "github.com/cybozu-go/mantle/internal/controller/metrics" "github.com/cybozu-go/mantle/pkg/controller/proto" + "github.com/prometheus/client_golang/prometheus" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" aerrors "k8s.io/apimachinery/pkg/api/errors" @@ -2068,6 +2071,19 @@ func (r *MantleBackupReconciler) primaryCleanup( return ctrl.Result{}, fmt.Errorf("failed to update SyncedToRemote to True: %w", err) } + duration := time.Since(target.GetCreationTimestamp().Time).Seconds() + source := "" + if _, ok := target.GetLabels()[backup.MantleBackupConfigUID]; ok { + source = "mantle-backup-config" + } + metrics.BackupCreationDuration. + With(prometheus.Labels{ + "namespace": r.managedCephClusterID, + "pvc": target.Spec.PVC, + "source": source, + }). + Observe(duration) + return ctrl.Result{}, nil } diff --git a/internal/controller/metrics/metrics.go b/internal/controller/metrics/metrics.go new file mode 100644 index 00000000..d5dcf1b9 --- /dev/null +++ b/internal/controller/metrics/metrics.go @@ -0,0 +1,24 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + runtimemetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +const subsystem = "mantle" + +var ( + BackupCreationDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: subsystem, + Name: "backup_creation_duration_seconds", + Help: "Duration in seconds of backup creation.", + Buckets: []float64{100, 250, 500, 750, 1_000, 2_500, 5_000, 7_500, 10_000, 25_000, 50_000, 75_000, 100_000, 250_000}, + }, + []string{"namespace", "pvc", "source"}, + ) +) + +func init() { + runtimemetrics.Registry.MustRegister(BackupCreationDuration) +} diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go index da6f350c..cf39dee5 100644 --- a/test/e2e/multik8s/suite_test.go +++ b/test/e2e/multik8s/suite_test.go @@ -48,11 +48,11 @@ var _ = Describe("Mantle", func() { func waitControllerToBeReady() { It("wait for mantle-controller to be ready", func() { Eventually(func() error { - return checkDeploymentReady(primaryK8sCluster, "rook-ceph", "mantle-controller") + return checkDeploymentReady(primaryK8sCluster, cephClusterNamespace, "mantle-controller") }).Should(Succeed()) Eventually(func() error { - return checkDeploymentReady(primaryK8sCluster, "rook-ceph", "mantle-controller") + return checkDeploymentReady(primaryK8sCluster, cephClusterNamespace, "mantle-controller") }).Should(Succeed()) }) } @@ -481,9 +481,36 @@ func replicationTestSuite() { ensureCorrectRestoration(primaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0) ensureCorrectRestoration(secondaryK8sCluster, ctx, namespace, backupName0, restoreName0, writtenDataHash0) }) + + It("should get metrics from the controller pod in the primary cluster", func(ctx SpecContext) { + metrics := []string{ + `mantle_backup_creation_duration_seconds_count`, + `mantle_backup_creation_duration_seconds_sum`, + } + ensureMetricsAreExposed(metrics) + }) }) } +func getControllerPodName(ns string) (string, error) { + stdout, _, err := kubectl(primaryK8sCluster, nil, "get", "pod", "-n", ns, + "-l", "app.kubernetes.io/name=mantle", "-o", "jsonpath={.items[0].metadata.name}") + return string(stdout), err +} + +func ensureMetricsAreExposed(metrics []string) { + GinkgoHelper() + controllerPod, err := getControllerPodName(cephClusterNamespace) + Expect(err).NotTo(HaveOccurred()) + + stdout, _, err := kubectl(primaryK8sCluster, nil, "exec", "-n", cephClusterNamespace, controllerPod, "--", + "curl", "-s", "http://localhost:8080/metrics") + Expect(err).NotTo(HaveOccurred()) + for _, metric := range metrics { + Expect(strings.Contains(string(stdout), metric)).To(BeTrue()) + } +} + func changeToStandalone() { Describe("change to standalone", func() { var namespace, pvcName, backupName string