feat(qrm): add message tag in qrm plugins alloc_failed,get_topology_h…

…ints_failed and remove_pod_failed metric
kubewharf · Jan 17, 2025 · 52ccd74 · 52ccd74
1 parent 0c24ce6
commit 52ccd74
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 9 deletions.
diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go
@@ -19,6 +19,7 @@ package dynamicpolicy
 import (
 	"context"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -694,7 +695,10 @@ func (p *DynamicPolicy) GetTopologyHints(ctx context.Context,
 	defer func() {
 		p.RUnlock()
 		if err != nil {
-			_ = p.emitter.StoreInt64(util.MetricNameGetTopologyHintsFailed, 1, metrics.MetricTypeNameRaw)
+			errorMessage := strings.ReplaceAll(fmt.Sprintf("%v", err), " ", "_")
+			_ = p.emitter.StoreInt64(util.MetricNameGetTopologyHintsFailed, 1, metrics.MetricTypeNameRaw,
+				metrics.MetricTag{Key: "error_message", Val: errorMessage})
+
 			general.ErrorS(err, "GetTopologyHints failed",
 				"podNamespace", req.PodNamespace,
 				"podName", req.PodName,
@@ -840,7 +844,9 @@ func (p *DynamicPolicy) Allocate(ctx context.Context,
 			}
 		} else if respErr != nil {
 			_ = p.removeContainer(req.PodUid, req.ContainerName)
-			_ = p.emitter.StoreInt64(util.MetricNameAllocateFailed, 1, metrics.MetricTypeNameRaw)
+			errorMessage := strings.ReplaceAll(fmt.Sprintf("%v", respErr), " ", "_")
+			_ = p.emitter.StoreInt64(util.MetricNameAllocateFailed, 1, metrics.MetricTypeNameRaw,
+				metrics.MetricTag{Key: "error_message", Val: errorMessage})
 		}
 
 		p.Unlock()
@@ -933,7 +939,9 @@ func (p *DynamicPolicy) RemovePod(ctx context.Context,
 	defer func() {
 		p.Unlock()
 		if err != nil {
-			_ = p.emitter.StoreInt64(util.MetricNameRemovePodFailed, 1, metrics.MetricTypeNameRaw)
+			errorMessage := strings.ReplaceAll(fmt.Sprintf("%v", err), " ", "_")
+			_ = p.emitter.StoreInt64(util.MetricNameRemovePodFailed, 1, metrics.MetricTypeNameRaw,
+				metrics.MetricTag{Key: "error_message", Val: errorMessage})
 			general.ErrorS(err, "RemovePod failed", "podUID", req.PodUid)
 		}
 		general.InfoS("finished", "duration", time.Since(startTime).String(), "podUID", req.PodUid)

diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -584,7 +585,9 @@ func (p *DynamicPolicy) GetTopologyHints(ctx context.Context,
 	defer func() {
 		p.RUnlock()
 		if err != nil {
-			_ = p.emitter.StoreInt64(util.MetricNameGetTopologyHintsFailed, 1, metrics.MetricTypeNameRaw)
+			errorMessage := strings.ReplaceAll(fmt.Sprintf("%v", err), " ", "_")
+			_ = p.emitter.StoreInt64(util.MetricNameGetTopologyHintsFailed, 1, metrics.MetricTypeNameRaw,
+				metrics.MetricTag{Key: "error_message", Val: errorMessage})
 			general.ErrorS(err, "GetTopologyHints failed",
 				"podNamespace", req.PodNamespace,
 				"podName", req.PodName,
@@ -626,7 +629,9 @@ func (p *DynamicPolicy) RemovePod(ctx context.Context,
 	defer func() {
 		p.Unlock()
 		if err != nil {
-			_ = p.emitter.StoreInt64(util.MetricNameRemovePodFailed, 1, metrics.MetricTypeNameRaw)
+			errorMessage := strings.ReplaceAll(fmt.Sprintf("%v", err), " ", "_")
+			_ = p.emitter.StoreInt64(util.MetricNameRemovePodFailed, 1, metrics.MetricTypeNameRaw,
+				metrics.MetricTag{Key: "error_message", Val: errorMessage})
 			general.ErrorS(err, "RemovePod failed", "podUID", req.PodUid)
 		}
 		general.InfoS("finished", "duration", time.Since(startTime), "podUID", req.PodUid)
@@ -652,7 +657,9 @@ func (p *DynamicPolicy) RemovePod(ctx context.Context,
 	err = p.removePod(req.PodUid)
 	if err != nil {
 		general.ErrorS(err, "remove pod failed with error", "podUID", req.PodUid)
-		_ = p.emitter.StoreInt64(util.MetricNameRemovePodFailed, 1, metrics.MetricTypeNameRaw)
+		errorMessage := strings.ReplaceAll(fmt.Sprintf("%v", err), " ", "_")
+		_ = p.emitter.StoreInt64(util.MetricNameRemovePodFailed, 1, metrics.MetricTypeNameRaw,
+			metrics.MetricTag{Key: "error_message", Val: errorMessage})
 		return nil, err
 	}
 
@@ -944,7 +951,9 @@ func (p *DynamicPolicy) Allocate(ctx context.Context,
 			}
 		} else if respErr != nil {
 			_ = p.removeContainer(req.PodUid, req.ContainerName)
-			_ = p.emitter.StoreInt64(util.MetricNameAllocateFailed, 1, metrics.MetricTypeNameRaw)
+			errorMessage := strings.ReplaceAll(fmt.Sprintf("%v", respErr), " ", "_")
+			_ = p.emitter.StoreInt64(util.MetricNameAllocateFailed, 1, metrics.MetricTypeNameRaw,
+				metrics.MetricTag{Key: "error_message", Val: errorMessage})
 		}
 
 		p.Unlock()

diff --git a/pkg/agent/qrm-plugins/network/staticpolicy/policy.go b/pkg/agent/qrm-plugins/network/staticpolicy/policy.go
@@ -290,7 +290,9 @@ func (p *StaticPolicy) GetTopologyHints(_ context.Context,
 	defer func() {
 		p.Unlock()
 		if err != nil {
-			_ = p.emitter.StoreInt64(util.MetricNameGetTopologyHintsFailed, 1, metrics.MetricTypeNameRaw)
+			errorMessage := strings.ReplaceAll(fmt.Sprintf("%v", err), " ", "_")
+			_ = p.emitter.StoreInt64(util.MetricNameGetTopologyHintsFailed, 1, metrics.MetricTypeNameRaw,
+				metrics.MetricTag{Key: "error_message", Val: errorMessage})
 		}
 	}()
 
@@ -531,7 +533,9 @@ func (p *StaticPolicy) Allocate(_ context.Context,
 	defer func() {
 		p.Unlock()
 		if err != nil {
-			_ = p.emitter.StoreInt64(util.MetricNameAllocateFailed, 1, metrics.MetricTypeNameRaw)
+			errorMessage := strings.ReplaceAll(fmt.Sprintf("%v", err), " ", "_")
+			_ = p.emitter.StoreInt64(util.MetricNameAllocateFailed, 1, metrics.MetricTypeNameRaw,
+				metrics.MetricTag{Key: "error_message", Val: errorMessage})
 		}
 	}()