Support canceling emergency priority alerts in pushover

prometheus · Jan 2, 2025 · 425d8c5 · 425d8c5
1 parent 3b61ae8
commit 425d8c5
Show file tree

Hide file tree

Showing 2 changed files with 82 additions and 62 deletions.
diff --git a/notify/pushover/pushover.go b/notify/pushover/pushover.go
@@ -16,6 +16,7 @@ package pushover
 import (
 	"context"
 	"fmt"
+	"github.com/prometheus/common/model"
 	"log/slog"
 	"net/http"
 	"net/url"
@@ -42,12 +43,13 @@ const (
 
 // Notifier implements a Notifier for Pushover notifications.
 type Notifier struct {
-	conf    *config.PushoverConfig
-	tmpl    *template.Template
-	logger  *slog.Logger
-	client  *http.Client
-	retrier *notify.Retrier
-	apiURL  string // for tests.
+	conf           *config.PushoverConfig
+	tmpl           *template.Template
+	logger         *slog.Logger
+	client         *http.Client
+	retrier        *notify.Retrier
+	apiMessagesURL string // for tests.
+	apiReceiptsURL string // for tests.
 }
 
 // New returns a new Pushover notifier.
@@ -57,30 +59,29 @@ func New(c *config.PushoverConfig, t *template.Template, l *slog.Logger, httpOpt
 		return nil, err
 	}
 	return &Notifier{
-		conf:    c,
-		tmpl:    t,
-		logger:  l,
-		client:  client,
-		retrier: &notify.Retrier{},
-		apiURL:  "https://api.pushover.net/1/messages.json",
+		conf:           c,
+		tmpl:           t,
+		logger:         l,
+		client:         client,
+		retrier:        &notify.Retrier{},
+		apiMessagesURL: "https://api.pushover.net/1/messages.json",
+		apiReceiptsURL: "https://api.pushover.net/1/receipts/cancel_by_tag",
 	}, nil
 }
 
 // Notify implements the Notifier interface.
 func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
-	key, ok := notify.GroupKey(ctx)
-	if !ok {
-		return false, fmt.Errorf("group key missing")
+	var err error
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
 	}
 	data := notify.GetTemplateData(ctx, n.tmpl, as, n.logger)
 
 	// @tjhop: should this use `group` for the keyval like most other notify implementations?
 	n.logger.Debug("extracted group key", "incident", key)
 
-	var (
-		err     error
-		message string
-	)
+	var message string
 	tmpl := notify.TmplText(n.tmpl, data, &err)
 	tmplHTML := notify.TmplHTML(n.tmpl, data, &err)
 
@@ -111,53 +112,69 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 	parameters.Add("token", tmpl(token))
 	parameters.Add("user", tmpl(userKey))
 
-	title, truncated := notify.TruncateInRunes(tmpl(n.conf.Title), maxTitleLenRunes)
-	if truncated {
-		n.logger.Warn("Truncated title", "incident", key, "max_runes", maxTitleLenRunes)
-	}
-	parameters.Add("title", title)
+	priority := tmpl(n.conf.Priority)
 
-	if n.conf.HTML {
-		parameters.Add("html", "1")
-		message = tmplHTML(n.conf.Message)
+	var (
+		alerts    = types.Alerts(as...)
+		dedupeKey = fmt.Sprintf("dedupeKey=%s", key.Hash())
+		u         *url.URL
+	)
+	if priority == "2" && alerts.Status() == model.AlertResolved {
+		u, err = url.Parse(fmt.Sprintf("%s%s.json", n.apiReceiptsURL, dedupeKey))
+		if err != nil {
+			return false, err
+		}
 	} else {
-		message = tmpl(n.conf.Message)
-	}
+		title, truncated := notify.TruncateInRunes(tmpl(n.conf.Title), maxTitleLenRunes)
+		if truncated {
+			n.logger.Warn("Truncated title", "incident", key, "max_runes", maxTitleLenRunes)
+		}
+		parameters.Add("title", title)
 
-	message, truncated = notify.TruncateInRunes(message, maxMessageLenRunes)
-	if truncated {
-		n.logger.Warn("Truncated message", "incident", key, "max_runes", maxMessageLenRunes)
-	}
-	message = strings.TrimSpace(message)
-	if message == "" {
-		// Pushover rejects empty messages.
-		message = "(no details)"
-	}
-	parameters.Add("message", message)
+		if n.conf.HTML {
+			parameters.Add("html", "1")
+			message = tmplHTML(n.conf.Message)
+		} else {
+			message = tmpl(n.conf.Message)
+		}
 
-	supplementaryURL, truncated := notify.TruncateInRunes(tmpl(n.conf.URL), maxURLLenRunes)
-	if truncated {
-		n.logger.Warn("Truncated URL", "incident", key, "max_runes", maxURLLenRunes)
-	}
-	parameters.Add("url", supplementaryURL)
-	parameters.Add("url_title", tmpl(n.conf.URLTitle))
-
-	parameters.Add("priority", tmpl(n.conf.Priority))
-	parameters.Add("retry", fmt.Sprintf("%d", int64(time.Duration(n.conf.Retry).Seconds())))
-	parameters.Add("expire", fmt.Sprintf("%d", int64(time.Duration(n.conf.Expire).Seconds())))
-	parameters.Add("device", tmpl(n.conf.Device))
-	parameters.Add("sound", tmpl(n.conf.Sound))
-
-	newttl := int64(time.Duration(n.conf.TTL).Seconds())
-	if newttl > 0 {
-		parameters.Add("ttl", fmt.Sprintf("%d", newttl))
-	}
+		message, truncated = notify.TruncateInRunes(message, maxMessageLenRunes)
+		if truncated {
+			n.logger.Warn("Truncated message", "incident", key, "max_runes", maxMessageLenRunes)
+		}
+		message = strings.TrimSpace(message)
+		if message == "" {
+			// Pushover rejects empty messages.
+			message = "(no details)"
+		}
+		parameters.Add("message", message)
 
-	if err != nil {
-		return false, err
+		supplementaryURL, truncated := notify.TruncateInRunes(tmpl(n.conf.URL), maxURLLenRunes)
+		if truncated {
+			n.logger.Warn("Truncated URL", "incident", key, "max_runes", maxURLLenRunes)
+		}
+		parameters.Add("url", supplementaryURL)
+		parameters.Add("url_title", tmpl(n.conf.URLTitle))
+
+		parameters.Add("priority", priority)
+		parameters.Add("retry", fmt.Sprintf("%d", int64(time.Duration(n.conf.Retry).Seconds())))
+		parameters.Add("expire", fmt.Sprintf("%d", int64(time.Duration(n.conf.Expire).Seconds())))
+		parameters.Add("device", tmpl(n.conf.Device))
+		parameters.Add("sound", tmpl(n.conf.Sound))
+		if priority == "2" {
+			parameters.Add("tags", fmt.Sprintf("dedupeKey=%s", key.Hash()))
+		}
+		newttl := int64(time.Duration(n.conf.TTL).Seconds())
+		if newttl > 0 {
+			parameters.Add("ttl", fmt.Sprintf("%d", newttl))
+		}
+
+		u, err = url.Parse(n.apiMessagesURL)
+		if err != nil {
+			return false, err
+		}
 	}
 
-	u, err := url.Parse(n.apiURL)
 	if err != nil {
 		return false, err
 	}

diff --git a/notify/pushover/pushover_test.go b/notify/pushover/pushover_test.go
@@ -56,7 +56,8 @@ func TestPushoverRedactedURL(t *testing.T) {
 		promslog.NewNopLogger(),
 	)
 	require.NoError(t, err)
-	notifier.apiURL = u.String()
+	notifier.apiMessagesURL = u.String()
+	notifier.apiReceiptsURL = u.String()
 
 	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key, token)
 }
@@ -80,7 +81,8 @@ func TestPushoverReadingUserKeyFromFile(t *testing.T) {
 		test.CreateTmpl(t),
 		promslog.NewNopLogger(),
 	)
-	notifier.apiURL = apiURL.String()
+	notifier.apiMessagesURL = apiURL.String()
+	notifier.apiReceiptsURL = apiURL.String()
 	require.NoError(t, err)
 
 	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, userKey)
@@ -105,7 +107,8 @@ func TestPushoverReadingTokenFromFile(t *testing.T) {
 		test.CreateTmpl(t),
 		promslog.NewNopLogger(),
 	)
-	notifier.apiURL = apiURL.String()
+	notifier.apiMessagesURL = apiURL.String()
+	notifier.apiReceiptsURL = apiURL.String()
 	require.NoError(t, err)
 
 	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, token)