[DEC-2216] Add no data alert to fast sync snapshots monitor. (#102)

dydxprotocol · May 24, 2024 · 0f2d2a0 · 0f2d2a0
1 parent 02a6ede
commit 0f2d2a0
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 6 deletions.
diff --git a/modules/indexer_monitors/locals.tf b/modules/indexer_monitors/locals.tf
@@ -1,8 +1,10 @@
 locals {
-  monitor_suffix_literal          = "{{#is_alert}}\\n${var.pagerduty_tag}\\n{{/is_alert}}\\n\\n{{#is_recovery}}\\n${var.pagerduty_tag}\\n{{/is_recovery}}\\n\\n${var.slack_channel}"
-  monitor_suffix                  = "{{#is_alert}}\n${var.pagerduty_tag}\n{{/is_alert}}\n\n{{#is_recovery}}\n${var.pagerduty_tag}\n{{/is_recovery}}\n\n${var.slack_channel}"
-  critical_monitor_suffix_literal = "{{#is_alert}}\\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\\n{{/is_alert}}\\n\\n{{#is_recovery}}\\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\\n{{/is_recovery}}\\n\\n${var.slack_channel}"
-  critical_monitor_suffix         = "{{#is_alert}}\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\n{{/is_alert}}\n\n{{#is_recovery}}\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\n{{/is_recovery}}\n\n${var.slack_channel}"
+  monitor_suffix_literal          = "{{#is_alert}}\\n${var.pagerduty_tag}\\n{{/is_alert}}\\n\\n{{#is_recovery}}\\n${var.pagerduty_tag}\\n{{/is_recovery}}\\n\\n${var.slack_channel}\\n\\n"
+  monitor_suffix                  = "{{#is_alert}}\n${var.pagerduty_tag}\n{{/is_alert}}\n\n{{#is_recovery}}\n${var.pagerduty_tag}\n{{/is_recovery}}\n\n${var.slack_channel}\n\n"
+  critical_monitor_suffix_literal = "{{#is_alert}}\\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\\n{{/is_alert}}\\n\\n{{#is_recovery}}\\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\\n{{/is_recovery}}\\n\\n${var.slack_channel}\\n\\n"
+  critical_monitor_suffix         = "{{#is_alert}}\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\n{{/is_alert}}\n\n{{#is_recovery}}\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\n{{/is_recovery}}\n\n${var.slack_channel}\n\n"
+  monitor_no_data_suffix_literal  = "{{#is_no_data}}\\n${var.pagerduty_tag}\\n{{/is_no_data}}\\n\\n{{#is_no_data_recovery}}\\n${var.pagerduty_tag}\\n{{/is_no_data_recovery}}\\n\\n"
+  monitor_no_data_suffix          = "{{#is_no_data}}\n${var.pagerduty_tag}\n{{/is_no_data}}\n\n{{#is_no_data_recovery}}\n${var.pagerduty_tag}\n{{/is_no_data_recovery}}\n\n"
   wss_url                         = "wss://${var.url}/v4/ws"
   https_url                       = "https://${var.url}/v4"
   tick_frequency                  = 300  # 5 minutes

diff --git a/modules/indexer_monitors/monitors.tf b/modules/indexer_monitors/monitors.tf
@@ -291,7 +291,7 @@ resource "datadog_monitor_json" "fast_sync_snapshots" {
     "name": "[${var.environment}] Indexer fast sync snapshots haven't been uploaded in the last day",
     "type": "query alert",
     "query": "sum(last_1d):sum:aws.s3.put_requests{bucketname:${local.snapshot_bucket_prefix}-full-node-snapshots}.as_count() < 1",
-    "message": "Indexer fast sync snapshots haven't been uploaded in the last day. Please investigate the snapshotting full node.\n\n${local.monitor_suffix_literal}",
+    "message": "Indexer fast sync snapshots haven't been uploaded in the last day. Please investigate the snapshotting full node.\n\n${local.monitor_suffix_literal}${local.monitor_no_data_suffix_literal}",
     "tags": [
         "team:${var.team}",
         "env:${var.env_tag}"
@@ -303,7 +303,8 @@ resource "datadog_monitor_json" "fast_sync_snapshots" {
         "notify_audit": false,
         "require_full_window": false,
         "notify_no_data": true,
-        "renotify_interval": 0,
+        "renotify_interval": 720,
+        "renotify_no_data": true,
         "include_tags": false,
         "evaluation_delay": 900,
         "no_data_timeframe": 1440,