From 0f2d2a07860f42ee7f756cce82f47e5e88b5532e Mon Sep 17 00:00:00 2001
From: vincentwschau <99756290+vincentwschau@users.noreply.github.com>
Date: Fri, 24 May 2024 15:44:11 -0400
Subject: [PATCH] [DEC-2216] Add no data alert to fast sync snapshots monitor.
 (#102)

---
 modules/indexer_monitors/locals.tf   | 10 ++++++----
 modules/indexer_monitors/monitors.tf |  5 +++--
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/modules/indexer_monitors/locals.tf b/modules/indexer_monitors/locals.tf
index 78ec2e97..7eb22972 100644
--- a/modules/indexer_monitors/locals.tf
+++ b/modules/indexer_monitors/locals.tf
@@ -1,8 +1,10 @@
 locals {
-  monitor_suffix_literal          = "{{#is_alert}}\\n${var.pagerduty_tag}\\n{{/is_alert}}\\n\\n{{#is_recovery}}\\n${var.pagerduty_tag}\\n{{/is_recovery}}\\n\\n${var.slack_channel}"
-  monitor_suffix                  = "{{#is_alert}}\n${var.pagerduty_tag}\n{{/is_alert}}\n\n{{#is_recovery}}\n${var.pagerduty_tag}\n{{/is_recovery}}\n\n${var.slack_channel}"
-  critical_monitor_suffix_literal = "{{#is_alert}}\\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\\n{{/is_alert}}\\n\\n{{#is_recovery}}\\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\\n{{/is_recovery}}\\n\\n${var.slack_channel}"
-  critical_monitor_suffix         = "{{#is_alert}}\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\n{{/is_alert}}\n\n{{#is_recovery}}\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\n{{/is_recovery}}\n\n${var.slack_channel}"
+  monitor_suffix_literal          = "{{#is_alert}}\\n${var.pagerduty_tag}\\n{{/is_alert}}\\n\\n{{#is_recovery}}\\n${var.pagerduty_tag}\\n{{/is_recovery}}\\n\\n${var.slack_channel}\\n\\n"
+  monitor_suffix                  = "{{#is_alert}}\n${var.pagerduty_tag}\n{{/is_alert}}\n\n{{#is_recovery}}\n${var.pagerduty_tag}\n{{/is_recovery}}\n\n${var.slack_channel}\n\n"
+  critical_monitor_suffix_literal = "{{#is_alert}}\\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\\n{{/is_alert}}\\n\\n{{#is_recovery}}\\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\\n{{/is_recovery}}\\n\\n${var.slack_channel}\\n\\n"
+  critical_monitor_suffix         = "{{#is_alert}}\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\n{{/is_alert}}\n\n{{#is_recovery}}\n${var.pagerduty_tag} ${var.secondary_pagerduty_tag}\n{{/is_recovery}}\n\n${var.slack_channel}\n\n"
+  monitor_no_data_suffix_literal  = "{{#is_no_data}}\\n${var.pagerduty_tag}\\n{{/is_no_data}}\\n\\n{{#is_no_data_recovery}}\\n${var.pagerduty_tag}\\n{{/is_no_data_recovery}}\\n\\n"
+  monitor_no_data_suffix          = "{{#is_no_data}}\n${var.pagerduty_tag}\n{{/is_no_data}}\n\n{{#is_no_data_recovery}}\n${var.pagerduty_tag}\n{{/is_no_data_recovery}}\n\n"
   wss_url                         = "wss://${var.url}/v4/ws"
   https_url                       = "https://${var.url}/v4"
   tick_frequency                  = 300  # 5 minutes
diff --git a/modules/indexer_monitors/monitors.tf b/modules/indexer_monitors/monitors.tf
index 34a47b4e..9ad1f84d 100644
--- a/modules/indexer_monitors/monitors.tf
+++ b/modules/indexer_monitors/monitors.tf
@@ -291,7 +291,7 @@ resource "datadog_monitor_json" "fast_sync_snapshots" {
     "name": "[${var.environment}] Indexer fast sync snapshots haven't been uploaded in the last day",
     "type": "query alert",
     "query": "sum(last_1d):sum:aws.s3.put_requests{bucketname:${local.snapshot_bucket_prefix}-full-node-snapshots}.as_count() < 1",
-    "message": "Indexer fast sync snapshots haven't been uploaded in the last day. Please investigate the snapshotting full node.\n\n${local.monitor_suffix_literal}",
+    "message": "Indexer fast sync snapshots haven't been uploaded in the last day. Please investigate the snapshotting full node.\n\n${local.monitor_suffix_literal}${local.monitor_no_data_suffix_literal}",
     "tags": [
         "team:${var.team}",
         "env:${var.env_tag}"
@@ -303,7 +303,8 @@ resource "datadog_monitor_json" "fast_sync_snapshots" {
         "notify_audit": false,
         "require_full_window": false,
         "notify_no_data": true,
-        "renotify_interval": 0,
+        "renotify_interval": 720,
+        "renotify_no_data": true,
         "include_tags": false,
         "evaluation_delay": 900,
         "no_data_timeframe": 1440,