Add monitor for backup full node liveness (#89)

dydxprotocol · Mar 27, 2024 · 8704bee · 8704bee
1 parent d0fe57a
commit 8704bee
Showing 1 changed file with 31 additions and 0 deletions.
diff --git a/modules/indexer_monitors/monitors.tf b/modules/indexer_monitors/monitors.tf
@@ -129,6 +129,37 @@ resource "datadog_monitor_json" "indexer_full_node_down" {
 EOF
 }
 
+resource "datadog_monitor_json" "indexer_backup_full_node_down" {
+  count = var.enable_precautionary_monitors ? 1 : 0
+
+  monitor = <<EOF
+{
+	"id": 142147978,
+	"name": "[mainnet] Indexer backup full node validator cluster has less than 1 task in past 5 minutes",
+	"type": "query alert",
+	"query": "avg(last_5m):avg:aws.ecs.service.running{cluster:mainnet-indexer-full-node-backup-cluster} < 1",
+	"message": "Indexer backup full node validator cluster has less than 1 task in past 5 minutes\n\n${local.monitor_suffix_literal}",
+	"tags": [
+		"team:${var.team}",
+		"env:${var.env_tag}"
+	],
+	"options": {
+		"thresholds": {
+			"critical": 1
+		},
+		"notify_audit": false,
+		"include_tags": false,
+		"evaluation_delay": 900,
+		"notify_no_data": false,
+		"new_host_delay": 300,
+		"silenced": {}
+	},
+	"priority": null,
+	"restricted_roles": null
+}
+EOF
+}
+
 resource "datadog_monitor_json" "last_processed_block_last_30min" {
   monitor = <<EOF
 {