From 28b6d655f9ca23761d7dd1f1b089d26929789678 Mon Sep 17 00:00:00 2001
From: Seth Grover <seth.d.grover@gmail.com>
Date: Tue, 12 Nov 2024 12:42:12 -0700
Subject: [PATCH] for idaholab/Malcolm#491; added a logstash health check

---
 api/project/__init__.py              | 22 +++++-----------------
 docker-compose-dev.yml               |  2 +-
 docker-compose.yml                   |  2 +-
 kubernetes/14-logstash.yml           |  7 +++----
 logstash/scripts/container_health.sh | 12 ++++++++++++
 5 files changed, 22 insertions(+), 23 deletions(-)
 create mode 100755 logstash/scripts/container_health.sh

diff --git a/api/project/__init__.py b/api/project/__init__.py
index ff7e7cfb1..46a7c0111 100644
--- a/api/project/__init__.py
+++ b/api/project/__init__.py
@@ -169,15 +169,6 @@
 missing_field_map['ip'] = '0.0.0.0'
 missing_field_map['long'] = 0
 
-logstash_default_pipelines = [
-    "malcolm-beats",
-    "malcolm-enrichment",
-    "malcolm-input",
-    "malcolm-output",
-    "malcolm-suricata",
-    "malcolm-zeek",
-]
-
 urllib3.disable_warnings()
 warnings.filterwarnings(
     "ignore",
@@ -937,7 +928,7 @@ def ready():
     logstash_lumberjack
         true or false, the ready status of Logstash's lumberjack protocol listener
     logstash_pipelines
-        true or false, the ready status of Logstash's default pipelines
+        true or false, the ready status of Logstash's pipelines
     netbox
         true or false, the ready status of NetBox
     opensearch
@@ -998,9 +989,9 @@ def ready():
             print(f"{type(e).__name__}: {str(e)} getting freq status")
 
     try:
-        logstashStats = requests.get(f'{logstashUrl}/_node').json()
+        logstashHealth = requests.get(f'{logstashUrl}/_health_report').json()
     except Exception as e:
-        logstashStats = {}
+        logstashHealth = {}
         if debugApi:
             print(f"{type(e).__name__}: {str(e)} getting Logstash node status")
 
@@ -1057,11 +1048,8 @@ def ready():
         filebeat_tcp=filebeatTcpJsonStatus,
         freq=freqStatus,
         logstash_lumberjack=logstashLJStatus,
-        logstash_pipelines=(malcolm_utils.deep_get(logstashStats, ["status"]) == "green")
-        and all(
-            pipeline in malcolm_utils.deep_get(logstashStats, ["pipelines"], {})
-            for pipeline in logstash_default_pipelines
-        ),
+        logstash_pipelines=(malcolm_utils.deep_get(logstashHealth, ["status"]) == "green")
+        and (malcolm_utils.deep_get(logstashHealth, ["indicators", "pipelines", "status"]) == "green"),
         netbox=bool(
             isinstance(netboxStatus, dict)
             and netboxStatus
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index f671b0379..8ae8694e0 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -290,7 +290,7 @@ services:
       target: /usr/share/logstash/malcolm-ruby
       read_only: true
     healthcheck:
-      test: ["CMD", "curl", "--silent", "--fail", "http://localhost:9600"]
+      test: ["CMD", "/usr/local/bin/container_health.sh"]
       interval: 30s
       timeout: 15s
       retries: 3
diff --git a/docker-compose.yml b/docker-compose.yml
index 770cd05ce..6128cc55d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -254,7 +254,7 @@ services:
       target: /certs/server.key
       read_only: true
     healthcheck:
-      test: ["CMD", "curl", "--silent", "--fail", "http://localhost:9600"]
+      test: ["CMD", "/usr/local/bin/container_health.sh"]
       interval: 30s
       timeout: 15s
       retries: 3
diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml
index 30b9b0f82..1239914e8 100644
--- a/kubernetes/14-logstash.yml
+++ b/kubernetes/14-logstash.yml
@@ -90,10 +90,9 @@ spec:
           - configMapRef:
               name: logstash-env
         livenessProbe:
-          httpGet:
-            path: /
-            port: 9600
-            scheme: HTTP
+          exec:
+            command:
+            - /usr/local/bin/container_health.sh
           initialDelaySeconds: 600
           periodSeconds: 30
           timeoutSeconds: 15
diff --git a/logstash/scripts/container_health.sh b/logstash/scripts/container_health.sh
new file mode 100755
index 000000000..de11eb407
--- /dev/null
+++ b/logstash/scripts/container_health.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -euo pipefail
+
+JQ_EVAL=$(
+    curl --fail --silent -XGET http://localhost:9600/_health_report | \
+        jq '(.status == "green") and
+            (.indicators.pipelines.status == "green") and
+            (all(.indicators.pipelines.indicators[]; .status == "green"))' 2>/dev/null
+)
+
+[[ "$JQ_EVAL" == "true" ]] && exit 0 && exit 1
\ No newline at end of file