From 4f4a9cd6410fdd059f51bdce289aa359697ad275 Mon Sep 17 00:00:00 2001
From: Tim Meusel <tim@bastelfreak.de>
Date: Thu, 25 Apr 2024 12:39:44 +0200
Subject: [PATCH] Add function to check if nodes are reachable via bolt

At the moment the plans assume that all nodes are available. I had a few
customer setups where one of the compilers wasn't reachable during a
convert/upgrade. To not put the PE infra into an undefined state, it
makes sense to check the availability before running the plans.
---
 REFERENCE.md                    | 25 +++++++++++++++++++++++++
 functions/check_availability.pp | 21 +++++++++++++++++++++
 plans/convert.pp                |  6 ++++++
 plans/install.pp                |  7 +++++++
 plans/upgrade.pp                |  6 ++++++
 5 files changed, 65 insertions(+)
 create mode 100644 functions/check_availability.pp

diff --git a/REFERENCE.md b/REFERENCE.md
index e09e7f17..173ab89d 100644
--- a/REFERENCE.md
+++ b/REFERENCE.md
@@ -20,6 +20,7 @@
 * [`peadm::assert_supported_pe_version`](#peadm--assert_supported_pe_version): Assert that the PE version given is supported by PEAdm
 * [`peadm::bolt_version`](#peadm--bolt_version)
 * [`peadm::certname`](#peadm--certname): Return the certname of the given target-like input
+* [`peadm::check_availability`](#peadm--check_availability): check if a group of targets are reachable for bolt
 * [`peadm::check_version_and_known_hosts`](#peadm--check_version_and_known_hosts): Checks PE verison and warns about setting r10k_known_hosts
 * [`peadm::convert_hash`](#peadm--convert_hash): converts two arrays into hash
 * [`peadm::convert_status`](#peadm--convert_status): Transforms a value in a human readable status with or without colors
@@ -266,6 +267,30 @@ Variant[Target,
 
 
 
+### <a name="peadm--check_availability"></a>`peadm::check_availability`
+
+Type: Puppet Language
+
+check if a group of targets are reachable for bolt
+
+#### `peadm::check_availability(TargetSpec $targets, Boolean $output_details = true)`
+
+The peadm::check_availability function.
+
+Returns: `Integer` counter for unavailable nodes
+
+##### `targets`
+
+Data type: `TargetSpec`
+
+list of targets that are going to be checked
+
+##### `output_details`
+
+Data type: `Boolean`
+
+flag to enable/disable error output for failed nodes
+
 ### <a name="peadm--check_version_and_known_hosts"></a>`peadm::check_version_and_known_hosts`
 
 Type: Puppet Language
diff --git a/functions/check_availability.pp b/functions/check_availability.pp
new file mode 100644
index 00000000..1e8fc071
--- /dev/null
+++ b/functions/check_availability.pp
@@ -0,0 +1,21 @@
+#
+# @summary check if a group of targets are reachable for bolt
+#
+# @param targets list of targets that are going to be checked
+# @param output_details flag to enable/disable error output for failed nodes
+#
+# @return counter for unavailable nodes
+#
+# @author Tim Meusel <tim@bastelfreak.de>
+#
+function peadm::check_availability(TargetSpec $targets, Boolean $output_details = true) >> Integer {
+  $check_result = wait_until_available($targets, wait_time => 2, _catch_errors => true)
+  if $check_result.ok {
+  } elsif $output_details {
+    $check_result.error_set.each |$result| {
+      out::message("## node ${result.target} has connection error '${result.error.kind}' with message '${result.error.msg}'")
+    }
+  }
+
+  return $check_result.error_set.count
+}
diff --git a/plans/convert.pp b/plans/convert.pp
index 1995a0b0..817e67d4 100644
--- a/plans/convert.pp
+++ b/plans/convert.pp
@@ -56,6 +56,12 @@
   )
 
   out::message('# Gathering information')
+  $failed_node_counter = puppet_installation::check_availability($pe_installer_targets)
+  if $failed_node_counter > 0 {
+    fail("# Stopping peadm::convert because ${$failed_node_counter} nodes aren't available")
+  } else {
+    out::message('# All nodes are reachable, continuing with peadm::convert')
+  }
 
   # Get trusted fact information for all compilers. Use peadm::certname() as
   # the hash key because the apply block below will break trying to parse the
diff --git a/plans/install.pp b/plans/install.pp
index 1f131533..b8fe51bd 100644
--- a/plans/install.pp
+++ b/plans/install.pp
@@ -76,7 +76,14 @@
 ) {
   peadm::assert_supported_bolt_version()
 
+  out::message('# Gathering information')
   peadm::assert_supported_pe_version($version, $permit_unsafe_versions)
+  $failed_node_counter = puppet_installation::check_availability($pe_installer_targets)
+  if $failed_node_counter > 0 {
+    fail("# Stopping peadm::upgrade because ${$failed_node_counter} nodes aren't available")
+  } else {
+    out::message('# All nodes are reachable, continuing with peadm::install')
+  }
 
   $install_result = run_plan('peadm::subplans::install',
     # Standard
diff --git a/plans/upgrade.pp b/plans/upgrade.pp
index 06ac068d..2d10a976 100644
--- a/plans/upgrade.pp
+++ b/plans/upgrade.pp
@@ -98,6 +98,12 @@
   ])
 
   out::message('# Gathering information')
+  $failed_node_counter = puppet_installation::check_availability($pe_installer_targets)
+  if $failed_node_counter > 0 {
+    fail("# Stopping peadm::upgrade because ${$failed_node_counter} nodes aren't available")
+  } else {
+    out::message('# All nodes are reachable, continuing with peadm::upgrade')
+  }
 
   # lint:ignore:strict_indent
   $primary_target.peadm::fail_on_transport('pcp', @(HEREDOC/n))