feat: opensearch

.github/actions/aurora-manage-cluster/README.md

@@ -10,6 +10,7 @@ This action will also install Terraform and awscli. It will output the Aurora cl
 
 | name | description | required | default |
 | --- | --- | --- | --- |
+| `aws-region` | <p>AWS region where the cluster will be deployed</p> | `true` | `""` |
 | `cluster-name` | <p>Name of the RDS Aurora cluster to deploy</p> | `true` | `""` |
 | `username` | <p>Username for the PostgreSQL admin user</p> | `true` | `""` |
 | `password` | <p>Password for the PostgreSQL admin user</p> | `true` | `""` |
@@ -47,6 +48,12 @@ This action is a `composite` action.
 ```yaml
 - uses: camunda/camunda-tf-eks-module/.github/actions/aurora-manage-cluster@main
   with:
+    aws-region:
+    # AWS region where the cluster will be deployed
+    #
+    # Required: true
+    # Default: ""
+
     cluster-name:
     # Name of the RDS Aurora cluster to deploy
     #

.github/actions/aurora-manage-cluster/action.yml

@@ -6,6 +6,10 @@ description: |
     This action will also install Terraform and awscli. It will output the Aurora cluster endpoint.
 
 inputs:
+    aws-region:
+        description: AWS region where the cluster will be deployed
+        required: true
+
     cluster-name:
         description: Name of the RDS Aurora cluster to deploy
         required: true
@@ -110,6 +114,8 @@ runs:
               awscli-version: ${{ inputs.awscli-version }}
               terraform-version: ${{ inputs.terraform-version }}
 
+              aws-region: ${{ inputs.aws-region }}
+
               s3-backend-bucket: ${{ inputs.s3-backend-bucket }}
               s3-bucket-region: ${{ inputs.s3-bucket-region }}
 
@@ -132,6 +138,8 @@ runs:
           id: init
           working-directory: ${{ inputs.tf-modules-path }}/modules/aurora/
           run: |
+              set -euxo pipefail
+
               cp ../fixtures/backend.tf ./
               terraform version
               terraform init -backend-config="bucket=${{ steps.utility.outputs.TFSTATE_BUCKET }}" -backend-config="key=${{ steps.utility.outputs.TFSTATE_KEY }}" \
@@ -143,6 +151,8 @@ runs:
           id: plan
           working-directory: ${{ inputs.tf-modules-path }}/modules/aurora/
           run: |
+              set -euxo pipefail
+
               echo '${{ inputs.additional-terraform-vars }}' > /tmp/var.tfvars.json
               terraform plan -no-color -out aurora.plan \
                 -var-file=/tmp/var.tfvars.json \
@@ -159,6 +169,8 @@ runs:
           id: apply
           working-directory: ${{ inputs.tf-modules-path }}/modules/aurora/
           run: |
+              set -euxo pipefail
+
               terraform apply -no-color aurora.plan
               export aurora_endpoint="$(terraform output -raw aurora_endpoint)"
               echo "aurora_endpoint=$aurora_endpoint" >> "$GITHUB_OUTPUT"
@@ -168,5 +180,7 @@ runs:
           id: fetch_outputs
           working-directory: ${{ inputs.tf-modules-path }}/modules/aurora/
           run: |
+              set -euxo pipefail
+
               all_outputs=$(terraform output -json | jq -c .)
               echo "all_terraform_outputs=$all_outputs" | tee -a "$GITHUB_OUTPUT"

.github/actions/eks-cleanup-resources/README.md

@@ -14,6 +14,7 @@ This GitHub Action automates the deletion of EKS resources using a shell script.
 | `max-age-hours` | <p>Maximum age of resources in hours</p> | `false` | `20` |
 | `target` | <p>Specify an ID to destroy specific resources or "all" to destroy all resources</p> | `false` | `all` |
 | `temp-dir` | <p>Temporary directory prefix used for storing resource data during processing</p> | `false` | `./tmp/eks-cleanup/` |
+| `module-name` | <p>Name of the module to destroy (e.g., "eks-cluster", "aurora", "opensearch"), or "all" to destroy all modules</p> | `false` | `all` |
 
 
 ## Runs
@@ -54,4 +55,10 @@ This action is a `composite` action.
     #
     # Required: false
     # Default: ./tmp/eks-cleanup/
+
+    module-name:
+    # Name of the module to destroy (e.g., "eks-cluster", "aurora", "opensearch"), or "all" to destroy all modules
+    #
+    # Required: false
+    # Default: all
 ```

.github/actions/eks-cleanup-resources/action.yml

@@ -4,7 +4,6 @@ name: Delete EKS resources
 description: |
     This GitHub Action automates the deletion of EKS resources using a shell script.
 
-
 inputs:
     tf-bucket:
         description: Bucket containing the resources states
@@ -26,16 +25,22 @@ inputs:
         description: Temporary directory prefix used for storing resource data during processing
         default: ./tmp/eks-cleanup/
 
+    module-name:
+        description: Name of the module to destroy (e.g., "eks-cluster", "aurora", "opensearch"), or "all" to destroy all modules
+        default: all
+
 runs:
     using: composite
     steps:
         - name: Delete resources
           id: delete_resources
           shell: bash
           run: |
+              set -euxo pipefail
+
               if [ -n "${{ inputs.tf-bucket-region }}" ]; then
                 export AWS_S3_REGION="${{ inputs.tf-bucket-region }}"
               fi
 
               ${{ github.action_path }}/scripts/destroy.sh "${{ inputs.tf-bucket }}" ${{ github.action_path }}/../../../modules/ \
-                "${{ inputs.temp-dir }}" ${{ inputs.max-age-hours }} ${{ inputs.target }}
+                "${{ inputs.temp-dir }}" ${{ inputs.max-age-hours }} ${{ inputs.target }} ${{ inputs.module-name }}

.github/actions/eks-cleanup-resources/scripts/destroy.sh

@@ -9,26 +9,27 @@ set -o pipefail
 # is successful, it removes the corresponding S3 objects.
 #
 # Usage:
-# ./destroy.sh <BUCKET> <MODULES_DIR> <TEMP_DIR_PREFIX> <MIN_AGE_IN_HOURS> <ID_OR_ALL>
+# ./destroy.sh <BUCKET> <MODULES_DIR> <TEMP_DIR_PREFIX> <MIN_AGE_IN_HOURS> <ID_OR_ALL> [MODULE_NAME]
 #
 # Arguments:
 #   BUCKET: The name of the S3 bucket containing the resource state files.
 #   MODULES_DIR: The directory containing the Terraform modules.
 #   TEMP_DIR_PREFIX: The prefix for the temporary directories created for each resource.
 #   MIN_AGE_IN_HOURS: The minimum age (in hours) of resources to be destroyed.
 #   ID_OR_ALL: The specific ID suffix to filter objects, or "all" to destroy all objects.
+#   MODULE_NAME (optional): The name of the module to destroy (e.g., "eks-cluster", "aurora", "opensearch"). Default is "all".
 #
 # Example:
 # ./destroy.sh tf-state-eks-ci-eu-west-3 ./modules/eks/ /tmp/eks/ 24 all
-# ./destroy.sh tf-state-eks-ci-eu-west-3 ./modules/eks/ /tmp/eks/ 24 4891048
+# ./destroy.sh tf-state-eks-ci-eu-west-3 ./modules/eks/ /tmp/eks/ 24 4891048 eks-cluster
 #
 # Requirements:
 # - AWS CLI installed and configured with the necessary permissions to access and modify the S3 bucket.
 # - Terraform installed and accessible in the PATH.
 
 # Check for required arguments
-if [ "$#" -ne 5 ]; then
-  echo "Usage: $0 <BUCKET> <MODULES_DIR> <TEMP_DIR_PREFIX> <MIN_AGE_IN_HOURS> <ID_OR_ALL>"
+if [ "$#" -lt 5 ] || [ "$#" -gt 6 ]; then
+  echo "Usage: $0 <BUCKET> <MODULES_DIR> <TEMP_DIR_PREFIX> <MIN_AGE_IN_HOURS> <ID_OR_ALL> [MODULE_NAME]"
   exit 1
 fi
 
@@ -50,6 +51,7 @@ MODULES_DIR=$2
 TEMP_DIR_PREFIX=$3
 MIN_AGE_IN_HOURS=$4
 ID_OR_ALL=$5
+MODULE_NAME=${6:-all}
 FAILED=0
 CURRENT_DIR=$(pwd)
 AWS_S3_REGION=${AWS_S3_REGION:-$AWS_REGION}
@@ -134,9 +136,7 @@ destroy_resource() {
 
   # Execute the terraform destroy command with appropriate variables (see https://github.com/hashicorp/terraform/issues/23552)
   if [ "$terraform_module" == "eks-cluster" ]; then
-    if terraform state list | grep -q "kubernetes_storage_class_v1.ebs_sc"; then
-      terraform state rm "kubernetes_storage_class_v1.ebs_sc"
-    fi
+    terraform state rm "kubernetes_storage_class_v1.ebs_sc" || true
 
     if ! terraform destroy -auto-approve \
       -var="region=$AWS_REGION" \
@@ -152,6 +152,16 @@ destroy_resource() {
       -var="subnet_ids=[]" \
       -var="cidr_blocks=[]" \
       -var="vpc_id=vpc-dummy"; then return 1; fi
+
+  elif [ "$terraform_module" == "opensearch" ]; then
+    if ! terraform destroy -auto-approve \
+      -var="domain_name=$cluster_name" \
+      -var="vpc_id=vpc-dummy" \
+      -var="advanced_security_master_user_password=dummy" \
+      -var="vpc_id=vpc-dummy" \
+      -var="cidr_blocks=[]" \
+      -var="subnet_ids=[]"; then return 1; fi
+
   else
     echo "Unsupported module: $terraform_module"
     return 1
@@ -175,60 +185,106 @@ if [ $aws_exit_code -ne 0 ]; then
   exit 1
 fi
 
-
+# Categorize resources by module type
 if [ "$ID_OR_ALL" == "all" ]; then
   resources=$(echo "$all_objects" | grep "/terraform.tfstate" | awk '{print $4}')
 else
   resources=$(echo "$all_objects" | grep "/terraform.tfstate" | grep "$ID_OR_ALL" | awk '{print $4}')
 fi
+
 # Check if resources is empty (i.e., no objects found)
 if [ -z "$resources" ]; then
   echo "No terraform.tfstate objects found in the S3 bucket. Exiting script." >&2
   exit 0
 fi
 
-current_timestamp=$($date_command +%s)
+# Initialise arrays for the resources by module type
+aurora_resources=()
+opensearch_resources=()
+eks_resources=()
 
+# Classify resources into different module types
 for resource_id in $resources; do
-  cd "$CURRENT_DIR" || return 1
-
   terraform_module=$(basename "$(dirname "$resource_id")")
-  echo "Checking resource $resource_id (terraform module=$terraform_module)"
 
-  last_modified=$(aws s3api head-object --bucket "$BUCKET" --key "$resource_id" --output json | grep LastModified | awk -F '"' '{print $4}')
-  if [ -z "$last_modified" ]; then
-    echo "Error: Failed to retrieve last modified timestamp for resource $resource_id"
-    exit 1
-  fi
+  case "$terraform_module" in
+    aurora)
+      aurora_resources+=("$resource_id")
+      ;;
+    opensearch)
+      opensearch_resources+=("$resource_id")
+      ;;
+    eks-cluster)
+      eks_resources+=("$resource_id")
+      ;;
+    *)
+      echo "Skipping unsupported module: $terraform_module"
+      ;;
+  esac
+done
 
-  last_modified_timestamp=$($date_command -d "$last_modified" +%s)
-  if [ -z "$last_modified_timestamp" ]; then
-    echo "Error: Failed to convert last modified timestamp to seconds since epoch for resource $resource_id"
-    exit 1
-  fi
-  echo "resource $resource_id last modification: $last_modified ($last_modified_timestamp)"
+current_timestamp=$($date_command +%s)
 
-  file_age_hours=$(( ($current_timestamp - $last_modified_timestamp) / 3600 ))
-  if [ -z "$file_age_hours" ]; then
-    echo "Error: Failed to calculate file age in hours for resource $resource_id"
-    exit 1
-  fi
-  echo "resource $resource_id is $file_age_hours hours old"
+# Function to process the destruction for a specific resource type
+process_resources_in_order() {
+  local resources=("$@")  # Accept an array of resources to process
 
-  if [ $file_age_hours -ge "$MIN_AGE_IN_HOURS" ]; then
-    # name of the cluster is always after terraform/
-    cluster_name=$(echo "$resource_id" | cut -d'/' -f2)
-    echo "Destroying resource $resource_id in $terraform_module (cluster_name=$cluster_name)"
+  for resource_id in "${resources[@]}"; do
+    cd "$CURRENT_DIR" || return 1
 
-    if ! destroy_resource "$resource_id" "$terraform_module" "$cluster_name"; then
-      echo "Error destroying resource $resource_id"
-      FAILED=1
+    terraform_module=$(basename "$(dirname "$resource_id")")
+    echo "Checking resource $resource_id (terraform module=$terraform_module)"
+
+    # Apply module name filter if specified
+    if [ "$MODULE_NAME" != "all" ] && [ "$MODULE_NAME" != "$terraform_module" ]; then
+      echo "Skipping resource $resource_id because it does not match the specified module name: $MODULE_NAME"
+      continue
     fi
 
-  else
-    echo "Skipping resource $resource_id as it does not meet the minimum age requirement of $MIN_AGE_IN_HOURS hours"
-  fi
-done
+    last_modified=$(aws s3api head-object --bucket "$BUCKET" --key "$resource_id" --output json | grep LastModified | awk -F '"' '{print $4}')
+    if [ -z "$last_modified" ]; then
+      echo "Error: Failed to retrieve last modified timestamp for resource $resource_id"
+      exit 1
+    fi
+
+    last_modified_timestamp=$($date_command -d "$last_modified" +%s)
+    if [ -z "$last_modified_timestamp" ]; then
+      echo "Error: Failed to convert last modified timestamp to seconds since epoch for resource $resource_id"
+      exit 1
+    fi
+    echo "Resource $resource_id last modification: $last_modified ($last_modified_timestamp)"
+
+    file_age_hours=$(( ($current_timestamp - $last_modified_timestamp) / 3600 ))
+    if [ -z "$file_age_hours" ]; then
+      echo "Error: Failed to calculate file age in hours for resource $resource_id"
+      exit 1
+    fi
+    echo "Resource $resource_id is $file_age_hours hours old"
+
+    if [ $file_age_hours -ge "$MIN_AGE_IN_HOURS" ]; then
+      # Name of the cluster is always after terraform/
+      cluster_name=$(echo "$resource_id" | cut -d'/' -f2)
+      echo "Destroying resource $resource_id in $terraform_module (cluster_name=$cluster_name)"
+
+      if ! destroy_resource "$resource_id" "$terraform_module" "$cluster_name"; then
+        echo "Error destroying resource $resource_id"
+        FAILED=1
+      fi
+    else
+      echo "Skipping resource $resource_id as it does not meet the minimum age requirement of $MIN_AGE_IN_HOURS hours"
+    fi
+  done
+}
+
+# Destroy resources in the specific order: Aurora, OpenSearch, then EKS
+echo "Destroying Aurora resources..."
+process_resources_in_order "${aurora_resources[@]}"
+
+echo "Destroying OpenSearch resources..."
+process_resources_in_order "${opensearch_resources[@]}"
+
+echo "Destroying EKS resources..."
+process_resources_in_order "${eks_resources[@]}"
 
 echo "Cleaning up empty folders in s3://$BUCKET"
 # Loop until no empty folders are found

.github/actions/eks-manage-cluster/action.yml

@@ -116,6 +116,8 @@ runs:
           id: init
           working-directory: ${{ inputs.tf-modules-path }}/modules/eks-cluster/
           run: |
+              set -euxo pipefail
+
               cp ../fixtures/backend.tf ./
               terraform version
               terraform init -backend-config="bucket=${{ steps.utility.outputs.TFSTATE_BUCKET }}" \
@@ -127,6 +129,8 @@ runs:
           id: plan
           working-directory: ${{ inputs.tf-modules-path }}/modules/eks-cluster/
           run: |
+              set -euxo pipefail
+
               echo '${{ inputs.additional-terraform-vars }}' > /tmp/var.tfvars.json
               terraform plan -no-color -out eks.plan \
                 -var-file=/tmp/var.tfvars.json \
@@ -139,6 +143,8 @@ runs:
           id: apply
           working-directory: ${{ inputs.tf-modules-path }}/modules/eks-cluster/
           run: |
+              set -euxo pipefail
+
               terraform apply -no-color eks.plan
               export cluster_endpoint="$(terraform output -raw cluster_endpoint)"
               echo "cluster_endpoint=$cluster_endpoint" >> "$GITHUB_OUTPUT"
@@ -148,6 +154,8 @@ runs:
           id: fetch_outputs
           working-directory: ${{ inputs.tf-modules-path }}/modules/eks-cluster/
           run: |
+              set -euxo pipefail
+
               all_outputs=$(terraform output -json | jq -c .)
               echo "all_terraform_outputs=$all_outputs" | tee -a "$GITHUB_OUTPUT"
 
@@ -156,11 +164,15 @@ runs:
           id: kube_config
           if: inputs.login == 'true'
           run: |
+              set -euxo pipefail
+
               aws eks --region ${{ inputs.aws-region }} update-kubeconfig --name ${{ inputs.cluster-name }}
 
         - name: Output Kube Config
           shell: bash
           if: inputs.login == 'true'
           run: |
+              set -euxo pipefail
+
               kubectl config view
               kubectl get ns