Reshard and Rebalance the Redis cluster on Redis Node Removal

inventory/cluster.ini

@@ -1,21 +1,21 @@
 [redis]
-redis1 ansible_ssh_host=172.31.57.91
-redis2 ansible_ssh_host=172.31.49.187
-redis3 ansible_ssh_host=172.31.62.128
-#redis4 ansible_ssh_host=172.31.55.4
+redis1 ansible_ssh_host=192.168.1.13
+redis2 ansible_ssh_host=192.168.1.14
+redis3 ansible_ssh_host=192.168.1.15
+redis4 ansible_ssh_host=192.168.1.16
 
 [redis_cluster:children]
 leader
 follower
 
 [leader]
-redis-leader1 ansible_ssh_host=172.31.57.91 redis_port=6379 exporter_port=9121 leader_id=0 node_status=ready
-redis-leader2 ansible_ssh_host=172.31.49.187 redis_port=6379 exporter_port=9121 leader_id=1 node_status=ready
-redis-leader3 ansible_ssh_host=172.31.62.128 redis_port=6379 exporter_port=9121 leader_id=2 node_status=ready
-#redis-leader4 ansible_ssh_host=172.31.55.4 redis_port=6379 exporter_port=9121 leader_id=3 node_status=remove
+redis-leader1 ansible_ssh_host=192.168.1.13 redis_port=6379 exporter_port=9121 leader_id=0 node_status=ready
+redis-leader2 ansible_ssh_host=192.168.1.14 redis_port=6379 exporter_port=9121 leader_id=1 node_status=ready
+redis-leader3 ansible_ssh_host=192.168.1.15 redis_port=6379 exporter_port=9121 leader_id=2 node_status=ready
+redis-leader4 ansible_ssh_host=192.168.1.16 redis_port=6379 exporter_port=9121 leader_id=3 node_status=ready
 
 [follower]
-redis-follower1 ansible_ssh_host=172.31.57.91 redis_port=6380 exporter_port=9122 leader_id=0 node_status=ready
-redis-follower2 ansible_ssh_host=172.31.49.187 redis_port=6380 exporter_port=9122 leader_id=1 node_status=ready
-redis-follower3 ansible_ssh_host=172.31.62.128 redis_port=6380 exporter_port=9122 leader_id=2 node_status=ready
-#redis-follower4 ansible_ssh_host=172.31.55.4 redis_port=6380 exporter_port=9122 leader_id=3 node_status=remove
+redis-follower1 ansible_ssh_host=192.168.1.13 redis_port=6380 exporter_port=9122 leader_id=0 node_status=ready
+redis-follower2 ansible_ssh_host=192.168.1.14 redis_port=6380 exporter_port=9122 leader_id=1 node_status=ready
+redis-follower3 ansible_ssh_host=192.168.1.15 redis_port=6380 exporter_port=9122 leader_id=2 node_status=ready
+redis-follower4 ansible_ssh_host=192.168.1.16 redis_port=6380 exporter_port=9122 leader_id=3 node_status=ready

playbooks/node.json.j2

@@ -0,0 +1,5 @@
+{
+    "removenode" :  {{ (lookup('file', '/tmp/node-removal')) }} ,
+    "presentnode" : {{ (lookup('file', '/tmp/node-ready')) }}
+}
+

playbooks/remove-node.yml

@@ -4,13 +4,21 @@
     redis_password: Opstree@1234
     leader_redis_port: 6379
     follower_redis_port: 6380
+    redis_slots: 4096
+
   tasks:
     - name: Ensuring the temporary file is created before execution
       local_action:
         module: file
         path: /tmp/node-removal
         state: touch
 
+    - name: Ensuring the temporary file is created before execution
+      local_action:
+        module: file
+        path: /tmp/node-ready
+        state: touch
+
     - name: Ensure redis-cli is generated without password
       set_fact:
         redis_command: redis-cli
@@ -24,28 +32,73 @@
     - name: Ensure if redis cluster exists or not
       command: |-
         {{ redis_command }} cluster nodes
-      register: command_result
-
+      register: command_result       
+    
     - name: Ensure the nodes need to removed are present in temporary file
       local_action:
         module: copy
         content: "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}"
         dest: /tmp/node-removal
       when: >
-        node_status == 'remove'
+        node_status == 'remove' and inventory_hostname in groups['leader']  
+
+    - name: Ensure the nodes need to transfer are present in temporary file
+      local_action:
+        module: copy
+        content: "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}"
+        dest: /tmp/node-ready
+      when: >
+        node_status == 'ready' and inventory_hostname in groups['leader'] 
+
+    - name: Create the Json file with the remove and ready redis node 
+      ansible.builtin.template:
+        src: node.json.j2
+        dest: /tmp/node.json
+      delegate_to: 127.0.0.1
+
+    - name:  Reshard the Node to be removed 
+      command : |-
+        {{ redis_command }} --cluster reshard {{ansible_ssh_host}}:{{redis_port}} 
+        --cluster-from {{ item.removenode[0].split(' ')[0] }} 
+        --cluster-to {{ item.presentnode[0].split(' ')[0] }} 
+        --cluster-slots  {{ redis_slots }}
+        --cluster-yes
+      when: >
+        node_status == 'remove' and inventory_hostname in groups['leader']
+      with_items: 
+        - "{{ lookup('file', '/tmp/node.json') | from_json }}"
 
     - name: Ensuring the removed marked nodes are not part of the cluster
       command: |-
         {{ redis_command }} --cluster del-node
-        {{ ansible_ssh_host }}:{{ leader_redis_port }} {{ item.split(' ')[0] }}
+        {{ ansible_ssh_host }}:{{ leader_redis_port }} {{ item.removenode[0].split(' ')[0] }}
       with_items:
-        - "{{ lookup('file', '/tmp/node-removal') }}"
+        - "{{ lookup('file', '/tmp/node.json') | from_json }}"
       when: >
         inventory_hostname in groups['leader'][0] and
-        lookup('file', '/tmp/node-removal')|length > 0
+        lookup('file', '/tmp/node.json')|length > 0
+
+    - name: Rebalance the Redis cluster
+      command: |- 
+        redis-cli -h  {{ansible_ssh_host}}  -p {{redis_port}} -a {{redis_password}}  
+        --cluster rebalance {{ansible_ssh_host}}:{{redis_port}}
+      when: >-
+        node_status == "ready" and inventory_hostname in groups['leader'][0]
+
+    - name: Ensuring the temporary file is removed after execution
+      local_action:
+        module: file
+        path: /tmp/node.json
+        state: absent
 
     - name: Ensuring the temporary file is removed after execution
       local_action:
         module: file
         path: /tmp/node-removal
         state: absent
+
+    - name: Ensuring the temporary file is removed after execution
+      local_action:
+        module: file
+        path: /tmp/node-ready
+        state: absent