From d9a10f2ab750d1edda0e1c51af5f71a5551f0e51 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 23 Dec 2022 15:39:28 +0530 Subject: [PATCH 1/5] Reshard the cluster using the json template --- playbooks/node.json.j2 | 5 ++++ playbooks/remove-node.yml | 55 ++++++++++++++++++++++++++++++--------- 2 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 playbooks/node.json.j2 diff --git a/playbooks/node.json.j2 b/playbooks/node.json.j2 new file mode 100644 index 0000000..8bb5963 --- /dev/null +++ b/playbooks/node.json.j2 @@ -0,0 +1,5 @@ +{ + "removenode" : "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}", + "presentnode" : "{{ transfer_node_id.stdout }}" +} + diff --git a/playbooks/remove-node.yml b/playbooks/remove-node.yml index 878a0e1..23d6239 100644 --- a/playbooks/remove-node.yml +++ b/playbooks/remove-node.yml @@ -5,12 +5,12 @@ leader_redis_port: 6379 follower_redis_port: 6380 tasks: - - name: Ensuring the temporary file is created before execution - local_action: - module: file - path: /tmp/node-removal - state: touch - + # - name: Ensuring the temporary file is created before execution + # local_action: + # module: file + # path: /tmp/node-removal + # state: touch + - name: Ensure redis-cli is generated without password set_fact: redis_command: redis-cli @@ -26,13 +26,37 @@ {{ redis_command }} cluster nodes register: command_result - - name: Ensure the nodes need to removed are present in temporary file - local_action: - module: copy - content: "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}" - dest: /tmp/node-removal + - name: Check the Node id of transfer node + shell : |- + redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} cluster myid + register: transfer_node_id + when: > + node_status == "ready" and inventory_hostname in groups['leader'] + + # - name: Ensure the nodes need to removed are present in temporary file + # local_action: + # module: copy + # content: "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}" + # dest: /tmp/node-removal + # when: > + # node_status == 'remove' + - name: Create the Json file with the remove and ready redis node + ansible.builtin.template: + src: node.json.j2 + dest: /tmp/node.json + delegate_to: 127.0.0.1 + + - name: Reshard the Node to be removed + command : |- + {{ redis_command }} --cluster reshard {{ansible_ssh_host}}:{{redis_port}} + --cluster-from {{item.removenode.split(' ')[0]}} + --cluster-to {{ item.presentnode }} + --cluster-slots {{ redis_slots }} + --cluster-yes when: > - node_status == 'remove' + node_status == 'remove' and inventory_hostname in groups['leader'] + with_items: + - "{{ lookup('file', '/tmp/node.json') | from_json }}" - name: Ensuring the removed marked nodes are not part of the cluster command: |- @@ -43,9 +67,14 @@ when: > inventory_hostname in groups['leader'][0] and lookup('file', '/tmp/node-removal')|length > 0 + + - name: Rebalance the Redis cluster + command: |- + redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} --cluster rebalance {{ansible_ssh_host}}:{{redis_port}} + when: node_status == "ready" and inventory_hostname in groups['leader'] - name: Ensuring the temporary file is removed after execution local_action: module: file - path: /tmp/node-removal + path: /tmp/node.json state: absent From 2b3b2cc48df0539ae0270c713f86265b5a2fa6ae Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 23 Dec 2022 17:07:19 +0530 Subject: [PATCH 2/5] some updates --- playbooks/node.json.j2 | 2 +- playbooks/remove-node.yml | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/playbooks/node.json.j2 b/playbooks/node.json.j2 index 8bb5963..6e21c0c 100644 --- a/playbooks/node.json.j2 +++ b/playbooks/node.json.j2 @@ -1,5 +1,5 @@ { - "removenode" : "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}", + "removenode" : "{{ remove_node_id.stdout }}", "presentnode" : "{{ transfer_node_id.stdout }}" } diff --git a/playbooks/remove-node.yml b/playbooks/remove-node.yml index 23d6239..41bfe62 100644 --- a/playbooks/remove-node.yml +++ b/playbooks/remove-node.yml @@ -4,12 +4,8 @@ redis_password: Opstree@1234 leader_redis_port: 6379 follower_redis_port: 6380 + redis_slots: 3446 tasks: - # - name: Ensuring the temporary file is created before execution - # local_action: - # module: file - # path: /tmp/node-removal - # state: touch - name: Ensure redis-cli is generated without password set_fact: @@ -25,6 +21,13 @@ command: |- {{ redis_command }} cluster nodes register: command_result + + - name: Check the Node id of the remove node + shell: |- + redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} cluster myid + register: remove_node_id + when: > + node_status == "remove" and inventory_hostname in groups['leader'] - name: Check the Node id of transfer node shell : |- @@ -40,6 +43,7 @@ # dest: /tmp/node-removal # when: > # node_status == 'remove' + - name: Create the Json file with the remove and ready redis node ansible.builtin.template: src: node.json.j2 @@ -49,7 +53,7 @@ - name: Reshard the Node to be removed command : |- {{ redis_command }} --cluster reshard {{ansible_ssh_host}}:{{redis_port}} - --cluster-from {{item.removenode.split(' ')[0]}} + --cluster-from {{item.removenode}} --cluster-to {{ item.presentnode }} --cluster-slots {{ redis_slots }} --cluster-yes @@ -61,12 +65,12 @@ - name: Ensuring the removed marked nodes are not part of the cluster command: |- {{ redis_command }} --cluster del-node - {{ ansible_ssh_host }}:{{ leader_redis_port }} {{ item.split(' ')[0] }} + {{ ansible_ssh_host }}:{{ leader_redis_port }} {{ item.removenode}} with_items: - - "{{ lookup('file', '/tmp/node-removal') }}" + - "{{ lookup('file', '/tmp/node.json') | from_json }}" when: > inventory_hostname in groups['leader'][0] and - lookup('file', '/tmp/node-removal')|length > 0 + lookup('file', '/tmp/node.json')|length > 0 - name: Rebalance the Redis cluster command: |- From cf65d85479ccd925996379614c37208a0eb24076 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Sat, 24 Dec 2022 02:23:01 +0530 Subject: [PATCH 3/5] final changes --- inventory/cluster.ini | 24 +++++------ playbooks/node.json.j2 | 4 +- playbooks/remove-node.yml | 84 ++++++++++++++++++++++++++++----------- 3 files changed, 74 insertions(+), 38 deletions(-) diff --git a/inventory/cluster.ini b/inventory/cluster.ini index 96bd497..fecb814 100644 --- a/inventory/cluster.ini +++ b/inventory/cluster.ini @@ -1,21 +1,21 @@ [redis] -redis1 ansible_ssh_host=172.31.57.91 -redis2 ansible_ssh_host=172.31.49.187 -redis3 ansible_ssh_host=172.31.62.128 -#redis4 ansible_ssh_host=172.31.55.4 +redis1 ansible_ssh_host=192.168.1.13 +redis2 ansible_ssh_host=192.168.1.14 +redis3 ansible_ssh_host=192.168.1.15 +redis4 ansible_ssh_host=192.168.1.16 [redis_cluster:children] leader follower [leader] -redis-leader1 ansible_ssh_host=172.31.57.91 redis_port=6379 exporter_port=9121 leader_id=0 node_status=ready -redis-leader2 ansible_ssh_host=172.31.49.187 redis_port=6379 exporter_port=9121 leader_id=1 node_status=ready -redis-leader3 ansible_ssh_host=172.31.62.128 redis_port=6379 exporter_port=9121 leader_id=2 node_status=ready -#redis-leader4 ansible_ssh_host=172.31.55.4 redis_port=6379 exporter_port=9121 leader_id=3 node_status=remove +redis-leader1 ansible_ssh_host=192.168.1.13 redis_port=6379 exporter_port=9121 leader_id=0 node_status=ready +redis-leader2 ansible_ssh_host=192.168.1.14 redis_port=6379 exporter_port=9121 leader_id=1 node_status=ready +redis-leader3 ansible_ssh_host=192.168.1.15 redis_port=6379 exporter_port=9121 leader_id=2 node_status=ready +redis-leader4 ansible_ssh_host=192.168.1.16 redis_port=6379 exporter_port=9121 leader_id=3 node_status=ready [follower] -redis-follower1 ansible_ssh_host=172.31.57.91 redis_port=6380 exporter_port=9122 leader_id=0 node_status=ready -redis-follower2 ansible_ssh_host=172.31.49.187 redis_port=6380 exporter_port=9122 leader_id=1 node_status=ready -redis-follower3 ansible_ssh_host=172.31.62.128 redis_port=6380 exporter_port=9122 leader_id=2 node_status=ready -#redis-follower4 ansible_ssh_host=172.31.55.4 redis_port=6380 exporter_port=9122 leader_id=3 node_status=remove +redis-follower1 ansible_ssh_host=192.168.1.13 redis_port=6380 exporter_port=9122 leader_id=0 node_status=ready +redis-follower2 ansible_ssh_host=192.168.1.14 redis_port=6380 exporter_port=9122 leader_id=1 node_status=ready +redis-follower3 ansible_ssh_host=192.168.1.15 redis_port=6380 exporter_port=9122 leader_id=2 node_status=ready +redis-follower4 ansible_ssh_host=192.168.1.16 redis_port=6380 exporter_port=9122 leader_id=3 node_status=ready diff --git a/playbooks/node.json.j2 b/playbooks/node.json.j2 index 6e21c0c..a076ad4 100644 --- a/playbooks/node.json.j2 +++ b/playbooks/node.json.j2 @@ -1,5 +1,5 @@ { - "removenode" : "{{ remove_node_id.stdout }}", - "presentnode" : "{{ transfer_node_id.stdout }}" + "removenode" : {{ (lookup('file', '/tmp/node-removal')) }} , + "presentnode" : {{ (lookup('file', '/tmp/node-ready')) }} } diff --git a/playbooks/remove-node.yml b/playbooks/remove-node.yml index 41bfe62..db27003 100644 --- a/playbooks/remove-node.yml +++ b/playbooks/remove-node.yml @@ -5,8 +5,20 @@ leader_redis_port: 6379 follower_redis_port: 6380 redis_slots: 3446 + tasks: - + - name: Ensuring the temporary file is created before execution + local_action: + module: file + path: /tmp/node-removal + state: touch + + - name: Ensuring the temporary file is created before execution + local_action: + module: file + path: /tmp/node-ready + state: touch + - name: Ensure redis-cli is generated without password set_fact: redis_command: redis-cli @@ -20,29 +32,41 @@ - name: Ensure if redis cluster exists or not command: |- {{ redis_command }} cluster nodes - register: command_result + register: command_result - - name: Check the Node id of the remove node - shell: |- - redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} cluster myid - register: remove_node_id - when: > - node_status == "remove" and inventory_hostname in groups['leader'] + # - name: Check the Node id of the remove node + # shell: |- + # redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} cluster myid + # register: register_remove_node_id + # vars: + # remove_node_id : "{{register_remove_node_id.stdout}}" + # when: > + # node_status == "remove" and inventory_hostname in groups['leader'] - - name: Check the Node id of transfer node - shell : |- - redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} cluster myid - register: transfer_node_id - when: > - node_status == "ready" and inventory_hostname in groups['leader'] + # - name: Check the Node id of transfer node + # shell : |- + # redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} cluster myid + # register: register_transfer_node_id + # vars: + # transfer_node_id : "{{register_transfer_node_id.stdout}}" + # when: > + # node_status == "ready" and inventory_hostname in groups['leader'] - # - name: Ensure the nodes need to removed are present in temporary file - # local_action: - # module: copy - # content: "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}" - # dest: /tmp/node-removal - # when: > - # node_status == 'remove' + - name: Ensure the nodes need to removed are present in temporary file + local_action: + module: copy + content: "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}" + dest: /tmp/node-removal + when: > + node_status == 'remove' + + - name: Ensure the nodes need to transfer are present in temporary file + local_action: + module: copy + content: "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}" + dest: /tmp/node-ready + when: > + node_status == 'ready' - name: Create the Json file with the remove and ready redis node ansible.builtin.template: @@ -53,8 +77,8 @@ - name: Reshard the Node to be removed command : |- {{ redis_command }} --cluster reshard {{ansible_ssh_host}}:{{redis_port}} - --cluster-from {{item.removenode}} - --cluster-to {{ item.presentnode }} + --cluster-from {{ item.removenode[0].split(' ')[0] }} + --cluster-to {{ item.presentnode[0].split(' ')[0] }} --cluster-slots {{ redis_slots }} --cluster-yes when: > @@ -65,7 +89,7 @@ - name: Ensuring the removed marked nodes are not part of the cluster command: |- {{ redis_command }} --cluster del-node - {{ ansible_ssh_host }}:{{ leader_redis_port }} {{ item.removenode}} + {{ ansible_ssh_host }}:{{ leader_redis_port }} {{ item.removenode[0].split(' ')[0] }} with_items: - "{{ lookup('file', '/tmp/node.json') | from_json }}" when: > @@ -82,3 +106,15 @@ module: file path: /tmp/node.json state: absent + + - name: Ensuring the temporary file is removed after execution + local_action: + module: file + path: /tmp/node-removal + state: absent + + - name: Ensuring the temporary file is removed after execution + local_action: + module: file + path: /tmp/node-ready + state: absent From e0417ee50b470ff4b3b09727096e64f106d67d54 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 12 Jan 2023 23:31:53 +0530 Subject: [PATCH 4/5] fix --- playbooks/remove-node.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/playbooks/remove-node.yml b/playbooks/remove-node.yml index db27003..15521d1 100644 --- a/playbooks/remove-node.yml +++ b/playbooks/remove-node.yml @@ -4,7 +4,7 @@ redis_password: Opstree@1234 leader_redis_port: 6379 follower_redis_port: 6380 - redis_slots: 3446 + redis_slots: 4096 tasks: - name: Ensuring the temporary file is created before execution @@ -58,7 +58,7 @@ content: "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}" dest: /tmp/node-removal when: > - node_status == 'remove' + node_status == 'remove' and inventory_hostname in groups['leader'] - name: Ensure the nodes need to transfer are present in temporary file local_action: @@ -66,7 +66,7 @@ content: "{{ command_result.stdout_lines | select('search', ansible_ssh_host + ':' + redis_port|string) }}" dest: /tmp/node-ready when: > - node_status == 'ready' + node_status == 'ready' and inventory_hostname in groups['leader'] - name: Create the Json file with the remove and ready redis node ansible.builtin.template: @@ -86,6 +86,13 @@ with_items: - "{{ lookup('file', '/tmp/node.json') | from_json }}" + - name: Rebalance the Redis cluster + command: |- + redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} + --cluster rebalance {{ansible_ssh_host}}:{{redis_port}} + when: >- + node_status == "ready" and inventory_hostname in groups['leader'] + - name: Ensuring the removed marked nodes are not part of the cluster command: |- {{ redis_command }} --cluster del-node @@ -96,11 +103,6 @@ inventory_hostname in groups['leader'][0] and lookup('file', '/tmp/node.json')|length > 0 - - name: Rebalance the Redis cluster - command: |- - redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} --cluster rebalance {{ansible_ssh_host}}:{{redis_port}} - when: node_status == "ready" and inventory_hostname in groups['leader'] - - name: Ensuring the temporary file is removed after execution local_action: module: file From 9d22fd4b66dcbb380dd758f4500f9d6b0daab376 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 13 Jan 2023 02:30:35 +0530 Subject: [PATCH 5/5] remove comment --- playbooks/remove-node.yml | 32 +++++++------------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/playbooks/remove-node.yml b/playbooks/remove-node.yml index 15521d1..0871f76 100644 --- a/playbooks/remove-node.yml +++ b/playbooks/remove-node.yml @@ -34,24 +34,6 @@ {{ redis_command }} cluster nodes register: command_result - # - name: Check the Node id of the remove node - # shell: |- - # redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} cluster myid - # register: register_remove_node_id - # vars: - # remove_node_id : "{{register_remove_node_id.stdout}}" - # when: > - # node_status == "remove" and inventory_hostname in groups['leader'] - - # - name: Check the Node id of transfer node - # shell : |- - # redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} cluster myid - # register: register_transfer_node_id - # vars: - # transfer_node_id : "{{register_transfer_node_id.stdout}}" - # when: > - # node_status == "ready" and inventory_hostname in groups['leader'] - - name: Ensure the nodes need to removed are present in temporary file local_action: module: copy @@ -86,13 +68,6 @@ with_items: - "{{ lookup('file', '/tmp/node.json') | from_json }}" - - name: Rebalance the Redis cluster - command: |- - redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} - --cluster rebalance {{ansible_ssh_host}}:{{redis_port}} - when: >- - node_status == "ready" and inventory_hostname in groups['leader'] - - name: Ensuring the removed marked nodes are not part of the cluster command: |- {{ redis_command }} --cluster del-node @@ -102,6 +77,13 @@ when: > inventory_hostname in groups['leader'][0] and lookup('file', '/tmp/node.json')|length > 0 + + - name: Rebalance the Redis cluster + command: |- + redis-cli -h {{ansible_ssh_host}} -p {{redis_port}} -a {{redis_password}} + --cluster rebalance {{ansible_ssh_host}}:{{redis_port}} + when: >- + node_status == "ready" and inventory_hostname in groups['leader'][0] - name: Ensuring the temporary file is removed after execution local_action: