replace_node.yml: Allow replacing an alive node

This patch changes the replace_node playbook to allow the replace
of an alive node by adding a variable named `alive_node_replace`,
which when set to true will stop the node being replaced
before starting the replace.

example-playbooks/replace_node/replace_node.yml

@@ -24,6 +24,46 @@
       when: hostvars[new_node]['broadcast_address'] in scylla_seeds or new_node in scylla_seeds
       run_once: true
 
+    - name: Stop scylla-server on the node being replaced
+      block:
+        - name: Disable Scylla service
+          service:
+            name: scylla-server
+            enabled:  false
+          become: true
+
+        - name: Run nodetool drain
+          async_task:
+            shell: |
+              nodetool drain
+            alias: drain_scylla
+            async: 60
+            retries: 60
+            delay: 1
+          ignore_errors: true
+          register: _drain_scylla_output
+
+        - name: Stop Scylla service
+          async_task:
+            shell: |
+              systemctl stop scylla-server
+            alias: stop_scylla
+            async: 5
+            retries: 5
+            delay: 1
+          ignore_errors: true
+          register: _stop_scylla_output
+          become: true
+
+        - name: Send SIGKILL to scylla
+          shell: |
+            pkill -9 scylla$
+          ignore_errors: true
+          become: true
+      delegate_to: "{{ replaced_node }}"
+      run_once: true
+      when: alive_node_replace is defined and alive_node_replace|bool
+
     - name: Save one of the seeds as a fact in order to run some checks
       set_fact:
         alive_node: "{{ item }}"

example-playbooks/replace_node/vars/main.yml

@@ -30,6 +30,10 @@ new_node_repair_timeout_seconds: 36000
 # Default bootstrap timeout for the new node
 new_node_bootstrap_wait_time_sec: 25200
 
+# If you set this to true, this playbook will allow you to replace an alive node
+# by draining/stopping the node being replaced before starting the replace
+alive_node_replace: false
+
 alive_nodes_listen_address: "{{ vars['ansible_'~scylla_nic].ipv4.address }}"
 alive_nodes_broadcast_address: "{{ vars['ansible_'~scylla_nic].ipv4.address }}"