From b5f996febdbea8b77af4ef2687b9d9ead06ee064 Mon Sep 17 00:00:00 2001
From: "Joshua M. Boniface" <joshua@boniface.me>
Date: Fri, 18 Aug 2023 10:56:03 -0400
Subject: [PATCH] Fix bugs for node flush for stop/shutdown/restart

Previously VMs in stop/shutdown/restart states wouldn't be properly
handled during a node flush. This fixes the bugs and ensures that the
transient VM states (shutdown/restart) are completed before proceeding,
and then avoids setting a stopped/shutdown VM to shutdown/auotstart.
---
 node-daemon/pvcnoded/objects/NodeInstance.py | 29 +++++++++++++++-----
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/node-daemon/pvcnoded/objects/NodeInstance.py b/node-daemon/pvcnoded/objects/NodeInstance.py
index dc2ea2ff..a59770e0 100644
--- a/node-daemon/pvcnoded/objects/NodeInstance.py
+++ b/node-daemon/pvcnoded/objects/NodeInstance.py
@@ -790,6 +790,19 @@ class NodeInstance(object):
                 self.flush_stopper = False
                 return
 
+            # Wait for a VM in "restart" or "shutdown" state to complete transition
+            while self.zkhandler.read(("domain.state", dom_uuid)) in [
+                "restart",
+                "shutdown",
+            ]:
+                self.logger.out(
+                    'Waiting 2s for VM state change completion for VM "{}"'.format(
+                        dom_uuid
+                    ),
+                    state="i",
+                )
+                time.sleep(2)
+
             self.logger.out(
                 'Selecting target to migrate VM "{}"'.format(dom_uuid), state="i"
             )
@@ -806,17 +819,19 @@ class NodeInstance(object):
 
             if target_node is None:
                 self.logger.out(
-                    'Failed to find migration target for VM "{}"; shutting down and setting autostart flag'.format(
+                    'Failed to find migration target for running VM "{}"; shutting down and setting autostart flag'.format(
                         dom_uuid
                     ),
                     state="e",
                 )
-                self.zkhandler.write(
-                    [
-                        (("domain.state", dom_uuid), "shutdown"),
-                        (("domain.meta.autostart", dom_uuid), "True"),
-                    ]
-                )
+
+                if self.zkhandler.read(("domain.state", dom_uuid)) in ["start"]:
+                    self.zkhandler.write(
+                        [
+                            (("domain.state", dom_uuid), "shutdown"),
+                            (("domain.meta.autostart", dom_uuid), "True"),
+                        ]
+                    )
             else:
                 self.logger.out(
                     'Migrating VM "{}" to node "{}"'.format(dom_uuid, target_node),