Avoid waits during node secondary

Waiting for the daemons to stop took too much time on some nodes and
could throw off the lockstep. Instead, leverage background=True to run
the systemctl os_commands in the background (when they complete is
irrelevant), stop the Metadata API first, and don't delay during its
stop at all.

node-daemon/pvcnoded/objects/MetadataAPIInstance.py

@@ -22,7 +22,6 @@
 import gevent.pywsgi
 import flask
 import sys
-import time
 import psycopg2
 
 from threading import Thread
@@ -123,9 +122,7 @@ class MetadataAPIInstance(object):
         self.logger.out("Stopping Metadata API at 169.254.169.254:80", state="i")
         try:
             self.md_http_server.stop()
-            time.sleep(0.1)
             self.md_http_server.close()
-            time.sleep(0.1)
             self.md_http_server = None
             self.logger.out("Successfully stopped Metadata API", state="o")
         except Exception as e:

node-daemon/pvcnoded/objects/NodeInstance.py

@@ -670,13 +670,13 @@ class NodeInstance(object):
         self.zkhandler.write([("base.config.primary_node.sync_lock", "")])
         lock.release()
         self.logger.out("Released write lock for synchronization phase B", state="o")
-        # 3. Stop client API
+        # 3. Stop metadata API
+        self.metadata_api.stop()
+        # 4. Stop client API
         if self.config["enable_api"]:
             self.logger.out("Stopping PVC API client service", state="i")
-            common.run_os_command("systemctl stop pvcapid.service")
-            common.run_os_command("systemctl disable pvcapid.service")
-        # 4. Stop metadata API
-        self.metadata_api.stop()
+            common.run_os_command("systemctl stop pvcapid.service", background=True)
+            common.run_os_command("systemctl disable pvcapid.service", background=True)
         time.sleep(0.1)  # Time fir new writer to acquire the lock
 
         # Synchronize nodes C (I am reader)