From e34a7d4d2aa50d3e3abf9b1f39c04ec5811af4dc Mon Sep 17 00:00:00 2001
From: "Joshua M. Boniface" <joshua@boniface.me>
Date: Mon, 14 Jun 2021 12:52:43 -0400
Subject: [PATCH] Handle hot reloads properly

A hot reload isn't possible due to DataWatch and ChildrenWatch
constructs, so we instead need to terminate the daemon to "apply" the
schema update. Thus we use exit code 150 (Application defined in LSB)
and reorder some of the elements of the schema validation to ensure
things happen in the right order.
---
 node-daemon/pvcnoded/Daemon.py | 35 ++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/node-daemon/pvcnoded/Daemon.py b/node-daemon/pvcnoded/Daemon.py
index 8040712d..bed7169d 100644
--- a/node-daemon/pvcnoded/Daemon.py
+++ b/node-daemon/pvcnoded/Daemon.py
@@ -546,17 +546,11 @@ zkhandler.schema.load(node_schema_version)
 
 # Record the latest intalled schema version
 latest_schema_version = zkhandler.schema.find_latest()
+logger.out('Latest installed schema is {}'.format(latest_schema_version), state='i')
 zkhandler.write([
     (('node.data.latest_schema', myhostname), latest_schema_version)
 ])
 
-# Validate our schema against that version
-if not zkhandler.schema.validate(zkhandler, logger):
-    logger.out('Found schema violations, applying', state='i')
-    zkhandler.schema.apply(zkhandler)
-else:
-    logger.out('Schema successfully validated', state='o')
-
 
 # Watch for a global schema update and fire
 # This will only change by the API when triggered after seeing all nodes can update
@@ -601,7 +595,7 @@ def update_schema(new_schema_version, stat, event=''):
         time.sleep(1)
 
     # Update the local schema version
-    logger.out('Updating local schema', state='s')
+    logger.out('Updating node schema version', state='s')
     zkhandler.schema.load(new_schema_version)
     zkhandler.write([
         (('node.data.active_schema', myhostname), new_schema_version)
@@ -609,15 +603,24 @@ def update_schema(new_schema_version, stat, event=''):
     node_schema_version = new_schema_version
     time.sleep(1)
 
-    # Restart the update timer
-    if update_timer is not None:
-        update_timer = startKeepaliveTimer()
-
     # Restart the API daemons if applicable
+    logger.out('Restarting API services', state='s')
     if zkhandler.read('base.config.primary_node') == myhostname:
         common.run_os_command('systemctl start pvcapid.service')
         common.run_os_command('systemctl start pvcapid-worker.service')
 
+    # Terminate ourselves and restart with the new schema
+    # THIS IS SUBOPTIMAL, but since DataWatch and ChildrenWatch elements in Kazoo cannot
+    # be hot updated, a restart of the service is required for the change to be picked up.
+    logger.out('Restarting node daemon', state='s')
+    try:
+        zkhandler.disconnect()
+        del zkhandler
+    except Exception:
+        pass
+
+    os._exit(150)
+
 
 # If we are the last node to get a schema update, fire the master update
 if latest_schema_version > node_schema_version:
@@ -632,6 +635,14 @@ if latest_schema_version > node_schema_version:
             ('base.schema.version', latest_schema_version)
         ])
 
+# Validate our schema against the active version
+if not zkhandler.schema.validate(zkhandler, logger):
+    logger.out('Found schema violations, applying', state='i')
+    zkhandler.schema.apply(zkhandler)
+else:
+    logger.out('Schema successfully validated', state='o')
+
+
 ###############################################################################
 # PHASE 5 - Gracefully handle termination
 ###############################################################################