Implement wait for node coordinator transition

References #72

api-daemon/pvcapid/flaskapi.py

@@ -618,7 +618,8 @@ class API_Node_CoordinatorState(Resource):
         return api_helper.node_coordinator_state(node)
 
     @RequestParser([
-        { 'name': 'state', 'choices': ('primary', 'secondary'), 'helptext': "A valid state must be specified", 'required': True }
+        { 'name': 'state', 'choices': ('primary', 'secondary'), 'helptext': "A valid state must be specified", 'required': True },
+        { 'name': 'wait' }
     ])
     @Authenticator
     def post(self, node, reqargs):
@@ -636,6 +637,10 @@ class API_Node_CoordinatorState(Resource):
             enum:
               - primary
               - secondary
+          - in: query
+            name: wait
+            type: boolean
+            description: Whether to block waiting for the full state transition
         responses:
           200:
             description: OK
@@ -648,10 +653,11 @@ class API_Node_CoordinatorState(Resource):
               type: object
               id: Message
         """
+        wait = bool(strtobool(reqargs.get('wait', 'false')))
         if reqargs['state'] == 'primary':
-            return api_helper.node_primary(node)
+            return api_helper.node_primary(node, wait)
         if reqargs['state'] == 'secondary':
-            return api_helper.node_secondary(node)
+            return api_helper.node_secondary(node, wait)
         abort(400)
 api.add_resource(API_Node_CoordinatorState, '/node/<node>/coordinator-state')
 

api-daemon/pvcapid/helper.py

@@ -213,12 +213,12 @@ def node_domain_state(node):
 
     return retdata, retcode
 
-def node_secondary(node):
+def node_secondary(node, wait):
     """
     Take NODE out of primary router mode.
     """
     zk_conn = pvc_common.startZKConnection(config['coordinators']) 
-    retflag, retdata = pvc_node.secondary_node(zk_conn, node)
+    retflag, retdata = pvc_node.secondary_node(zk_conn, node, wait)
     pvc_common.stopZKConnection(zk_conn)
 
     if retflag:
@@ -231,12 +231,12 @@ def node_secondary(node):
     }
     return output, retcode
 
-def node_primary(node):
+def node_primary(node, wait):
     """
     Set NODE to primary router mode.
     """
     zk_conn = pvc_common.startZKConnection(config['coordinators']) 
-    retflag, retdata = pvc_node.primary_node(zk_conn, node)
+    retflag, retdata = pvc_node.primary_node(zk_conn, node, wait)
     pvc_common.stopZKConnection(zk_conn)
 
     if retflag:

client-cli/cli_lib/node.py

@@ -26,7 +26,7 @@ from cli_lib.common import call_api
 #
 # Primary functions
 #
-def node_coordinator_state(config, node, action):
+def node_coordinator_state(config, node, action, wait):
     """
     Set node coordinator state state (primary/secondary)
 
@@ -35,7 +35,8 @@ def node_coordinator_state(config, node, action):
     API schema: {"message": "{data}"}
     """
     params={
-        'state': action
+        'state': action,
+        'wait': str(wait).lower()
     }
     response = call_api(config, 'post', '/node/{node}/coordinator-state'.format(node=node), params=params)
 

client-cli/pvc.py

@@ -348,7 +348,11 @@ def cli_node():
 @click.argument(
     'node'
 )
-def node_secondary(node):
+@click.option(
+    '-w', '--wait', 'wait', is_flag=True, default=False,
+    help='Wait for transition to complete before returning.'
+)
+def node_secondary(node, wait):
     """
     Take NODE out of primary router mode.
     """
@@ -360,7 +364,7 @@ def node_secondary(node):
         click.echo("      node returns to primary state.")
         click.echo()
 
-    retcode, retmsg = pvc_node.node_coordinator_state(config, node, 'secondary')
+    retcode, retmsg = pvc_node.node_coordinator_state(config, node, 'secondary', wait)
     cleanup(retcode, retmsg)
 
 ###############################################################################
@@ -370,7 +374,11 @@ def node_secondary(node):
 @click.argument(
     'node'
 )
-def node_primary(node):
+@click.option(
+    '-w', '--wait', 'wait', is_flag=True, default=False,
+    help='Wait for transition to complete before returning.'
+)
+def node_primary(node, wait):
     """
     Put NODE into primary router mode.
     """
@@ -382,7 +390,7 @@ def node_primary(node):
         click.echo("      node returns to primary state.")
         click.echo()
 
-    retcode, retmsg = pvc_node.node_coordinator_state(config, node, 'primary')
+    retcode, retmsg = pvc_node.node_coordinator_state(config, node, 'primary', wait)
     cleanup(retcode, retmsg)
 
 ###############################################################################

daemon-common/node.py

@@ -89,7 +89,7 @@ def getNodeInformation(zk_conn, node_name):
 #
 # Direct Functions
 #
-def secondary_node(zk_conn, node):
+def secondary_node(zk_conn, node, wait=False):
     # Verify node is valid
     if not common.verifyNode(zk_conn, node):
         return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
@@ -111,12 +111,21 @@ def secondary_node(zk_conn, node):
         zkhandler.writedata(zk_conn, {
             '/primary_node': 'none'
         })
+
+        if wait:
+            # Wait 1 second for lock acquisition
+            time.sleep(1)
+            # Set up and block on a write lock of /primary_node
+            transition_lock = zkhandler.writelock(zk_conn, '/primary_node')
+            transition_lock.acquire()
+            transition_lock.release()
+            retmsg = 'Set node {} in secondary router mode.'.format(node)
     else:
         return False, 'Node "{}" is already in secondary router mode.'.format(node)
 
     return True, retmsg
 
-def primary_node(zk_conn, node):
+def primary_node(zk_conn, node, wait=False):
     # Verify node is valid
     if not common.verifyNode(zk_conn, node):
         return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
@@ -138,6 +147,15 @@ def primary_node(zk_conn, node):
         zkhandler.writedata(zk_conn, {
             '/primary_node': node
         })
+
+        if wait:
+            # Wait 1 second for lock acquisition
+            time.sleep(1)
+            # Set up and block on a write lock of /primary_node
+            transition_lock = zkhandler.writelock(zk_conn, '/primary_node')
+            transition_lock.acquire()
+            transition_lock.release()
+            retmsg = 'Set node {} in primary router mode.'.format(node)
     else:
         return False, 'Node "{}" is already in primary router mode.'.format(node)