Lock primary_node key during primary switchover

Also implements a looping to switch over the Patroni leader to ensure
this always follows the primary and clean up the code around here a bit.

node-daemon/pvcd/NodeInstance.py

@@ -115,6 +115,7 @@ class NodeInstance(object):
                 # We're a coordinator so we care about networking
                 if data != self.router_state:
                     self.router_state = data
+                    if self.config['enable_networking']:
                         if self.router_state == 'primary':
                             self.become_primary()
                         else:
@@ -259,7 +260,6 @@ class NodeInstance(object):
 
     # Routing primary/secondary states
     def become_secondary(self):
-        if self.config['enable_networking']:
         self.logger.out('Setting router {} to secondary state'.format(self.name), state='i')
         self.logger.out('Network list: {}'.format(', '.join(self.network_list)), state='i')
         time.sleep(2)
@@ -273,8 +273,11 @@ class NodeInstance(object):
         self.dns_aggregator.stop_aggregator()
 
     def become_primary(self):
-        if self.config['enable_networking']:
+        # Establish a lock
+        with zkhandler.writelock(self.zk_conn, '/primary_node'):
             self.logger.out('Setting router {} to primary state'.format(self.name), state='i')
+
+            # Create floating addresses
             self.logger.out('Network list: {}'.format(', '.join(self.network_list)), state='i')
             self.createFloatingAddresses()
             # Start up the gateways and DHCP servers
@@ -285,8 +288,10 @@ class NodeInstance(object):
                 self.logger.out('Starting PVC API client service', state='i')
                 common.run_os_command("systemctl start pvc-api.service")
             time.sleep(1)
-            # Force Patroni to switch to the local instance
+
+            # Switch Patroni leader to the local instance
             self.logger.out('Setting Patroni leader to this node', state='i')
+            while True:
                 retcode, stdout, stderr = common.run_os_command(
                     """
                     patronictl
@@ -300,10 +305,13 @@ class NodeInstance(object):
                 )
                 if stdout:
                     self.logger.out('Successfully switched Patroni leader\n{}'.format(stdout), state='o')
+                    break
                 else:
-                self.logger.out('Failed to switch Patroni leader\n{}'.format(stderr), state='e')
-            time.sleep(1)
+                    self.logger.out('Failed to switch Patroni leader; retrying\n{}'.format(stderr), state='e')
+                    time.sleep(2)
+
             # Start the DNS aggregator instance
+            time.sleep(1)
             self.dns_aggregator.start_aggregator()
 
     def createFloatingAddresses(self):