parallelvirtualcluster
/
pvc
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 111 Wiki Activity

Lock primary_node key during primary switchover 

Also implements a looping to switch over the Patroni leader to ensure
this always follows the primary and clean up the code around here a bit.
This commit is contained in:
Joshua Boniface 2019-08-04 16:42:06 -04:00
parent 710d2cf9c2
commit a329376d33
1 changed files with 42 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
node-daemon/pvcd/NodeInstance.py
View File

@ -115,6 +115,7 @@ class NodeInstance(object):
# We're a coordinator so we care about networking # We're a coordinator so we care about networking
if data != self.router_state: if data != self.router_state:
self.router_state = data self.router_state = data
if self.config['enable_networking']:
if self.router_state == 'primary': if self.router_state == 'primary':
self.become_primary() self.become_primary()
else: else:
@ -259,7 +260,6 @@ class NodeInstance(object):
# Routing primary/secondary states # Routing primary/secondary states
def become_secondary(self): def become_secondary(self):
if self.config['enable_networking']:
self.logger.out('Setting router {} to secondary state'.format(self.name), state='i') self.logger.out('Setting router {} to secondary state'.format(self.name), state='i')
self.logger.out('Network list: {}'.format(', '.join(self.network_list)), state='i') self.logger.out('Network list: {}'.format(', '.join(self.network_list)), state='i')
time.sleep(2) time.sleep(2)
@ -273,8 +273,11 @@ class NodeInstance(object):
self.dns_aggregator.stop_aggregator() self.dns_aggregator.stop_aggregator()
def become_primary(self): def become_primary(self):
if self.config['enable_networking']: # Establish a lock
with zkhandler.writelock(self.zk_conn, '/primary_node'):
self.logger.out('Setting router {} to primary state'.format(self.name), state='i') self.logger.out('Setting router {} to primary state'.format(self.name), state='i')
# Create floating addresses
self.logger.out('Network list: {}'.format(', '.join(self.network_list)), state='i') self.logger.out('Network list: {}'.format(', '.join(self.network_list)), state='i')
self.createFloatingAddresses() self.createFloatingAddresses()
# Start up the gateways and DHCP servers # Start up the gateways and DHCP servers
@ -285,8 +288,10 @@ class NodeInstance(object):
self.logger.out('Starting PVC API client service', state='i') self.logger.out('Starting PVC API client service', state='i')
common.run_os_command("systemctl start pvc-api.service") common.run_os_command("systemctl start pvc-api.service")
time.sleep(1) time.sleep(1)
# Force Patroni to switch to the local instance
# Switch Patroni leader to the local instance
self.logger.out('Setting Patroni leader to this node', state='i') self.logger.out('Setting Patroni leader to this node', state='i')
while True:
retcode, stdout, stderr = common.run_os_command( retcode, stdout, stderr = common.run_os_command(
""" """
patronictl patronictl
@ -300,10 +305,13 @@ class NodeInstance(object):
) )
if stdout: if stdout:
self.logger.out('Successfully switched Patroni leader\n{}'.format(stdout), state='o') self.logger.out('Successfully switched Patroni leader\n{}'.format(stdout), state='o')
break
else: else:
self.logger.out('Failed to switch Patroni leader\n{}'.format(stderr), state='e') self.logger.out('Failed to switch Patroni leader; retrying\n{}'.format(stderr), state='e')
time.sleep(1) time.sleep(2)
# Start the DNS aggregator instance # Start the DNS aggregator instance
time.sleep(1)
self.dns_aggregator.start_aggregator() self.dns_aggregator.start_aggregator()
def createFloatingAddresses(self): def createFloatingAddresses(self):