Reorder last keepalive during cleanup

Make sure the stopping of the keepalive timer and final keepalive update
are done as the last step before complete shutdown. The previous setup
could conceivably result in a node being fenced should the cleanup
operations take longer than ~45 seconds, for instance if primary node
switchover took too long or blocked, or log watchers failed to stop
quickly enough. Ensures that keepalives will continue to be run during
the shutdown process until the last possible moment.
This commit is contained in:
Joshua Boniface 2020-04-12 03:49:29 -04:00
parent b413e042a6
commit 611e0edd80
1 changed files with 11 additions and 11 deletions

View File

@ -557,17 +557,6 @@ def cleanup():
while this_node.flush_thread is not None: while this_node.flush_thread is not None:
time.sleep(0.5) time.sleep(0.5)
logger.out('Performing final keepalive update', state='s')
update_zookeeper()
# Stop keepalive thread
try:
stopKeepaliveTimer()
except NameError:
pass
except AttributeError:
pass
# Stop console logging on all VMs # Stop console logging on all VMs
logger.out('Stopping domain console watchers', state='s') logger.out('Stopping domain console watchers', state='s')
for domain in d_domain: for domain in d_domain:
@ -592,6 +581,17 @@ def cleanup():
except: except:
pass pass
# Stop keepalive thread
try:
stopKeepaliveTimer()
except NameError:
pass
except AttributeError:
pass
logger.out('Performing final keepalive update', state='s')
update_zookeeper()
# Set stop state in Zookeeper # Set stop state in Zookeeper
zkhandler.writedata(zk_conn, { '/nodes/{}/daemonstate'.format(myhostname): 'stop' }) zkhandler.writedata(zk_conn, { '/nodes/{}/daemonstate'.format(myhostname): 'stop' })