From 7e77752ce57591954bbb38fe6fd9a4c0db79a647 Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Wed, 7 Aug 2019 11:46:42 -0400 Subject: [PATCH] Add limit to Patroni switchover attempts --- node-daemon/pvcd/NodeInstance.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/node-daemon/pvcd/NodeInstance.py b/node-daemon/pvcd/NodeInstance.py index 6885ab97..bc5c15eb 100644 --- a/node-daemon/pvcd/NodeInstance.py +++ b/node-daemon/pvcd/NodeInstance.py @@ -291,6 +291,7 @@ class NodeInstance(object): # Switch Patroni leader to the local instance self.logger.out('Setting Patroni leader to this node', state='i') + tick = 1 while True: retcode, stdout, stderr = common.run_os_command( """ @@ -309,8 +310,12 @@ class NodeInstance(object): elif stderr == "Error: Switchover target and source are the same.\n": self.logger.out('Failed to switch Patroni leader to ourselves; this is fine\n{}'.format(stderr), state='w') break + elif tick >= 5: + self.logger.out('Failed to switch Patroni leader after 5 tries; aborting\n{}'.format(stderr), state='e') + break else: - self.logger.out('Failed to switch Patroni leader; retrying\n{}'.format(stderr), state='e') + self.logger.out('Failed to switch Patroni leader; retrying [{}/5]\n{}'.format(tick, stderr), state='e') + tick += 1 time.sleep(2) # Start the DNS aggregator instance