Avoid running fault updates in maintenance mode

When the cluster is in maintenance mode, all faults should be ignored.
This commit is contained in:
Joshua Boniface 2023-12-01 16:08:13 -05:00
parent 3dc48c1783
commit 4c3f235e05
1 changed files with 14 additions and 5 deletions

View File

@ -512,7 +512,7 @@ class MonitoringInstance(object):
# If a fault already exists with this ID, just update the time # If a fault already exists with this ID, just update the time
if not self.zkhandler.exists("base.faults"): if not self.zkhandler.exists("base.faults"):
self.logger.out( self.logger.out(
"Skipping fault reporting for {fault_id} due to missing Zookeeper schemas", f"Skipping fault reporting for {fault_id} due to missing Zookeeper schemas",
state="w", state="w",
) )
return return
@ -522,7 +522,20 @@ class MonitoringInstance(object):
self.logger.out( self.logger.out(
f"Updating fault {fault_id}: {fault_message} @ {fault_time}", state="i" f"Updating fault {fault_id}: {fault_message} @ {fault_time}", state="i"
) )
else:
self.logger.out(
f"Generating fault {fault_id}: {fault_message} @ {fault_time}",
state="i",
)
if self.zkhandler.read("base.config.maintenance") == "true":
self.logger.out(
f"Skipping fault reporting for {fault_id} due to maintenance mode",
state="w",
)
return
if fault_id in existing_faults:
self.zkhandler.write( self.zkhandler.write(
[ [
(("faults.last_time", fault_id), str(fault_time)), (("faults.last_time", fault_id), str(fault_time)),
@ -530,10 +543,6 @@ class MonitoringInstance(object):
) )
# Otherwise, generate a new fault event # Otherwise, generate a new fault event
else: else:
self.logger.out(
f"Generating fault {fault_id}: {fault_message} @ {fault_time}",
state="i",
)
self.zkhandler.write( self.zkhandler.write(
[ [
(("faults.id", fault_id), ""), (("faults.id", fault_id), ""),