From d69c4b9662ea129d5e657d0939be3b0da3f2c7fd Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Thu, 23 Feb 2023 22:28:34 -0500 Subject: [PATCH] Correct error handling if monitoring plugins fail --- .../pvcnoded/objects/MonitoringInstance.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/node-daemon/pvcnoded/objects/MonitoringInstance.py b/node-daemon/pvcnoded/objects/MonitoringInstance.py index 58cde66b..3c458379 100644 --- a/node-daemon/pvcnoded/objects/MonitoringInstance.py +++ b/node-daemon/pvcnoded/objects/MonitoringInstance.py @@ -332,7 +332,14 @@ class MonitoringInstance(object): def run_plugin(self, plugin): time_start = datetime.now() - result = plugin.run() + try: + result = plugin.run() + except Exception as e: + self.logger.out( + f"Monitoring plugin {plugin.plugin_name} failed: {e}", + state="e", + ) + return None time_end = datetime.now() time_delta = time_end - time_start runtime = "{:0.02f}".format(time_delta.total_seconds()) @@ -357,13 +364,13 @@ class MonitoringInstance(object): plugin_results.append(future.result()) for result in sorted(plugin_results, key=lambda x: x.plugin_name): - if self.config["log_keepalive_plugin_details"]: - self.logger.out( - result.message + f" [-{result.health_delta}]", - state="t", - prefix=f"{result.plugin_name} ({result.runtime}s)", - ) if result is not None: + if self.config["log_keepalive_plugin_details"]: + self.logger.out( + result.message + f" [-{result.health_delta}]", + state="t", + prefix=f"{result.plugin_name} ({result.runtime}s)", + ) total_health -= result.health_delta if total_health < 0: