From 4d786c11e375a7059eaf91328104039cbb7f6f4a Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Mon, 13 Feb 2023 12:13:56 -0500 Subject: [PATCH] Move Ceph cluster health reporting to plugin Also removes several outputs from the normal keepalive that were superfluous/static so that the main output fits on one line. --- node-daemon/plugins/ceph-cluster | 126 ++++++++++++++++++ .../pvcnoded/objects/MonitoringInstance.py | 18 ++- 2 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 node-daemon/plugins/ceph-cluster diff --git a/node-daemon/plugins/ceph-cluster b/node-daemon/plugins/ceph-cluster new file mode 100644 index 00000000..48788925 --- /dev/null +++ b/node-daemon/plugins/ceph-cluster @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +# ceph-cluster.py - PVC Monitoring example plugin for Ceph status +# Part of the Parallel Virtual Cluster (PVC) system +# +# Copyright (C) 2018-2022 Joshua M. Boniface +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### + +# This script provides an example of a PVC monitoring plugin script. It will create +# a simple plugin to check the Ceph cluster health for anomalies, and return a health +# delta reflective of the overall Ceph status (HEALTH_WARN = 10, HEALTH_ERR = 50). + +# This script can thus be used as an example or reference implementation of a +# PVC monitoring pluginscript and expanded upon as required. + +# A monitoring plugin script must implement the class "MonitoringPluginScript" which +# extends "MonitoringPlugin", providing the 3 functions indicated. Detailed explanation +# of the role of each function is provided in context of the example; see the other +# examples for more potential uses. + +# WARNING: +# +# This script will run in the context of the node daemon keepalives as root. +# DO NOT install untrusted, unvetted plugins under any circumstances. + + +# This import is always required here, as MonitoringPlugin is used by the +# MonitoringPluginScript class +from pvcnoded.objects.MonitoringInstance import MonitoringPlugin + + +# A monitoring plugin script must always expose its nice name, which must be identical to +# the file name +PLUGIN_NAME = "ceph-cluster" + + +# The MonitoringPluginScript class must be named as such, and extend MonitoringPlugin. +class MonitoringPluginScript(MonitoringPlugin): + def setup(self): + """ + setup(): Perform special setup steps during node daemon startup + + This step is optional and should be used sparingly. + """ + + pass + + def run(self): + """ + run(): Perform the check actions and return a PluginResult object + """ + + # Run any imports first + from rados import Rados + from json import loads, dumps + + # Connect to the Ceph cluster + try: + ceph_conn = Rados( + conffile=self.config["ceph_config_file"], + conf=dict(keyring=self.config["ceph_admin_keyring"]), + ) + ceph_conn.connect(timeout=1) + except Exception as e: + self.log(f"Failed to connect to Ceph cluster: {e}", state="e") + return self.plugin_result + + # Get the Ceph cluster health + try: + health_status = loads( + ceph_conn.mon_command(dumps({"prefix": "health", "format": "json"}), b"", timeout=1)[1] + ) + ceph_health = health_status["status"] + except Exception as e: + self.log(f"Failed to get health data from Ceph cluster: {e}", state="e") + return self.plugin_result + finally: + ceph_conn.shutdown() + + # Get a list of error entries in the health status output + error_entries = health_status["checks"].keys() + + # Set the health delta based on the errors presented + if ceph_health == "HEALTH_ERR": + health_delta = 50 + message = f"Ceph cluster in ERROR state: {', '.join(error_entries)}" + elif ceph_health == "HEALTH_WARN": + health_delta = 10 + message = f"Ceph cluster in WARNING state: {', '.join(error_entries)}" + else: + health_delta = 0 + message = "Ceph cluster in OK state" + + # Set the health delta in our local PluginResult object + self.plugin_result.set_health_delta(health_delta) + + # Set the message in our local PluginResult object + self.plugin_result.set_message(message) + + # Set the detailed data in our local PluginResult object + self.plugin_result.set_data(dumps(health_status)) + + # Return our local PluginResult object + return self.plugin_result + + def cleanup(self): + """ + cleanup(): Perform special cleanup steps during node daemon termination + + This step is optional and should be used sparingly. + """ + + pass diff --git a/node-daemon/pvcnoded/objects/MonitoringInstance.py b/node-daemon/pvcnoded/objects/MonitoringInstance.py index 7b79e1fc..c4d0f3f2 100644 --- a/node-daemon/pvcnoded/objects/MonitoringInstance.py +++ b/node-daemon/pvcnoded/objects/MonitoringInstance.py @@ -129,6 +129,9 @@ class MonitoringPlugin(object): self.plugin_name, ) + def __str__(self): + return self.plugin_name + # # Helper functions; exposed to child MonitoringPluginScript instances # @@ -309,15 +312,15 @@ class MonitoringInstance(object): time_delta = time_end - time_start runtime = "{:0.02f}".format(time_delta.total_seconds()) result.set_runtime(runtime) - self.logger.out( - result.message, state="t", prefix=f"{plugin.plugin_name} ({runtime}s)" - ) result.to_zookeeper() return result def run_plugins(self): total_health = 100 - self.logger.out("Running monitoring plugins:", state="t") + self.logger.out( + f"Running monitoring plugins: {', '.join([x.plugin_name for x in self.all_plugins])}", + state="t", + ) plugin_results = list() with concurrent.futures.ThreadPoolExecutor(max_workers=99) as executor: to_future_plugin_results = { @@ -327,7 +330,12 @@ class MonitoringInstance(object): for future in concurrent.futures.as_completed(to_future_plugin_results): plugin_results.append(future.result()) - for result in plugin_results: + for result in sorted(plugin_results, key=lambda x: x.plugin_name): + self.logger.out( + result.message, + state="t", + prefix=f"{result.plugin_name} ({result.runtime}s)", + ) if result is not None: total_health -= result.health_delta