Move Ceph cluster health reporting to plugin
Also removes several outputs from the normal keepalive that were superfluous/static so that the main output fits on one line.
This commit is contained in:
parent
2ee52e44d3
commit
d8f346abdd
|
|
@ -0,0 +1,126 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# ceph-cluster.py - PVC Monitoring example plugin for Ceph status
|
||||
# Part of the Parallel Virtual Cluster (PVC) system
|
||||
#
|
||||
# Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, version 3.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
#
|
||||
###############################################################################
|
||||
|
||||
# This script provides an example of a PVC monitoring plugin script. It will create
|
||||
# a simple plugin to check the Ceph cluster health for anomalies, and return a health
|
||||
# delta reflective of the overall Ceph status (HEALTH_WARN = 10, HEALTH_ERR = 50).
|
||||
|
||||
# This script can thus be used as an example or reference implementation of a
|
||||
# PVC monitoring pluginscript and expanded upon as required.
|
||||
|
||||
# A monitoring plugin script must implement the class "MonitoringPluginScript" which
|
||||
# extends "MonitoringPlugin", providing the 3 functions indicated. Detailed explanation
|
||||
# of the role of each function is provided in context of the example; see the other
|
||||
# examples for more potential uses.
|
||||
|
||||
# WARNING:
|
||||
#
|
||||
# This script will run in the context of the node daemon keepalives as root.
|
||||
# DO NOT install untrusted, unvetted plugins under any circumstances.
|
||||
|
||||
|
||||
# This import is always required here, as MonitoringPlugin is used by the
|
||||
# MonitoringPluginScript class
|
||||
from pvcnoded.objects.MonitoringInstance import MonitoringPlugin
|
||||
|
||||
|
||||
# A monitoring plugin script must always expose its nice name, which must be identical to
|
||||
# the file name
|
||||
PLUGIN_NAME = "ceph-cluster"
|
||||
|
||||
|
||||
# The MonitoringPluginScript class must be named as such, and extend MonitoringPlugin.
|
||||
class MonitoringPluginScript(MonitoringPlugin):
|
||||
def setup(self):
|
||||
"""
|
||||
setup(): Perform special setup steps during node daemon startup
|
||||
|
||||
This step is optional and should be used sparingly.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
run(): Perform the check actions and return a PluginResult object
|
||||
"""
|
||||
|
||||
# Run any imports first
|
||||
from rados import Rados
|
||||
from json import loads, dumps
|
||||
|
||||
# Connect to the Ceph cluster
|
||||
try:
|
||||
ceph_conn = Rados(
|
||||
conffile=self.config["ceph_config_file"],
|
||||
conf=dict(keyring=self.config["ceph_admin_keyring"]),
|
||||
)
|
||||
ceph_conn.connect(timeout=1)
|
||||
except Exception as e:
|
||||
self.log(f"Failed to connect to Ceph cluster: {e}", state="e")
|
||||
return self.plugin_result
|
||||
|
||||
# Get the Ceph cluster health
|
||||
try:
|
||||
health_status = loads(
|
||||
ceph_conn.mon_command(dumps({"prefix": "health", "format": "json"}), b"", timeout=1)[1]
|
||||
)
|
||||
ceph_health = health_status["status"]
|
||||
except Exception as e:
|
||||
self.log(f"Failed to get health data from Ceph cluster: {e}", state="e")
|
||||
return self.plugin_result
|
||||
finally:
|
||||
ceph_conn.shutdown()
|
||||
|
||||
# Get a list of error entries in the health status output
|
||||
error_entries = health_status["checks"].keys()
|
||||
|
||||
# Set the health delta based on the errors presented
|
||||
if ceph_health == "HEALTH_ERR":
|
||||
health_delta = 50
|
||||
message = f"Ceph cluster in ERROR state: {', '.join(error_entries)}"
|
||||
elif ceph_health == "HEALTH_WARN":
|
||||
health_delta = 10
|
||||
message = f"Ceph cluster in WARNING state: {', '.join(error_entries)}"
|
||||
else:
|
||||
health_delta = 0
|
||||
message = "Ceph cluster in OK state"
|
||||
|
||||
# Set the health delta in our local PluginResult object
|
||||
self.plugin_result.set_health_delta(health_delta)
|
||||
|
||||
# Set the message in our local PluginResult object
|
||||
self.plugin_result.set_message(message)
|
||||
|
||||
# Set the detailed data in our local PluginResult object
|
||||
self.plugin_result.set_data(dumps(health_status))
|
||||
|
||||
# Return our local PluginResult object
|
||||
return self.plugin_result
|
||||
|
||||
def cleanup(self):
|
||||
"""
|
||||
cleanup(): Perform special cleanup steps during node daemon termination
|
||||
|
||||
This step is optional and should be used sparingly.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
|
@ -129,6 +129,9 @@ class MonitoringPlugin(object):
|
|||
self.plugin_name,
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
return self.plugin_name
|
||||
|
||||
#
|
||||
# Helper functions; exposed to child MonitoringPluginScript instances
|
||||
#
|
||||
|
|
@ -309,15 +312,15 @@ class MonitoringInstance(object):
|
|||
time_delta = time_end - time_start
|
||||
runtime = "{:0.02f}".format(time_delta.total_seconds())
|
||||
result.set_runtime(runtime)
|
||||
self.logger.out(
|
||||
result.message, state="t", prefix=f"{plugin.plugin_name} ({runtime}s)"
|
||||
)
|
||||
result.to_zookeeper()
|
||||
return result
|
||||
|
||||
def run_plugins(self):
|
||||
total_health = 100
|
||||
self.logger.out("Running monitoring plugins:", state="t")
|
||||
self.logger.out(
|
||||
f"Running monitoring plugins: {', '.join([x.plugin_name for x in self.all_plugins])}",
|
||||
state="t",
|
||||
)
|
||||
plugin_results = list()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=99) as executor:
|
||||
to_future_plugin_results = {
|
||||
|
|
@ -327,7 +330,12 @@ class MonitoringInstance(object):
|
|||
for future in concurrent.futures.as_completed(to_future_plugin_results):
|
||||
plugin_results.append(future.result())
|
||||
|
||||
for result in plugin_results:
|
||||
for result in sorted(plugin_results, key=lambda x: x.plugin_name):
|
||||
self.logger.out(
|
||||
result.message,
|
||||
state="t",
|
||||
prefix=f"{result.plugin_name} ({result.runtime}s)",
|
||||
)
|
||||
if result is not None:
|
||||
total_health -= result.health_delta
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue