198 lines
7.9 KiB
Python
198 lines
7.9 KiB
Python
#!/usr/bin/env python3
|
|
|
|
# nics.py - PVC Monitoring example plugin for NIC interfaces
|
|
# Part of the Parallel Virtual Cluster (PVC) system
|
|
#
|
|
# Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, version 3.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
#
|
|
###############################################################################
|
|
|
|
# This script provides an example of a PVC monitoring plugin script. It will create
|
|
# a simple plugin to check the network interfaces of the host, specifically for speed
|
|
# and 802.3ad status (if applicable).
|
|
|
|
# This script can thus be used as an example or reference implementation of a
|
|
# PVC monitoring pluginscript and expanded upon as required.
|
|
|
|
# A monitoring plugin script must implement the class "MonitoringPluginScript" which
|
|
# extends "MonitoringPlugin", providing the 3 functions indicated. Detailed explanation
|
|
# of the role of each function is provided in context of the example; see the other
|
|
# examples for more potential uses.
|
|
|
|
# WARNING:
|
|
#
|
|
# This script will run in the context of the node daemon keepalives as root.
|
|
# DO NOT install untrusted, unvetted plugins under any circumstances.
|
|
|
|
|
|
# This import is always required here, as MonitoringPlugin is used by the
|
|
# MonitoringPluginScript class
|
|
from pvcnoded.objects.MonitoringInstance import MonitoringPlugin
|
|
|
|
|
|
# A monitoring plugin script must always expose its nice name, which must be identical to
|
|
# the file name
|
|
PLUGIN_NAME = "nics"
|
|
|
|
# Set a minimum link speed variable used below
|
|
# For PVC at least 10 Gbps is required for proper operation of a cluster
|
|
MINIMUM_LINKSPEED = 10000
|
|
|
|
|
|
# The MonitoringPluginScript class must be named as such, and extend MonitoringPlugin.
|
|
class MonitoringPluginScript(MonitoringPlugin):
|
|
def setup(self):
|
|
"""
|
|
setup(): Perform special setup steps during node daemon startup
|
|
|
|
This step is optional and should be used sparingly.
|
|
"""
|
|
|
|
pass
|
|
|
|
def run(self):
|
|
"""
|
|
run(): Perform the check actions and return a PluginResult object
|
|
"""
|
|
|
|
# Run any imports first
|
|
import daemon_lib.common as common
|
|
from re import match, search, findall
|
|
|
|
messages = list()
|
|
health_delta = 0
|
|
|
|
# Get a list of the various underlying devices
|
|
_core_nics = set()
|
|
|
|
for dev in [
|
|
self.config['bridge_dev'],
|
|
self.config['upstream_dev'],
|
|
self.config['cluster_dev'],
|
|
self.config['storage_dev'],
|
|
]:
|
|
with open(f'/sys/class/net/{dev}/uevent', 'r') as uevent:
|
|
_devtype = uevent.readlines()[0].split('=')[-1].strip()
|
|
|
|
if _devtype == 'vlan':
|
|
with open(f"/proc/net/vlan/{dev}") as devfh:
|
|
vlan_info = devfh.read().split('\n')
|
|
for line in vlan_info:
|
|
if match(r'^Device:', line):
|
|
dev = line.split()[-1]
|
|
|
|
_core_nics.add(dev)
|
|
|
|
core_nics = sorted(list(_core_nics))
|
|
|
|
for dev in core_nics:
|
|
with open(f'/sys/class/net/{dev}/uevent', 'r') as uevent:
|
|
_devtype = uevent.readlines()[0].split('=')[-1].strip()
|
|
|
|
if _devtype == "bond":
|
|
syspath = f"/proc/net/bonding/{dev}"
|
|
|
|
with open(syspath) as devfh:
|
|
bonding_stats = devfh.read()
|
|
|
|
_, _mode, _info, *_slaves = bonding_stats.split('\n\n')
|
|
|
|
slave_interfaces = list()
|
|
for slavedev in _slaves:
|
|
lines = slavedev.split('\n')
|
|
for line in lines:
|
|
if match(r'^Slave Interface:', line):
|
|
interface_name = line.split()[-1]
|
|
if match(r'^MII Status:', line):
|
|
interface_status = line.split()[-1]
|
|
if match(r'^Speed:', line):
|
|
try:
|
|
interface_speed_mbps = int(line.split()[-2])
|
|
except Exception:
|
|
interface_speed_mbps = 0
|
|
if match(r'^Duplex:', line):
|
|
interface_duplex = line.split()[-1]
|
|
slave_interfaces.append((interface_name, interface_status, interface_speed_mbps, interface_duplex))
|
|
|
|
# Ensure at least 2 slave interfaces are up
|
|
slave_interface_up_count = 0
|
|
for slave_interface in slave_interfaces:
|
|
if slave_interface[1] == 'up':
|
|
slave_interface_up_count += 1
|
|
if slave_interface_up_count < 2:
|
|
messages.append(f"{dev} DEGRADED with {slave_interface_up_count} active slaves")
|
|
health_delta += 10
|
|
else:
|
|
messages.append(f"{dev} OK with {slave_interface_up_count} active slaves")
|
|
|
|
# Get ethtool supported speeds for slave interfaces
|
|
supported_link_speeds = set()
|
|
for slave_interface in slave_interfaces:
|
|
slave_dev = slave_interface[0]
|
|
_, ethtool_stdout, _ = common.run_os_command(f"ethtool {slave_dev}")
|
|
in_modes = False
|
|
for line in ethtool_stdout.split('\n'):
|
|
if search('Supported link modes:', line):
|
|
in_modes = True
|
|
if search('Supported pause frame use:', line):
|
|
in_modes = False
|
|
break
|
|
if in_modes:
|
|
speed = int(findall(r'\d+', line.split()[-1])[0])
|
|
supported_link_speeds.add(speed)
|
|
else:
|
|
# Get ethtool supported speeds for interface
|
|
supported_link_speeds = set()
|
|
_, ethtool_stdout, _ = common.run_os_command(f"ethtool {dev}")
|
|
in_modes = False
|
|
for line in ethtool_stdout.split('\n'):
|
|
if search('Supported link modes:', line):
|
|
in_modes = True
|
|
if search('Supported pause frame use:', line):
|
|
in_modes = False
|
|
break
|
|
if in_modes:
|
|
speed = int(line.split()[-1].replace('baseT', '').split('/')[0])
|
|
supported_link_speeds.add(speed)
|
|
|
|
max_supported_link_speed = sorted(list(supported_link_speeds))[-1]
|
|
|
|
# Ensure interface is running at MINIMUM_LINKSPEED
|
|
with open(f"/sys/class/net/{dev}/speed") as devfh:
|
|
dev_speed = int(devfh.read())
|
|
if dev_speed < max_supported_link_speed:
|
|
messages.append(f"{dev} DEGRADED at {dev_speed} Mbps")
|
|
health_delta += 10
|
|
else:
|
|
messages.append(f"{dev} OK at {dev_speed} Mbps")
|
|
|
|
# Set the health delta in our local PluginResult object
|
|
self.plugin_result.set_health_delta(health_delta)
|
|
|
|
# Set the message in our local PluginResult object
|
|
self.plugin_result.set_message(', '.join(messages))
|
|
|
|
# Return our local PluginResult object
|
|
return self.plugin_result
|
|
|
|
def cleanup(self):
|
|
"""
|
|
cleanup(): Perform special cleanup steps during node daemon termination
|
|
|
|
This step is optional and should be used sparingly.
|
|
"""
|
|
|
|
pass
|