Fix several bugs and optimize output

Also removes several outputs from the normal keepalive that were
superfluous/static so that the main output fits on one line.

client-cli/pvc/cli_lib/node.py

@@ -215,6 +215,19 @@ def node_list(
 # Output display functions
 #
 def getOutputColours(node_information):
+    node_health = node_information.get("health", "N/A")
+    if isinstance(node_health, int):
+        if node_health <= 50:
+            health_colour = ansiprint.red()
+        elif node_health <= 90:
+            health_colour = ansiprint.yellow()
+        elif node_health <= 100:
+            health_colour = ansiprint.green()
+        else:
+            health_colour = ansiprint.blue()
+    else:
+        health_colour = ansiprint.blue()
+
     if node_information["daemon_state"] == "run":
         daemon_state_colour = ansiprint.green()
     elif node_information["daemon_state"] == "stop":
@@ -251,6 +264,7 @@ def getOutputColours(node_information):
         mem_provisioned_colour = ""
 
     return (
+        health_colour,
         daemon_state_colour,
         coordinator_state_colour,
         domain_state_colour,
@@ -261,6 +275,7 @@ def getOutputColours(node_information):
 
 def format_info(node_information, long_output):
     (
+        health_colour,
         daemon_state_colour,
         coordinator_state_colour,
         domain_state_colour,
@@ -273,14 +288,56 @@ def format_info(node_information, long_output):
     # Basic information
     ainformation.append(
         "{}Name:{}                  {}".format(
-            ansiprint.purple(), ansiprint.end(), node_information["name"]
+            ansiprint.purple(),
+            ansiprint.end(),
+            node_information["name"],
         )
     )
     ainformation.append(
         "{}PVC Version:{}           {}".format(
-            ansiprint.purple(), ansiprint.end(), node_information["pvc_version"]
+            ansiprint.purple(),
+            ansiprint.end(),
+            node_information["pvc_version"],
         )
     )
+
+    node_health = node_information.get("health", "N/A")
+    if isinstance(node_health, int):
+        node_health_text = f"{node_health}%"
+    else:
+        node_health_text = node_health
+    ainformation.append(
+        "{}Health:{}                {}{}{}".format(
+            ansiprint.purple(),
+            ansiprint.end(),
+            health_colour,
+            node_health_text,
+            ansiprint.end(),
+        )
+    )
+
+    node_health_details = node_information.get("health_details", [])
+    if long_output:
+        node_health_messages = "\n                       ".join(
+            [f"{plugin['name']}: {plugin['message']}" for plugin in node_health_details]
+        )
+    else:
+        node_health_messages = "\n                       ".join(
+            [
+                f"{plugin['name']}: {plugin['message']}"
+                for plugin in node_health_details
+                if int(plugin.get("health_delta", 0)) > 0
+            ]
+        )
+
+    if len(node_health_messages) > 0:
+        ainformation.append(
+            "{}Health Plugin Details:{} {}".format(
+                ansiprint.purple(), ansiprint.end(), node_health_messages
+            )
+        )
+    ainformation.append("")
+
     ainformation.append(
         "{}Daemon State:{}          {}{}{}".format(
             ansiprint.purple(),
@@ -308,11 +365,6 @@ def format_info(node_information, long_output):
             ansiprint.end(),
         )
     )
-    ainformation.append(
-        "{}Active VM Count:{}       {}".format(
-            ansiprint.purple(), ansiprint.end(), node_information["domains_count"]
-        )
-    )
     if long_output:
         ainformation.append("")
         ainformation.append(
@@ -331,6 +383,11 @@ def format_info(node_information, long_output):
             )
         )
     ainformation.append("")
+    ainformation.append(
+        "{}Active VM Count:{}       {}".format(
+            ansiprint.purple(), ansiprint.end(), node_information["domains_count"]
+        )
+    )
     ainformation.append(
         "{}Host CPUs:{}             {}".format(
             ansiprint.purple(), ansiprint.end(), node_information["vcpu"]["total"]
@@ -397,6 +454,7 @@ def format_list(node_list, raw):
     # Determine optimal column widths
     node_name_length = 5
     pvc_version_length = 8
+    health_length = 7
     daemon_state_length = 7
     coordinator_state_length = 12
     domain_state_length = 7
@@ -417,6 +475,15 @@ def format_list(node_list, raw):
         _pvc_version_length = len(node_information.get("pvc_version", "N/A")) + 1
         if _pvc_version_length > pvc_version_length:
             pvc_version_length = _pvc_version_length
+        # node_health column
+        node_health = node_information.get("health", "N/A")
+        if isinstance(node_health, int):
+            node_health_text = f"{node_health}%"
+        else:
+            node_health_text = node_health
+        _health_length = len(node_health_text) + 1
+        if _health_length > health_length:
+            health_length = _health_length
         # daemon_state column
         _daemon_state_length = len(node_information["daemon_state"]) + 1
         if _daemon_state_length > daemon_state_length:
@@ -466,7 +533,10 @@ def format_list(node_list, raw):
     # Format the string (header)
     node_list_output.append(
         "{bold}{node_header: <{node_header_length}} {state_header: <{state_header_length}} {resource_header: <{resource_header_length}} {memory_header: <{memory_header_length}}{end_bold}".format(
-            node_header_length=node_name_length + pvc_version_length + 1,
+            node_header_length=node_name_length
+            + pvc_version_length
+            + health_length
+            + 2,
             state_header_length=daemon_state_length
             + coordinator_state_length
             + domain_state_length
@@ -484,7 +554,14 @@ def format_list(node_list, raw):
             bold=ansiprint.bold(),
             end_bold=ansiprint.end(),
             node_header="Nodes "
-            + "".join(["-" for _ in range(6, node_name_length + pvc_version_length)]),
+            + "".join(
+                [
+                    "-"
+                    for _ in range(
+                        6, node_name_length + pvc_version_length + health_length + 1
+                    )
+                ]
+            ),
             state_header="States "
             + "".join(
                 [
@@ -526,12 +603,13 @@ def format_list(node_list, raw):
     )
 
     node_list_output.append(
-        "{bold}{node_name: <{node_name_length}} {node_pvc_version: <{pvc_version_length}} \
+        "{bold}{node_name: <{node_name_length}} {node_pvc_version: <{pvc_version_length}} {node_health: <{health_length}} \
 {daemon_state_colour}{node_daemon_state: <{daemon_state_length}}{end_colour} {coordinator_state_colour}{node_coordinator_state: <{coordinator_state_length}}{end_colour} {domain_state_colour}{node_domain_state: <{domain_state_length}}{end_colour} \
 {node_domains_count: <{domains_count_length}} {node_cpu_count: <{cpu_count_length}} {node_load: <{load_length}} \
 {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {node_mem_allocated: <{mem_alloc_length}} {node_mem_provisioned: <{mem_prov_length}}{end_bold}".format(
             node_name_length=node_name_length,
             pvc_version_length=pvc_version_length,
+            health_length=health_length,
             daemon_state_length=daemon_state_length,
             coordinator_state_length=coordinator_state_length,
             domain_state_length=domain_state_length,
@@ -551,6 +629,7 @@ def format_list(node_list, raw):
             end_colour="",
             node_name="Name",
             node_pvc_version="Version",
+            node_health="Health",
             node_daemon_state="Daemon",
             node_coordinator_state="Coordinator",
             node_domain_state="Domain",
@@ -568,19 +647,28 @@ def format_list(node_list, raw):
     # Format the string (elements)
     for node_information in sorted(node_list, key=lambda n: n["name"]):
         (
+            health_colour,
             daemon_state_colour,
             coordinator_state_colour,
             domain_state_colour,
             mem_allocated_colour,
             mem_provisioned_colour,
         ) = getOutputColours(node_information)
+
+        node_health = node_information.get("health", "N/A")
+        if isinstance(node_health, int):
+            node_health_text = f"{node_health}%"
+        else:
+            node_health_text = node_health
+
         node_list_output.append(
-            "{bold}{node_name: <{node_name_length}} {node_pvc_version: <{pvc_version_length}} \
+            "{bold}{node_name: <{node_name_length}} {node_pvc_version: <{pvc_version_length}} {health_colour}{node_health: <{health_length}}{end_colour} \
 {daemon_state_colour}{node_daemon_state: <{daemon_state_length}}{end_colour} {coordinator_state_colour}{node_coordinator_state: <{coordinator_state_length}}{end_colour} {domain_state_colour}{node_domain_state: <{domain_state_length}}{end_colour} \
 {node_domains_count: <{domains_count_length}} {node_cpu_count: <{cpu_count_length}} {node_load: <{load_length}} \
 {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {mem_allocated_colour}{node_mem_allocated: <{mem_alloc_length}}{end_colour} {mem_provisioned_colour}{node_mem_provisioned: <{mem_prov_length}}{end_colour}{end_bold}".format(
                 node_name_length=node_name_length,
                 pvc_version_length=pvc_version_length,
+                health_length=health_length,
                 daemon_state_length=daemon_state_length,
                 coordinator_state_length=coordinator_state_length,
                 domain_state_length=domain_state_length,
@@ -594,6 +682,7 @@ def format_list(node_list, raw):
                 mem_prov_length=mem_prov_length,
                 bold="",
                 end_bold="",
+                health_colour=health_colour,
                 daemon_state_colour=daemon_state_colour,
                 coordinator_state_colour=coordinator_state_colour,
                 domain_state_colour=domain_state_colour,
@@ -602,6 +691,7 @@ def format_list(node_list, raw):
                 end_colour=ansiprint.end(),
                 node_name=node_information["name"],
                 node_pvc_version=node_information.get("pvc_version", "N/A"),
+                node_health=node_health_text,
                 node_daemon_state=node_information["daemon_state"],
                 node_coordinator_state=node_information["coordinator_state"],
                 node_domain_state=node_information["domain_state"],

daemon-common/migrations/versions/9.json

@@ -1 +1 @@
-{"version": "9", "root": "", "base": {"root": "", "schema": "/schema", "schema.version": "/schema/version", "config": "/config", "config.maintenance": "/config/maintenance", "config.primary_node": "/config/primary_node", "config.primary_node.sync_lock": "/config/primary_node/sync_lock", "config.upstream_ip": "/config/upstream_ip", "config.migration_target_selector": "/config/migration_target_selector", "cmd": "/cmd", "cmd.node": "/cmd/nodes", "cmd.domain": "/cmd/domains", "cmd.ceph": "/cmd/ceph", "logs": "/logs", "node": "/nodes", "domain": "/domains", "network": "/networks", "storage": "/ceph", "storage.util": "/ceph/util", "osd": "/ceph/osds", "pool": "/ceph/pools", "volume": "/ceph/volumes", "snapshot": "/ceph/snapshots"}, "logs": {"node": "", "messages": "/messages"}, "node": {"name": "", "keepalive": "/keepalive", "mode": "/daemonmode", "data.active_schema": "/activeschema", "data.latest_schema": "/latestschema", "data.static": "/staticdata", "data.pvc_version": "/pvcversion", "running_domains": "/runningdomains", "count.provisioned_domains": "/domainscount", "count.networks": "/networkscount", "state.daemon": "/daemonstate", "state.router": "/routerstate", "state.domain": "/domainstate", "cpu.load": "/cpuload", "vcpu.allocated": "/vcpualloc", "memory.total": "/memtotal", "memory.used": "/memused", "memory.free": "/memfree", "memory.allocated": "/memalloc", "memory.provisioned": "/memprov", "ipmi.hostname": "/ipmihostname", "ipmi.username": "/ipmiusername", "ipmi.password": "/ipmipassword", "sriov": "/sriov", "sriov.pf": "/sriov/pf", "sriov.vf": "/sriov/vf", "monitoring.plugins": "/monitoring_plugins", "monitoring.data": "/monitoring_data"}, "monitoring_plugin": {"name": "", "last_run": "/last_run", "health_delta": "/health_delta", "message": "/message", "data": "/data", "runtime": "/runtime"}, "sriov_pf": {"phy": "", "mtu": "/mtu", "vfcount": "/vfcount"}, "sriov_vf": {"phy": "", "pf": "/pf", "mtu": "/mtu", "mac": "/mac", "phy_mac": "/phy_mac", "config": "/config", "config.vlan_id": "/config/vlan_id", "config.vlan_qos": "/config/vlan_qos", "config.tx_rate_min": "/config/tx_rate_min", "config.tx_rate_max": "/config/tx_rate_max", "config.spoof_check": "/config/spoof_check", "config.link_state": "/config/link_state", "config.trust": "/config/trust", "config.query_rss": "/config/query_rss", "pci": "/pci", "pci.domain": "/pci/domain", "pci.bus": "/pci/bus", "pci.slot": "/pci/slot", "pci.function": "/pci/function", "used": "/used", "used_by": "/used_by"}, "domain": {"name": "", "xml": "/xml", "state": "/state", "profile": "/profile", "stats": "/stats", "node": "/node", "last_node": "/lastnode", "failed_reason": "/failedreason", "storage.volumes": "/rbdlist", "console.log": "/consolelog", "console.vnc": "/vnc", "meta.autostart": "/node_autostart", "meta.migrate_method": "/migration_method", "meta.node_selector": "/node_selector", "meta.node_limit": "/node_limit", "meta.tags": "/tags", "migrate.sync_lock": "/migrate_sync_lock"}, "tag": {"name": "", "type": "/type", "protected": "/protected"}, "network": {"vni": "", "type": "/nettype", "mtu": "/mtu", "rule": "/firewall_rules", "rule.in": "/firewall_rules/in", "rule.out": "/firewall_rules/out", "nameservers": "/name_servers", "domain": "/domain", "reservation": "/dhcp4_reservations", "lease": "/dhcp4_leases", "ip4.gateway": "/ip4_gateway", "ip4.network": "/ip4_network", "ip4.dhcp": "/dhcp4_flag", "ip4.dhcp_start": "/dhcp4_start", "ip4.dhcp_end": "/dhcp4_end", "ip6.gateway": "/ip6_gateway", "ip6.network": "/ip6_network", "ip6.dhcp": "/dhcp6_flag"}, "reservation": {"mac": "", "ip": "/ipaddr", "hostname": "/hostname"}, "lease": {"mac": "", "ip": "/ipaddr", "hostname": "/hostname", "expiry": "/expiry", "client_id": "/clientid"}, "rule": {"description": "", "rule": "/rule", "order": "/order"}, "osd": {"id": "", "node": "/node", "device": "/device", "db_device": "/db_device", "fsid": "/fsid", "ofsid": "/fsid/osd", "cfsid": "/fsid/cluster", "lvm": "/lvm", "vg": "/lvm/vg", "lv": "/lvm/lv", "stats": "/stats"}, "pool": {"name": "", "pgs": "/pgs", "tier": "/tier", "stats": "/stats"}, "volume": {"name": "", "stats": "/stats"}, "snapshot": {"name": "", "stats": "/stats"}}
+{"version": "9", "root": "", "base": {"root": "", "schema": "/schema", "schema.version": "/schema/version", "config": "/config", "config.maintenance": "/config/maintenance", "config.primary_node": "/config/primary_node", "config.primary_node.sync_lock": "/config/primary_node/sync_lock", "config.upstream_ip": "/config/upstream_ip", "config.migration_target_selector": "/config/migration_target_selector", "cmd": "/cmd", "cmd.node": "/cmd/nodes", "cmd.domain": "/cmd/domains", "cmd.ceph": "/cmd/ceph", "logs": "/logs", "node": "/nodes", "domain": "/domains", "network": "/networks", "storage": "/ceph", "storage.util": "/ceph/util", "osd": "/ceph/osds", "pool": "/ceph/pools", "volume": "/ceph/volumes", "snapshot": "/ceph/snapshots"}, "logs": {"node": "", "messages": "/messages"}, "node": {"name": "", "keepalive": "/keepalive", "mode": "/daemonmode", "data.active_schema": "/activeschema", "data.latest_schema": "/latestschema", "data.static": "/staticdata", "data.pvc_version": "/pvcversion", "running_domains": "/runningdomains", "count.provisioned_domains": "/domainscount", "count.networks": "/networkscount", "state.daemon": "/daemonstate", "state.router": "/routerstate", "state.domain": "/domainstate", "cpu.load": "/cpuload", "vcpu.allocated": "/vcpualloc", "memory.total": "/memtotal", "memory.used": "/memused", "memory.free": "/memfree", "memory.allocated": "/memalloc", "memory.provisioned": "/memprov", "ipmi.hostname": "/ipmihostname", "ipmi.username": "/ipmiusername", "ipmi.password": "/ipmipassword", "sriov": "/sriov", "sriov.pf": "/sriov/pf", "sriov.vf": "/sriov/vf", "monitoring.plugins": "/monitoring_plugins", "monitoring.data": "/monitoring_data", "monitoring.health": "/monitoring_health"}, "monitoring_plugin": {"name": "", "last_run": "/last_run", "health_delta": "/health_delta", "message": "/message", "data": "/data", "runtime": "/runtime"}, "sriov_pf": {"phy": "", "mtu": "/mtu", "vfcount": "/vfcount"}, "sriov_vf": {"phy": "", "pf": "/pf", "mtu": "/mtu", "mac": "/mac", "phy_mac": "/phy_mac", "config": "/config", "config.vlan_id": "/config/vlan_id", "config.vlan_qos": "/config/vlan_qos", "config.tx_rate_min": "/config/tx_rate_min", "config.tx_rate_max": "/config/tx_rate_max", "config.spoof_check": "/config/spoof_check", "config.link_state": "/config/link_state", "config.trust": "/config/trust", "config.query_rss": "/config/query_rss", "pci": "/pci", "pci.domain": "/pci/domain", "pci.bus": "/pci/bus", "pci.slot": "/pci/slot", "pci.function": "/pci/function", "used": "/used", "used_by": "/used_by"}, "domain": {"name": "", "xml": "/xml", "state": "/state", "profile": "/profile", "stats": "/stats", "node": "/node", "last_node": "/lastnode", "failed_reason": "/failedreason", "storage.volumes": "/rbdlist", "console.log": "/consolelog", "console.vnc": "/vnc", "meta.autostart": "/node_autostart", "meta.migrate_method": "/migration_method", "meta.node_selector": "/node_selector", "meta.node_limit": "/node_limit", "meta.tags": "/tags", "migrate.sync_lock": "/migrate_sync_lock"}, "tag": {"name": "", "type": "/type", "protected": "/protected"}, "network": {"vni": "", "type": "/nettype", "mtu": "/mtu", "rule": "/firewall_rules", "rule.in": "/firewall_rules/in", "rule.out": "/firewall_rules/out", "nameservers": "/name_servers", "domain": "/domain", "reservation": "/dhcp4_reservations", "lease": "/dhcp4_leases", "ip4.gateway": "/ip4_gateway", "ip4.network": "/ip4_network", "ip4.dhcp": "/dhcp4_flag", "ip4.dhcp_start": "/dhcp4_start", "ip4.dhcp_end": "/dhcp4_end", "ip6.gateway": "/ip6_gateway", "ip6.network": "/ip6_network", "ip6.dhcp": "/dhcp6_flag"}, "reservation": {"mac": "", "ip": "/ipaddr", "hostname": "/hostname"}, "lease": {"mac": "", "ip": "/ipaddr", "hostname": "/hostname", "expiry": "/expiry", "client_id": "/clientid"}, "rule": {"description": "", "rule": "/rule", "order": "/order"}, "osd": {"id": "", "node": "/node", "device": "/device", "db_device": "/db_device", "fsid": "/fsid", "ofsid": "/fsid/osd", "cfsid": "/fsid/cluster", "lvm": "/lvm", "vg": "/lvm/vg", "lv": "/lvm/lv", "stats": "/stats"}, "pool": {"name": "", "pgs": "/pgs", "tier": "/tier", "stats": "/stats"}, "volume": {"name": "", "stats": "/stats"}, "snapshot": {"name": "", "stats": "/stats"}}

daemon-common/node.py

@@ -21,6 +21,7 @@
 
 import time
 import re
+import json
 
 import daemon_lib.common as common
 
@@ -49,6 +50,38 @@ def getNodeInformation(zkhandler, node_name):
         zkhandler.read(("node.count.provisioned_domains", node_name))
     )
     node_running_domains = zkhandler.read(("node.running_domains", node_name)).split()
+    try:
+        node_health = int(zkhandler.read(("node.monitoring.health", node_name)))
+    except ValueError:
+        node_health = "N/A"
+    node_health_plugins = zkhandler.read(("node.monitoring.plugins", node_name)).split()
+    node_health_details = list()
+    for plugin in node_health_plugins:
+        plugin_last_run = zkhandler.read(
+            ("node.monitoring.data", node_name, "monitoring_plugin.last_run", plugin)
+        )
+        plugin_health_delta = zkhandler.read(
+            (
+                "node.monitoring.data",
+                node_name,
+                "monitoring_plugin.health_delta",
+                plugin,
+            )
+        )
+        plugin_message = zkhandler.read(
+            ("node.monitoring.data", node_name, "monitoring_plugin.message", plugin)
+        )
+        plugin_data = zkhandler.read(
+            ("node.monitoring.data", node_name, "monitoring_plugin.data", plugin)
+        )
+        plugin_output = {
+            "name": plugin,
+            "last_run": int(plugin_last_run),
+            "health_delta": int(plugin_health_delta),
+            "message": plugin_message,
+            "data": json.loads(plugin_data),
+        }
+        node_health_details.append(plugin_output)
 
     # Construct a data structure to represent the data
     node_information = {
@@ -61,10 +94,16 @@ def getNodeInformation(zkhandler, node_name):
         "kernel": node_kernel,
         "os": node_os,
         "arch": node_arch,
+        "health": node_health,
+        "health_plugins": node_health_plugins,
+        "health_details": node_health_details,
         "load": node_load,
         "domains_count": node_domains_count,
         "running_domains": node_running_domains,
-        "vcpu": {"total": node_cpu_count, "allocated": node_vcpu_allocated},
+        "vcpu": {
+            "total": node_cpu_count,
+            "allocated": node_vcpu_allocated,
+        },
         "memory": {
             "total": node_mem_total,
             "allocated": node_mem_allocated,

daemon-common/zkhandler.py

@@ -610,6 +610,7 @@ class ZKSchema(object):
             "sriov.vf": "/sriov/vf",
             "monitoring.plugins": "/monitoring_plugins",
             "monitoring.data": "/monitoring_data",
+            "monitoring.health": "/monitoring_health",
         },
         # The schema of an individual monitoring plugin data entry (/nodes/{node_name}/monitoring_data/{plugin})
         "monitoring_plugin": {

node-daemon/plugins/ceph

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# ceph.py - PVC Monitoring example plugin for ceph status
+# ceph.py - PVC Monitoring example plugin for Ceph status
 # Part of the Parallel Virtual Cluster (PVC) system
 #
 #    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
@@ -111,7 +111,7 @@ class MonitoringPluginScript(MonitoringPlugin):
         self.plugin_result.set_message(message)
 
         # Set the detailed data in our local PluginResult object
-        self.plugin_result.set_data(dumps(health_status))
+        self.plugin_result.set_data(health_status)
 
         # Return our local PluginResult object
         return self.plugin_result

node-daemon/plugins/dpkg

@@ -66,7 +66,6 @@ class MonitoringPluginScript(MonitoringPlugin):
 
         # Run any imports first
         from re import match
-        from json import dumps
         import daemon_lib.common as pvc_common
 
         # Get Debian version
@@ -143,7 +142,7 @@ class MonitoringPluginScript(MonitoringPlugin):
             "inconsistent_packages": list_inconsistent,
             "upgradable_packages": list_upgradable,
         }
-        self.plugin_result.set_data(dumps(detailed_data))
+        self.plugin_result.set_data(detailed_data)
 
         # Return our local PluginResult object
         return self.plugin_result

node-daemon/pvcnoded/objects/MonitoringInstance.py

@@ -25,6 +25,7 @@ import importlib.util
 
 from os import walk
 from datetime import datetime
+from json import dumps
 
 
 class PluginResult(object):
@@ -37,7 +38,7 @@ class PluginResult(object):
         self.current_time = int(time.time())
         self.health_delta = 0
         self.message = None
-        self.data = None
+        self.data = {}
         self.runtime = "0.00"
 
     def set_health_delta(self, new_delta):
@@ -98,7 +99,7 @@ class PluginResult(object):
                         "monitoring_plugin.data",
                         self.plugin_name,
                     ),
-                    self.data,
+                    dumps(self.data),
                 ),
                 (
                     (
@@ -129,6 +130,9 @@ class MonitoringPlugin(object):
             self.plugin_name,
         )
 
+    def __str__(self):
+        return self.plugin_name
+
     #
     # Helper functions; exposed to child MonitoringPluginScript instances
     #
@@ -256,7 +260,7 @@ class MonitoringInstance(object):
                                 "monitoring_plugin.data",
                                 plugin.plugin_name,
                             ),
-                            None,
+                            dumps({}),
                         ),
                         (
                             (
@@ -283,7 +287,7 @@ class MonitoringInstance(object):
             [
                 (
                     ("node.monitoring.plugins", self.this_node.name),
-                    self.all_plugin_names,
+                    " ".join(self.all_plugin_names),
                 ),
             ]
         )
@@ -309,15 +313,15 @@ class MonitoringInstance(object):
         time_delta = time_end - time_start
         runtime = "{:0.02f}".format(time_delta.total_seconds())
         result.set_runtime(runtime)
-        self.logger.out(
-            result.message, state="t", prefix=f"{plugin.plugin_name} ({runtime}s)"
-        )
         result.to_zookeeper()
         return result
 
     def run_plugins(self):
         total_health = 100
-        self.logger.out("Running monitoring plugins:", state="t")
+        self.logger.out(
+            f"Running monitoring plugins: {', '.join([x.plugin_name for x in self.all_plugins])}",
+            state="t",
+        )
         plugin_results = list()
         with concurrent.futures.ThreadPoolExecutor(max_workers=99) as executor:
             to_future_plugin_results = {
@@ -327,7 +331,12 @@ class MonitoringInstance(object):
             for future in concurrent.futures.as_completed(to_future_plugin_results):
                 plugin_results.append(future.result())
 
-        for result in plugin_results:
+        for result in sorted(plugin_results, key=lambda x: x.plugin_name):
+            self.logger.out(
+                result.message,
+                state="t",
+                prefix=f"{result.plugin_name} ({result.runtime}s)",
+            )
             if result is not None:
                 total_health -= result.health_delta
 
@@ -338,6 +347,14 @@ class MonitoringInstance(object):
         else:
             health_colour = self.logger.fmt_red
 
+        self.zkhandler.write(
+            [
+                (
+                    ("node.monitoring.health", self.this_node.name),
+                    total_health,
+                ),
+            ]
+        )
         self.logger.out(
             f"System health: {health_colour}{total_health}/100{self.logger.fmt_end}",
             state="t",
@@ -355,3 +372,12 @@ class MonitoringInstance(object):
             for future in concurrent.futures.as_completed(to_future_plugin_results):
                 # This doesn't do anything, just lets us wait for them all to complete
                 pass
+        # Set the node health to None as no previous checks are now valid
+        self.zkhandler.write(
+            [
+                (
+                    ("node.monitoring.health", self.this_node.name),
+                    None,
+                ),
+            ]
+        )