Improve output formatting for simplicity

client-cli/pvc/cli/cli.py

@@ -511,11 +511,12 @@ def cli_cluster_fault():
 @click.argument("limit", default=None, required=False)
 @format_opt(
     {
-        "pretty": cli_cluster_fault_list_format_pretty,
+        "short": cli_cluster_fault_list_format_short,
-        #        "short": cli_cluster_status_format_short,
+        "long": cli_cluster_fault_list_format_long,
         "json": lambda d: jdumps(d),
         "json-pretty": lambda d: jdumps(d, indent=2),
-    }
+    },
+    default_format="short",
 )
 @connection_req
 def cli_cluster_fault_list(limit, format_function):

client-cli/pvc/cli/formatters.py

@@ -19,6 +19,7 @@
 #
 ###############################################################################
 
+from pvc.cli.helpers import MAX_CONTENT_WIDTH
 from pvc.lib.node import format_info as node_format_info
 from pvc.lib.node import format_list as node_format_list
 from pvc.lib.vm import format_vm_tags as vm_format_tags
@@ -96,6 +97,11 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data):
     output.append(f"{ansii['bold']}PVC cluster status:{ansii['end']}")
     output.append("")
 
+    output.append(f"{ansii['purple']}Primary node:{ansii['end']}  {primary_node}")
+    output.append(f"{ansii['purple']}PVC version:{ansii['end']}   {pvc_version}")
+    output.append(f"{ansii['purple']}Upstream IP:{ansii['end']}   {upstream_ip}")
+    output.append("")
+
     if health != "-1":
         health = f"{health}%"
     else:
@@ -105,18 +111,33 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data):
         health = f"{health} (maintenance on)"
 
     output.append(
-        f"{ansii['purple']}Cluster health:{ansii['end']}   {health_colour}{health}{ansii['end']}"
+        f"{ansii['purple']}Health:{ansii['end']}        {health_colour}{health}{ansii['end']}"
     )
 
     if messages is not None and len(messages) > 0:
-        messages = "\n                  ".join(sorted(messages))
+        message_list = list()
-        output.append(f"{ansii['purple']}Health messages:{ansii['end']}  {messages}")
+        for message in messages:
+            if message["health_delta"] >= 50:
+                message_colour = ansii["red"]
+            elif message["health_delta"] >= 10:
+                message_colour = ansii["yellow"]
+            else:
+                message_colour = ansii["green"]
+            message_delta = (
+                f"({message_colour}-{message['health_delta']}%{ansii['end']})"
+            )
+            message_list.append(
+                # 15 length due to ANSI colour sequences
+                "{id} {delta:<15} {text}".format(
+                    id=message["id"],
+                    delta=message_delta,
+                    text=message["text"],
+                )
+            )
 
-    output.append("")
+        messages = "\n               ".join(message_list)
+        output.append(f"{ansii['purple']}New Faults:{ansii['end']}    {messages}")
 
-    output.append(f"{ansii['purple']}Primary node:{ansii['end']}     {primary_node}")
-    output.append(f"{ansii['purple']}PVC version:{ansii['end']}      {pvc_version}")
-    output.append(f"{ansii['purple']}Upstream IP:{ansii['end']}      {upstream_ip}")
     output.append("")
 
     node_states = ["run,ready"]
@@ -145,7 +166,7 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data):
 
     nodes_string = ", ".join(nodes_strings)
 
-    output.append(f"{ansii['purple']}Nodes:{ansii['end']}            {nodes_string}")
+    output.append(f"{ansii['purple']}Nodes:{ansii['end']}         {nodes_string}")
 
     vm_states = ["start", "disable"]
     vm_states.extend(
@@ -175,7 +196,7 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data):
 
     vms_string = ", ".join(vms_strings)
 
-    output.append(f"{ansii['purple']}VMs:{ansii['end']}              {vms_string}")
+    output.append(f"{ansii['purple']}VMs:{ansii['end']}           {vms_string}")
 
     osd_states = ["up,in"]
     osd_states.extend(
@@ -201,15 +222,15 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data):
 
     osds_string = " ".join(osds_strings)
 
-    output.append(f"{ansii['purple']}OSDs:{ansii['end']}             {osds_string}")
+    output.append(f"{ansii['purple']}OSDs:{ansii['end']}          {osds_string}")
 
-    output.append(f"{ansii['purple']}Pools:{ansii['end']}            {total_pools}")
+    output.append(f"{ansii['purple']}Pools:{ansii['end']}         {total_pools}")
 
-    output.append(f"{ansii['purple']}Volumes:{ansii['end']}          {total_volumes}")
+    output.append(f"{ansii['purple']}Volumes:{ansii['end']}       {total_volumes}")
 
-    output.append(f"{ansii['purple']}Snapshots:{ansii['end']}        {total_snapshots}")
+    output.append(f"{ansii['purple']}Snapshots:{ansii['end']}     {total_snapshots}")
 
-    output.append(f"{ansii['purple']}Networks:{ansii['end']}         {total_networks}")
+    output.append(f"{ansii['purple']}Networks:{ansii['end']}      {total_networks}")
 
     output.append("")
 
@@ -249,19 +270,143 @@ def cli_cluster_status_format_short(CLI_CONFIG, data):
         health = f"{health} (maintenance on)"
 
     output.append(
-        f"{ansii['purple']}Cluster health:{ansii['end']}   {health_colour}{health}{ansii['end']}"
+        f"{ansii['purple']}Health:{ansii['end']}  {health_colour}{health}{ansii['end']}"
     )
 
     if messages is not None and len(messages) > 0:
-        messages = "\n                  ".join(sorted(messages))
+        messages = "\n         ".join(sorted(messages))
-        output.append(f"{ansii['purple']}Health messages:{ansii['end']}  {messages}")
+        output.append(f"{ansii['purple']}Faults:{ansii['end']}  {messages}")
 
     output.append("")
 
     return "\n".join(output)
 
 
-def cli_cluster_fault_list_format_pretty(CLI_CONFIG, fault_data):
+def cli_cluster_fault_list_format_short(CLI_CONFIG, fault_data):
+    """
+    Short pretty format the output of cli_cluster_fault_list
+    """
+
+    fault_list_output = []
+
+    # Determine optimal column widths
+    fault_id_length = 3  # "ID"
+    fault_last_reported_length = 14  # "Last Reported"
+    fault_health_delta_length = 7  # "Health"
+    fault_message_length = 8  # "Message"
+
+    for fault in fault_data:
+        # fault_id column
+        _fault_id_length = len(str(fault["id"])) + 1
+        if _fault_id_length > fault_id_length:
+            fault_id_length = _fault_id_length
+
+        # health_delta column
+        _fault_health_delta_length = len(str(fault["health_delta"])) + 1
+        if _fault_health_delta_length > fault_health_delta_length:
+            fault_health_delta_length = _fault_health_delta_length
+
+        # last_reported column
+        _fault_last_reported_length = len(str(fault["last_reported"])) + 1
+        if _fault_last_reported_length > fault_last_reported_length:
+            fault_last_reported_length = _fault_last_reported_length
+
+        # message column
+        _fault_message_length = len(str(fault["message"])) + 1
+        if _fault_message_length > fault_message_length:
+            fault_message_length = _fault_message_length
+
+    message_prefix_len = (
+        fault_id_length
+        + 1
+        + fault_health_delta_length
+        + 1
+        + fault_last_reported_length
+        + 1
+    )
+    message_length = MAX_CONTENT_WIDTH - message_prefix_len
+
+    if fault_message_length > message_length:
+        fault_message_length = message_length
+
+    meta_header_length = fault_id_length + fault_health_delta_length + 1
+    detail_header_length = (
+        fault_health_delta_length
+        + fault_last_reported_length
+        + fault_message_length
+        + 2
+        - meta_header_length
+        + 8
+    )
+
+    # Format the string (header)
+    fault_list_output.append(
+        "{bold}Meta {meta_dashes} Fault {detail_dashes}{end_bold}".format(
+            bold=ansii["bold"],
+            end_bold=ansii["end"],
+            meta_dashes="-" * (meta_header_length - len("Meta ")),
+            detail_dashes="-" * (detail_header_length - len("Fault ")),
+        )
+    )
+
+    fault_list_output.append(
+        "{bold}{fault_id: <{fault_id_length}} {fault_health_delta: <{fault_health_delta_length}} {fault_last_reported: <{fault_last_reported_length}} {fault_message}{end_bold}".format(
+            bold=ansii["bold"],
+            end_bold=ansii["end"],
+            fault_id_length=fault_id_length,
+            fault_health_delta_length=fault_health_delta_length,
+            fault_last_reported_length=fault_last_reported_length,
+            fault_id="ID",
+            fault_health_delta="Health",
+            fault_last_reported="Last Reported",
+            fault_message="Message",
+        )
+    )
+
+    for fault in sorted(
+        fault_data,
+        key=lambda x: (x["health_delta"], x["last_reported"]),
+        reverse=True,
+    ):
+        health_delta = fault["health_delta"]
+        if fault["acknowledged_at"] != "":
+            health_colour = ansii["blue"]
+        elif health_delta >= 50:
+            health_colour = ansii["red"]
+        elif health_delta >= 10:
+            health_colour = ansii["yellow"]
+        else:
+            health_colour = ansii["green"]
+
+        if len(fault["message"]) > message_length:
+            split_message = list(
+                fault["message"][0 + i : message_length + i].strip()
+                for i in range(0, len(fault["message"]), message_length)
+            )
+            message = f"\n{' ' * message_prefix_len}".join(split_message)
+        else:
+            message = fault["message"]
+
+        fault_list_output.append(
+            "{bold}{fault_id: <{fault_id_length}} {health_colour}{fault_health_delta: <{fault_health_delta_length}}{end_colour} {fault_last_reported: <{fault_last_reported_length}} {fault_message}{end_bold}".format(
+                bold="",
+                end_bold="",
+                health_colour=health_colour,
+                end_colour=ansii["end"],
+                fault_id_length=fault_id_length,
+                fault_health_delta_length=fault_health_delta_length,
+                fault_last_reported_length=fault_last_reported_length,
+                fault_id=fault["id"],
+                fault_health_delta=f"-{fault['health_delta']}%",
+                fault_last_reported=fault["last_reported"],
+                fault_message=message,
+            )
+        )
+
+    return "\n".join(fault_list_output)
+
+
+def cli_cluster_fault_list_format_long(CLI_CONFIG, fault_data):
     """
     Pretty format the output of cli_cluster_fault_list
     """
@@ -272,9 +417,9 @@ def cli_cluster_fault_list_format_pretty(CLI_CONFIG, fault_data):
     fault_id_length = 3  # "ID"
     fault_status_length = 7  # "Status"
     fault_health_delta_length = 7  # "Health"
-    fault_acknowledged_at_length = 6  # "Ack'd"
+    fault_acknowledged_at_length = 9  # "Ack'd On"
-    fault_last_reported_length = 5  # "Last"
+    fault_last_reported_length = 14  # "Last Reported"
-    fault_first_reported_length = 6  # "First"
+    fault_first_reported_length = 15  # "First Reported"
     # Message goes on its own line
 
     for fault in fault_data:
@@ -322,9 +467,9 @@ def cli_cluster_fault_list_format_pretty(CLI_CONFIG, fault_data):
             fault_id="ID",
             fault_status="Status",
             fault_health_delta="Health",
-            fault_acknowledged_at="Ack'd",
+            fault_acknowledged_at="Ack'd On",
-            fault_last_reported="Last",
+            fault_last_reported="Last Reported",
-            fault_first_reported="First",
+            fault_first_reported="First Reported",
         )
     )
     fault_list_output.append(

client-cli/pvc/cli/helpers.py

@@ -26,7 +26,7 @@ from distutils.util import strtobool
 from getpass import getuser
 from json import load as jload
 from json import dump as jdump
-from os import chmod, environ, getpid, path, makedirs
+from os import chmod, environ, getpid, path, makedirs, get_terminal_size
 from re import findall
 from socket import gethostname
 from subprocess import run, PIPE
@@ -45,7 +45,9 @@ DEFAULT_STORE_FILENAME = "pvc.json"
 DEFAULT_API_PREFIX = "/api/v1"
 DEFAULT_NODE_HOSTNAME = gethostname().split(".")[0]
 DEFAULT_AUTOBACKUP_FILENAME = "/etc/pvc/pvc.conf"
-MAX_CONTENT_WIDTH = 120
+
+# Define the content width to be the maximum temminal size
+MAX_CONTENT_WIDTH = get_terminal_size().columns - 1
 
 
 def echo(config, message, newline=True, stderr=False):

daemon-common/cluster.py

@@ -22,6 +22,7 @@
 from json import loads
 
 import daemon_lib.common as common
+import daemon_lib.faults as faults
 import daemon_lib.vm as pvc_vm
 import daemon_lib.node as pvc_node
 import daemon_lib.network as pvc_network
@@ -44,6 +45,39 @@ def set_maintenance(zkhandler, maint_state):
         return True, "Successfully set cluster in normal mode"
 
 
+def getClusterHealthFromFaults(zkhandler):
+    faults_list = faults.getAllFaults(zkhandler)
+
+    unacknowledged_faults = [fault for fault in faults_list if fault["status"] != "ack"]
+
+    # Generate total cluster health numbers
+    cluster_health_value = 100
+    cluster_health_messages = list()
+
+    for fault in sorted(
+        unacknowledged_faults,
+        key=lambda x: (x["health_delta"], x["last_reported"]),
+        reverse=True,
+    ):
+        cluster_health_value = -fault["health_delta"]
+        message = {
+            "id": fault["id"],
+            "health_delta": fault["health_delta"],
+            "text": fault["message"],
+        }
+        cluster_health_messages.append(message)
+
+    if cluster_health_value < 0:
+        cluster_health_value = 0
+
+    cluster_health = {
+        "health": cluster_health_value,
+        "messages": cluster_health_messages,
+    }
+
+    return cluster_health
+
+
 def getClusterHealth(zkhandler, node_list, vm_list, ceph_osd_list):
     health_delta_map = {
         "node_stopped": 50,
@@ -318,9 +352,7 @@ def getClusterInformation(zkhandler):
 
     # Format the status data
     cluster_information = {
-        "cluster_health": getClusterHealth(
+        "cluster_health": getClusterHealthFromFaults(zkhandler),
-            zkhandler, node_list, vm_list, ceph_osd_list
-        ),
         "node_health": getNodeHealth(zkhandler, node_list),
         "maintenance": maintenance_state,
         "primary_node": primary_node,

daemon-common/faults.py

@@ -37,6 +37,10 @@ def getFault(zkhandler, fault_id):
     fault_delta = int(zkhandler.read(("faults.delta", fault_id)))
     fault_message = zkhandler.read(("faults.message", fault_id))
 
+    # Acknowledged faults have a delta of 0
+    if fault_ack_time != "":
+        fault_delta = 0
+
     fault = {
         "id": fault_id,
         "last_reported": fault_last_time,