From 2267a9c85dfd7e03501846c260e2d7a22a35a757 Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Mon, 4 Dec 2023 15:48:49 -0500 Subject: [PATCH] Improve output formatting for simplicity --- client-cli/pvc/cli/cli.py | 7 +- client-cli/pvc/cli/formatters.py | 193 +++++++++++++++++++++++++++---- client-cli/pvc/cli/helpers.py | 6 +- daemon-common/cluster.py | 38 +++++- daemon-common/faults.py | 4 + 5 files changed, 216 insertions(+), 32 deletions(-) diff --git a/client-cli/pvc/cli/cli.py b/client-cli/pvc/cli/cli.py index 19422013..1f19d127 100644 --- a/client-cli/pvc/cli/cli.py +++ b/client-cli/pvc/cli/cli.py @@ -511,11 +511,12 @@ def cli_cluster_fault(): @click.argument("limit", default=None, required=False) @format_opt( { - "pretty": cli_cluster_fault_list_format_pretty, - # "short": cli_cluster_status_format_short, + "short": cli_cluster_fault_list_format_short, + "long": cli_cluster_fault_list_format_long, "json": lambda d: jdumps(d), "json-pretty": lambda d: jdumps(d, indent=2), - } + }, + default_format="short", ) @connection_req def cli_cluster_fault_list(limit, format_function): diff --git a/client-cli/pvc/cli/formatters.py b/client-cli/pvc/cli/formatters.py index 3e6cefe8..30c9414c 100644 --- a/client-cli/pvc/cli/formatters.py +++ b/client-cli/pvc/cli/formatters.py @@ -19,6 +19,7 @@ # ############################################################################### +from pvc.cli.helpers import MAX_CONTENT_WIDTH from pvc.lib.node import format_info as node_format_info from pvc.lib.node import format_list as node_format_list from pvc.lib.vm import format_vm_tags as vm_format_tags @@ -96,6 +97,11 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data): output.append(f"{ansii['bold']}PVC cluster status:{ansii['end']}") output.append("") + output.append(f"{ansii['purple']}Primary node:{ansii['end']} {primary_node}") + output.append(f"{ansii['purple']}PVC version:{ansii['end']} {pvc_version}") + output.append(f"{ansii['purple']}Upstream IP:{ansii['end']} {upstream_ip}") + output.append("") + if health != "-1": health = f"{health}%" else: @@ -105,18 +111,33 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data): health = f"{health} (maintenance on)" output.append( - f"{ansii['purple']}Cluster health:{ansii['end']} {health_colour}{health}{ansii['end']}" + f"{ansii['purple']}Health:{ansii['end']} {health_colour}{health}{ansii['end']}" ) if messages is not None and len(messages) > 0: - messages = "\n ".join(sorted(messages)) - output.append(f"{ansii['purple']}Health messages:{ansii['end']} {messages}") + message_list = list() + for message in messages: + if message["health_delta"] >= 50: + message_colour = ansii["red"] + elif message["health_delta"] >= 10: + message_colour = ansii["yellow"] + else: + message_colour = ansii["green"] + message_delta = ( + f"({message_colour}-{message['health_delta']}%{ansii['end']})" + ) + message_list.append( + # 15 length due to ANSI colour sequences + "{id} {delta:<15} {text}".format( + id=message["id"], + delta=message_delta, + text=message["text"], + ) + ) - output.append("") + messages = "\n ".join(message_list) + output.append(f"{ansii['purple']}New Faults:{ansii['end']} {messages}") - output.append(f"{ansii['purple']}Primary node:{ansii['end']} {primary_node}") - output.append(f"{ansii['purple']}PVC version:{ansii['end']} {pvc_version}") - output.append(f"{ansii['purple']}Upstream IP:{ansii['end']} {upstream_ip}") output.append("") node_states = ["run,ready"] @@ -145,7 +166,7 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data): nodes_string = ", ".join(nodes_strings) - output.append(f"{ansii['purple']}Nodes:{ansii['end']} {nodes_string}") + output.append(f"{ansii['purple']}Nodes:{ansii['end']} {nodes_string}") vm_states = ["start", "disable"] vm_states.extend( @@ -175,7 +196,7 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data): vms_string = ", ".join(vms_strings) - output.append(f"{ansii['purple']}VMs:{ansii['end']} {vms_string}") + output.append(f"{ansii['purple']}VMs:{ansii['end']} {vms_string}") osd_states = ["up,in"] osd_states.extend( @@ -201,15 +222,15 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data): osds_string = " ".join(osds_strings) - output.append(f"{ansii['purple']}OSDs:{ansii['end']} {osds_string}") + output.append(f"{ansii['purple']}OSDs:{ansii['end']} {osds_string}") - output.append(f"{ansii['purple']}Pools:{ansii['end']} {total_pools}") + output.append(f"{ansii['purple']}Pools:{ansii['end']} {total_pools}") - output.append(f"{ansii['purple']}Volumes:{ansii['end']} {total_volumes}") + output.append(f"{ansii['purple']}Volumes:{ansii['end']} {total_volumes}") - output.append(f"{ansii['purple']}Snapshots:{ansii['end']} {total_snapshots}") + output.append(f"{ansii['purple']}Snapshots:{ansii['end']} {total_snapshots}") - output.append(f"{ansii['purple']}Networks:{ansii['end']} {total_networks}") + output.append(f"{ansii['purple']}Networks:{ansii['end']} {total_networks}") output.append("") @@ -249,19 +270,143 @@ def cli_cluster_status_format_short(CLI_CONFIG, data): health = f"{health} (maintenance on)" output.append( - f"{ansii['purple']}Cluster health:{ansii['end']} {health_colour}{health}{ansii['end']}" + f"{ansii['purple']}Health:{ansii['end']} {health_colour}{health}{ansii['end']}" ) if messages is not None and len(messages) > 0: - messages = "\n ".join(sorted(messages)) - output.append(f"{ansii['purple']}Health messages:{ansii['end']} {messages}") + messages = "\n ".join(sorted(messages)) + output.append(f"{ansii['purple']}Faults:{ansii['end']} {messages}") output.append("") return "\n".join(output) -def cli_cluster_fault_list_format_pretty(CLI_CONFIG, fault_data): +def cli_cluster_fault_list_format_short(CLI_CONFIG, fault_data): + """ + Short pretty format the output of cli_cluster_fault_list + """ + + fault_list_output = [] + + # Determine optimal column widths + fault_id_length = 3 # "ID" + fault_last_reported_length = 14 # "Last Reported" + fault_health_delta_length = 7 # "Health" + fault_message_length = 8 # "Message" + + for fault in fault_data: + # fault_id column + _fault_id_length = len(str(fault["id"])) + 1 + if _fault_id_length > fault_id_length: + fault_id_length = _fault_id_length + + # health_delta column + _fault_health_delta_length = len(str(fault["health_delta"])) + 1 + if _fault_health_delta_length > fault_health_delta_length: + fault_health_delta_length = _fault_health_delta_length + + # last_reported column + _fault_last_reported_length = len(str(fault["last_reported"])) + 1 + if _fault_last_reported_length > fault_last_reported_length: + fault_last_reported_length = _fault_last_reported_length + + # message column + _fault_message_length = len(str(fault["message"])) + 1 + if _fault_message_length > fault_message_length: + fault_message_length = _fault_message_length + + message_prefix_len = ( + fault_id_length + + 1 + + fault_health_delta_length + + 1 + + fault_last_reported_length + + 1 + ) + message_length = MAX_CONTENT_WIDTH - message_prefix_len + + if fault_message_length > message_length: + fault_message_length = message_length + + meta_header_length = fault_id_length + fault_health_delta_length + 1 + detail_header_length = ( + fault_health_delta_length + + fault_last_reported_length + + fault_message_length + + 2 + - meta_header_length + + 8 + ) + + # Format the string (header) + fault_list_output.append( + "{bold}Meta {meta_dashes} Fault {detail_dashes}{end_bold}".format( + bold=ansii["bold"], + end_bold=ansii["end"], + meta_dashes="-" * (meta_header_length - len("Meta ")), + detail_dashes="-" * (detail_header_length - len("Fault ")), + ) + ) + + fault_list_output.append( + "{bold}{fault_id: <{fault_id_length}} {fault_health_delta: <{fault_health_delta_length}} {fault_last_reported: <{fault_last_reported_length}} {fault_message}{end_bold}".format( + bold=ansii["bold"], + end_bold=ansii["end"], + fault_id_length=fault_id_length, + fault_health_delta_length=fault_health_delta_length, + fault_last_reported_length=fault_last_reported_length, + fault_id="ID", + fault_health_delta="Health", + fault_last_reported="Last Reported", + fault_message="Message", + ) + ) + + for fault in sorted( + fault_data, + key=lambda x: (x["health_delta"], x["last_reported"]), + reverse=True, + ): + health_delta = fault["health_delta"] + if fault["acknowledged_at"] != "": + health_colour = ansii["blue"] + elif health_delta >= 50: + health_colour = ansii["red"] + elif health_delta >= 10: + health_colour = ansii["yellow"] + else: + health_colour = ansii["green"] + + if len(fault["message"]) > message_length: + split_message = list( + fault["message"][0 + i : message_length + i].strip() + for i in range(0, len(fault["message"]), message_length) + ) + message = f"\n{' ' * message_prefix_len}".join(split_message) + else: + message = fault["message"] + + fault_list_output.append( + "{bold}{fault_id: <{fault_id_length}} {health_colour}{fault_health_delta: <{fault_health_delta_length}}{end_colour} {fault_last_reported: <{fault_last_reported_length}} {fault_message}{end_bold}".format( + bold="", + end_bold="", + health_colour=health_colour, + end_colour=ansii["end"], + fault_id_length=fault_id_length, + fault_health_delta_length=fault_health_delta_length, + fault_last_reported_length=fault_last_reported_length, + fault_id=fault["id"], + fault_health_delta=f"-{fault['health_delta']}%", + fault_last_reported=fault["last_reported"], + fault_message=message, + ) + ) + + return "\n".join(fault_list_output) + + +def cli_cluster_fault_list_format_long(CLI_CONFIG, fault_data): """ Pretty format the output of cli_cluster_fault_list """ @@ -272,9 +417,9 @@ def cli_cluster_fault_list_format_pretty(CLI_CONFIG, fault_data): fault_id_length = 3 # "ID" fault_status_length = 7 # "Status" fault_health_delta_length = 7 # "Health" - fault_acknowledged_at_length = 6 # "Ack'd" - fault_last_reported_length = 5 # "Last" - fault_first_reported_length = 6 # "First" + fault_acknowledged_at_length = 9 # "Ack'd On" + fault_last_reported_length = 14 # "Last Reported" + fault_first_reported_length = 15 # "First Reported" # Message goes on its own line for fault in fault_data: @@ -322,9 +467,9 @@ def cli_cluster_fault_list_format_pretty(CLI_CONFIG, fault_data): fault_id="ID", fault_status="Status", fault_health_delta="Health", - fault_acknowledged_at="Ack'd", - fault_last_reported="Last", - fault_first_reported="First", + fault_acknowledged_at="Ack'd On", + fault_last_reported="Last Reported", + fault_first_reported="First Reported", ) ) fault_list_output.append( diff --git a/client-cli/pvc/cli/helpers.py b/client-cli/pvc/cli/helpers.py index 450ce031..b6c9a6f9 100644 --- a/client-cli/pvc/cli/helpers.py +++ b/client-cli/pvc/cli/helpers.py @@ -26,7 +26,7 @@ from distutils.util import strtobool from getpass import getuser from json import load as jload from json import dump as jdump -from os import chmod, environ, getpid, path, makedirs +from os import chmod, environ, getpid, path, makedirs, get_terminal_size from re import findall from socket import gethostname from subprocess import run, PIPE @@ -45,7 +45,9 @@ DEFAULT_STORE_FILENAME = "pvc.json" DEFAULT_API_PREFIX = "/api/v1" DEFAULT_NODE_HOSTNAME = gethostname().split(".")[0] DEFAULT_AUTOBACKUP_FILENAME = "/etc/pvc/pvc.conf" -MAX_CONTENT_WIDTH = 120 + +# Define the content width to be the maximum temminal size +MAX_CONTENT_WIDTH = get_terminal_size().columns - 1 def echo(config, message, newline=True, stderr=False): diff --git a/daemon-common/cluster.py b/daemon-common/cluster.py index 9e3f50a1..2ea8b05f 100644 --- a/daemon-common/cluster.py +++ b/daemon-common/cluster.py @@ -22,6 +22,7 @@ from json import loads import daemon_lib.common as common +import daemon_lib.faults as faults import daemon_lib.vm as pvc_vm import daemon_lib.node as pvc_node import daemon_lib.network as pvc_network @@ -44,6 +45,39 @@ def set_maintenance(zkhandler, maint_state): return True, "Successfully set cluster in normal mode" +def getClusterHealthFromFaults(zkhandler): + faults_list = faults.getAllFaults(zkhandler) + + unacknowledged_faults = [fault for fault in faults_list if fault["status"] != "ack"] + + # Generate total cluster health numbers + cluster_health_value = 100 + cluster_health_messages = list() + + for fault in sorted( + unacknowledged_faults, + key=lambda x: (x["health_delta"], x["last_reported"]), + reverse=True, + ): + cluster_health_value = -fault["health_delta"] + message = { + "id": fault["id"], + "health_delta": fault["health_delta"], + "text": fault["message"], + } + cluster_health_messages.append(message) + + if cluster_health_value < 0: + cluster_health_value = 0 + + cluster_health = { + "health": cluster_health_value, + "messages": cluster_health_messages, + } + + return cluster_health + + def getClusterHealth(zkhandler, node_list, vm_list, ceph_osd_list): health_delta_map = { "node_stopped": 50, @@ -318,9 +352,7 @@ def getClusterInformation(zkhandler): # Format the status data cluster_information = { - "cluster_health": getClusterHealth( - zkhandler, node_list, vm_list, ceph_osd_list - ), + "cluster_health": getClusterHealthFromFaults(zkhandler), "node_health": getNodeHealth(zkhandler, node_list), "maintenance": maintenance_state, "primary_node": primary_node, diff --git a/daemon-common/faults.py b/daemon-common/faults.py index f8a9d0e0..2a97743d 100644 --- a/daemon-common/faults.py +++ b/daemon-common/faults.py @@ -37,6 +37,10 @@ def getFault(zkhandler, fault_id): fault_delta = int(zkhandler.read(("faults.delta", fault_id))) fault_message = zkhandler.read(("faults.message", fault_id)) + # Acknowledged faults have a delta of 0 + if fault_ack_time != "": + fault_delta = 0 + fault = { "id": fault_id, "last_reported": fault_last_time,