From 59c9d8998687be0234920f1c31fcfea7761a933c Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Tue, 2 May 2023 22:31:44 -0400 Subject: [PATCH] Port cluster management functions --- cli-client-new/pvc/cli/cli.py | 229 ++++++++++++++++++++++++++- cli-client-new/pvc/cli/formatters.py | 197 +++++++++++++++++++++++ cli-client-new/pvc/cli/helpers.py | 33 ++-- cli-client-new/pvc/lib/cluster.py | 197 ----------------------- 4 files changed, 436 insertions(+), 220 deletions(-) diff --git a/cli-client-new/pvc/cli/cli.py b/cli-client-new/pvc/cli/cli.py index 3eb4a4d7..3ce1ea63 100644 --- a/cli-client-new/pvc/cli/cli.py +++ b/cli-client-new/pvc/cli/cli.py @@ -20,7 +20,9 @@ ############################################################################### from functools import wraps +from json import dump as jdump from json import dumps as jdumps +from json import loads as jloads from os import environ, makedirs, path from pkg_resources import get_distribution @@ -126,7 +128,12 @@ def connection_req(function): @wraps(function) def validate_connection(*args, **kwargs): - if CLI_CONFIG.get("badcfg", None): + if CLI_CONFIG.get("badcfg", None) and CLI_CONFIG.get("connection"): + echo( + f"""Invalid connection "{CLI_CONFIG.get('connection')}" specified; set a valid connection and try again.""" + ) + exit(1) + elif CLI_CONFIG.get("badcfg", None): echo( 'No connection specified and no local API configuration found. Use "pvc connection" to add a connection.' ) @@ -142,11 +149,11 @@ def connection_req(function): echo( f'''Using connection "{CLI_CONFIG.get('connection')}" - Host: "{CLI_CONFIG.get('api_host')}" Scheme: "{CLI_CONFIG.get('api_scheme')}{ssl_verify_msg}" Prefix: "{CLI_CONFIG.get('api_prefix')}"''', - stderr=True, + err=True, ) echo( "", - stderr=True, + err=True, ) return function(*args, **kwargs) @@ -308,12 +315,206 @@ def testing(vm, restart_flag, format_function): finish(True, data, format_function) +############################################################################### +# pvc cluster +############################################################################### +@click.group( + name="cluster", + short_help="Manage PVC cluster.", + context_settings=CONTEXT_SETTINGS, +) +def cli_cluster(): + """ + Manage and view status of a PVC cluster. + """ + pass + + +############################################################################### +# pvc cluster status +############################################################################### +@click.command( + name="status", + short_help="Show cluster status.", +) +@format_opt( + { + "pretty": cli_cluster_status_format_pretty, + "short": cli_cluster_status_format_short, + "json": lambda d: jdumps(d), + "json-pretty": lambda d: jdumps(d, indent=2), + } +) +@connection_req +def cli_cluster_status(format_function): + """ + Show information and health about a PVC cluster. + + \b + Format options: + "pretty": Output all details in a nice colourful format. + "short" Output only details about cluster health in a nice colourful format. + "json": Output in unformatted JSON. + "json-pretty": Output in formatted JSON. + """ + + retcode, retdata = pvc.lib.cluster.get_info(CLI_CONFIG) + finish(retcode, retdata, format_function) + + +############################################################################### +# pvc cluster init +############################################################################### +@click.command( + name="init", + short_help="Initialize a new cluster.", +) +@click.option( + "-o", + "--overwrite", + "overwrite_flag", + is_flag=True, + default=False, + help="Remove and overwrite any existing data (DANGEROUS)", +) +@confirm_opt +@connection_req +def cli_cluster_init(overwrite_flag): + """ + Perform initialization of a new PVC cluster. + + If the "-o"/"--overwrite" option is specified, all existing data in the cluster will be deleted + before new, empty data is written. THIS IS DANGEROUS. YOU WILL LOSE ALL DATA ON THE CLUSTER. Do + not "--overwrite" to an existing cluster unless you are absolutely sure what you are doing. + + It is not advisable to initialize a running cluster as this can cause undefined behaviour. + Instead, stop all node daemons first and start the API daemon manually before running this + command. + """ + + echo("Some music while we're Layin' Pipe? https://youtu.be/sw8S_Kv89IU") + + retcode, retmsg = pvc.lib.cluster.initialize(CLI_CONFIG, overwrite_flag) + finish(retcode, retmsg) + + +############################################################################### +# pvc cluster backup +############################################################################### +@click.command( + name="backup", + short_help="Create JSON backup of cluster.", +) +@click.option( + "-f", + "--file", + "filename", + default=None, + type=click.File(mode="w"), + help="Write backup data to this file.", +) +@connection_req +def cli_cluster_backup(filename): + """ + Create a JSON-format backup of the cluster Zookeeper state database. + """ + + retcode, retdata = pvc.lib.cluster.backup(CLI_CONFIG) + json_data = jloads(retdata) + if retcode and filename is not None: + jdump(json_data, filename) + finish(retcode, f'''Backup written to file "{filename.name}"''') + else: + finish(retcode, json_data) + + +############################################################################### +# pvc cluster restore +############################################################################### +@click.command( + name="restore", + short_help="Restore JSON backup to cluster.", +) +@click.option( + "-f", + "--filename", + "filename", + required=True, + default=None, + type=click.File(), + help="Read backup data from this file.", +) +@confirm_opt +@connection_req +def cli_cluster_restore(filename): + """ + Restore a JSON-format backup to the cluster Zookeeper state database. + + All existing data in the cluster will be deleted before the restored data is written. THIS IS + DANGEROUS. YOU WILL LOSE ALL (CURRENT) DATA ON THE CLUSTER. Do not restore to an existing + cluster unless you are absolutely sure what you are doing. + + It is not advisable to restore to a running cluster as this can cause undefined behaviour. + Instead, stop all node daemons first and start the API daemon manually before running this + command. + """ + + +############################################################################### +# pvc cluster maintenance +############################################################################### +@click.group( + name="maintenance", + short_help="Manage PVC cluster maintenance state.", + context_settings=CONTEXT_SETTINGS, +) +def cli_cluster_maintenance(): + """ + Manage the maintenance mode of a PVC cluster. + """ + pass + + +############################################################################### +# pvc cluster maintenance on +############################################################################### +@click.command( + name="on", + short_help="Enable cluster maintenance mode.", +) +@connection_req +def cli_cluster_maintenance_on(): + """ + Enable maintenance mode on a PVC cluster. + """ + + retcode, retdata = pvc.lib.cluster.maintenance_mode(CLI_CONFIG, "true") + finish(retcode, retdata) + + +############################################################################### +# pvc cluster maintenance off +############################################################################### +@click.command( + name="off", + short_help="Disable cluster maintenance mode.", +) +@connection_req +def cli_cluster_maintenance_off(): + """ + Disable maintenance mode on a PVC cluster. + """ + + retcode, retdata = pvc.lib.cluster.maintenance_mode(CLI_CONFIG, "false") + finish(retcode, retdata) + + ############################################################################### # pvc connection ############################################################################### @click.group( name="connection", - short_help="Manage PVC cluster connections.", + short_help="Manage PVC API connections.", context_settings=CONTEXT_SETTINGS, ) def cli_connection(): @@ -459,7 +660,7 @@ def cli_connection_list(show_keys_flag, format_function): \b Format options: - "pretty": Output a nice tabular list of all details. + "pretty": Output all details in a a nice tabular list format. "raw": Output connection names one per line. "json": Output in unformatted JSON. "json-pretty": Output in formatted JSON. @@ -582,11 +783,15 @@ def cli(_connection, _debug, _quiet, _unsafe, _colour): global CLI_CONFIG store_data = get_store(store_path) - CLI_CONFIG = get_config(store_data, _connection) - # There is only one connection and no local connection, so even if nothing was passed, use it - if len(store_data) == 1 and _connection is None and CLI_CONFIG.get("badcfg", None): + # If no connection is specified, use the first connection in the store + if _connection is None: CLI_CONFIG = get_config(store_data, list(store_data.keys())[0]) + # If the connection isn't in the store, mark it bad but pass the value + elif _connection not in store_data.keys(): + CLI_CONFIG = {"badcfg": True, "connection": _connection} + else: + CLI_CONFIG = get_config(store_data, _connection) if not CLI_CONFIG.get("badcfg", None): CLI_CONFIG["debug"] = _debug @@ -601,6 +806,14 @@ def cli(_connection, _debug, _quiet, _unsafe, _colour): # Click command tree ############################################################################### +cli_cluster.add_command(cli_cluster_status) +cli_cluster.add_command(cli_cluster_init) +cli_cluster.add_command(cli_cluster_backup) +cli_cluster.add_command(cli_cluster_restore) +cli_cluster_maintenance.add_command(cli_cluster_maintenance_on) +cli_cluster_maintenance.add_command(cli_cluster_maintenance_off) +cli_cluster.add_command(cli_cluster_maintenance) +cli.add_command(cli_cluster) cli_connection.add_command(cli_connection_add) cli_connection.add_command(cli_connection_remove) cli_connection.add_command(cli_connection_list) diff --git a/cli-client-new/pvc/cli/formatters.py b/cli-client-new/pvc/cli/formatters.py index 77857bb8..a4cd95c9 100644 --- a/cli-client-new/pvc/cli/formatters.py +++ b/cli-client-new/pvc/cli/formatters.py @@ -35,6 +35,203 @@ ansii = { } +def cli_cluster_status_format_pretty(data): + """ + Pretty format the full output of cli_cluster_status + """ + + # Normalize data to local variables + health = data.get("cluster_health", {}).get("health", -1) + messages = data.get("cluster_health", {}).get("messages", None) + maintenance = data.get("maintenance", "N/A") + primary_node = data.get("primary_node", "N/A") + pvc_version = data.get("pvc_version", "N/A") + upstream_ip = data.get("upstream_ip", "N/A") + total_nodes = data.get("nodes", {}).get("total", 0) + total_vms = data.get("vms", {}).get("total", 0) + total_networks = data.get("networks", 0) + total_osds = data.get("osds", {}).get("total", 0) + total_pools = data.get("pools", 0) + total_volumes = data.get("volumes", 0) + total_snapshots = data.get("snapshots", 0) + + if maintenance == "true" or health == -1: + health_colour = ansii["blue"] + elif health > 90: + health_colour = ansii["green"] + elif health > 50: + health_colour = ansii["yellow"] + else: + health_colour = ansii["red"] + + output = list() + + output.append(f"{ansii['bold']}PVC cluster status:{ansii['end']}") + output.append("") + + if health != "-1": + health = f"{health}%" + else: + health = "N/A" + + if maintenance == "true": + health = f"{health} (maintenance on)" + + output.append( + f"{ansii['purple']}Cluster health:{ansii['end']} {health_colour}{health}{ansii['end']}" + ) + + if messages is not None and len(messages) > 0: + messages = "\n ".join(sorted(messages)) + output.append(f"{ansii['purple']}Health messages:{ansii['end']} {messages}") + + output.append("") + + output.append(f"{ansii['purple']}Primary node:{ansii['end']} {primary_node}") + output.append(f"{ansii['purple']}PVC version:{ansii['end']} {pvc_version}") + output.append(f"{ansii['purple']}Upstream IP:{ansii['end']} {upstream_ip}") + output.append("") + + node_states = ["run,ready"] + node_states.extend( + [ + state + for state in data.get("nodes", {}).keys() + if state not in ["total", "run,ready"] + ] + ) + + nodes_strings = list() + for state in node_states: + if state in ["run,ready"]: + state_colour = ansii["green"] + elif state in ["run,flush", "run,unflush", "run,flushed"]: + state_colour = ansii["blue"] + elif "dead" in state or "stop" in state: + state_colour = ansii["red"] + else: + state_colour = ansii["yellow"] + + nodes_strings.append( + f"{data.get('nodes', {}).get(state)}/{total_nodes} {state_colour}{state}{ansii['end']}" + ) + + nodes_string = ", ".join(nodes_strings) + + output.append(f"{ansii['purple']}Nodes:{ansii['end']} {nodes_string}") + + vm_states = ["start", "disable"] + vm_states.extend( + [ + state + for state in data.get("vms", {}).keys() + if state not in ["total", "start", "disable"] + ] + ) + + vms_strings = list() + for state in vm_states: + if state in ["start"]: + state_colour = ansii["green"] + elif state in ["migrate", "disable"]: + state_colour = ansii["blue"] + elif state in ["stop", "fail"]: + state_colour = ansii["red"] + else: + state_colour = ansii["yellow"] + + vms_strings.append( + f"{data.get('vms', {}).get(state)}/{total_vms} {state_colour}{state}{ansii['end']}" + ) + + vms_string = ", ".join(vms_strings) + + output.append(f"{ansii['purple']}VMs:{ansii['end']} {vms_string}") + + osd_states = ["up,in"] + osd_states.extend( + [ + state + for state in data.get("osds", {}).keys() + if state not in ["total", "up,in"] + ] + ) + + osds_strings = list() + for state in osd_states: + if state in ["up,in"]: + state_colour = ansii["green"] + elif state in ["down,out"]: + state_colour = ansii["red"] + else: + state_colour = ansii["yellow"] + + osds_strings.append( + f"{data.get('osds', {}).get(state)}/{total_osds} {state_colour}{state}{ansii['end']}" + ) + + osds_string = " ".join(osds_strings) + + output.append(f"{ansii['purple']}OSDs:{ansii['end']} {osds_string}") + + output.append(f"{ansii['purple']}Pools:{ansii['end']} {total_pools}") + + output.append(f"{ansii['purple']}Volumes:{ansii['end']} {total_volumes}") + + output.append(f"{ansii['purple']}Snapshots:{ansii['end']} {total_snapshots}") + + output.append(f"{ansii['purple']}Networks:{ansii['end']} {total_networks}") + + output.append("") + + return "\n".join(output) + + +def cli_cluster_status_format_short(data): + """ + Pretty format the health-only output of cli_cluster_status + """ + + # Normalize data to local variables + health = data.get("cluster_health", {}).get("health", -1) + messages = data.get("cluster_health", {}).get("messages", None) + maintenance = data.get("maintenance", "N/A") + + if maintenance == "true" or health == -1: + health_colour = ansii["blue"] + elif health > 90: + health_colour = ansii["green"] + elif health > 50: + health_colour = ansii["yellow"] + else: + health_colour = ansii["red"] + + output = list() + + output.append(f"{ansii['bold']}PVC cluster status:{ansii['end']}") + output.append("") + + if health != "-1": + health = f"{health}%" + else: + health = "N/A" + + if maintenance == "true": + health = f"{health} (maintenance on)" + + output.append( + f"{ansii['purple']}Cluster health:{ansii['end']} {health_colour}{health}{ansii['end']}" + ) + + if messages is not None and len(messages) > 0: + messages = "\n ".join(sorted(messages)) + output.append(f"{ansii['purple']}Health messages:{ansii['end']} {messages}") + + output.append("") + + return "\n".join(output) + + def cli_connection_list_format_pretty(data): """ Pretty format the output of cli_connection_list diff --git a/cli-client-new/pvc/cli/helpers.py b/cli-client-new/pvc/cli/helpers.py index 0c4e45d6..ec3120c7 100644 --- a/cli-client-new/pvc/cli/helpers.py +++ b/cli-client-new/pvc/cli/helpers.py @@ -80,7 +80,7 @@ def read_config_from_yaml(cfgfile): return cfgfile, host, port, scheme, api_key -def get_config(store_data, cluster=None): +def get_config(store_data, connection=None): """ Load CLI configuration from store data """ @@ -88,38 +88,41 @@ def get_config(store_data, cluster=None): if store_data is None: return {"badcfg": True} - cluster_details = store_data.get(cluster, None) + connection_details = store_data.get(connection, None) - if not cluster_details: - cluster = "local" - cluster_details = DEFAULT_STORE_DATA + if not connection_details: + connection = "local" + connection_details = DEFAULT_STORE_DATA - if cluster_details.get("cfgfile", None) is not None: - if path.isfile(cluster_details.get("cfgfile", None)): + if connection_details.get("cfgfile", None) is not None: + if path.isfile(connection_details.get("cfgfile", None)): description, host, port, scheme, api_key = read_config_from_yaml( - cluster_details.get("cfgfile", None) + connection_details.get("cfgfile", None) ) if None in [description, host, port, scheme]: return {"badcfg": True} else: return {"badcfg": True} + # Rewrite a wildcard listener to use localhost instead + if host == "0.0.0.0": + host = "127.0.0.1" else: # This is a static configuration, get the details directly - description = cluster_details["description"] - host = cluster_details["host"] - port = cluster_details["port"] - scheme = cluster_details["scheme"] - api_key = cluster_details["api_key"] + description = connection_details["description"] + host = connection_details["host"] + port = connection_details["port"] + scheme = connection_details["scheme"] + api_key = connection_details["api_key"] config = dict() config["debug"] = False - config["cluster"] = cluster + config["connection"] = connection config["description"] = description config["api_host"] = f"{host}:{port}" config["api_scheme"] = scheme config["api_key"] = api_key config["api_prefix"] = DEFAULT_API_PREFIX - if cluster == "local": + if connection == "local": config["verify_ssl"] = False else: config["verify_ssl"] = bool( diff --git a/cli-client-new/pvc/lib/cluster.py b/cli-client-new/pvc/lib/cluster.py index c93fdb2d..6decfd8b 100644 --- a/cli-client-new/pvc/lib/cluster.py +++ b/cli-client-new/pvc/lib/cluster.py @@ -21,7 +21,6 @@ import json -import pvc.lib.ansiprint as ansiprint from pvc.lib.common import call_api @@ -115,199 +114,3 @@ def get_info(config): return True, response.json() else: return False, response.json().get("message", "") - - -def format_info(cluster_information, oformat): - if oformat == "json": - return json.dumps(cluster_information) - - if oformat == "json-pretty": - return json.dumps(cluster_information, indent=4) - - # Plain formatting, i.e. human-readable - if ( - cluster_information.get("maintenance") == "true" - or cluster_information.get("cluster_health", {}).get("health", "N/A") == "N/A" - ): - health_colour = ansiprint.blue() - elif cluster_information.get("cluster_health", {}).get("health", 100) > 90: - health_colour = ansiprint.green() - elif cluster_information.get("cluster_health", {}).get("health", 100) > 50: - health_colour = ansiprint.yellow() - else: - health_colour = ansiprint.red() - - ainformation = [] - - ainformation.append( - "{}PVC cluster status:{}".format(ansiprint.bold(), ansiprint.end()) - ) - ainformation.append("") - - health_text = ( - f"{cluster_information.get('cluster_health', {}).get('health', 'N/A')}" - ) - if health_text != "N/A": - health_text += "%" - if cluster_information.get("maintenance") == "true": - health_text += " (maintenance on)" - - ainformation.append( - "{}Cluster health:{} {}{}{}".format( - ansiprint.purple(), - ansiprint.end(), - health_colour, - health_text, - ansiprint.end(), - ) - ) - if cluster_information.get("cluster_health", {}).get("messages"): - health_messages = "\n > ".join( - sorted(cluster_information["cluster_health"]["messages"]) - ) - ainformation.append( - "{}Health messages:{} > {}".format( - ansiprint.purple(), - ansiprint.end(), - health_messages, - ) - ) - else: - ainformation.append( - "{}Health messages:{} N/A".format( - ansiprint.purple(), - ansiprint.end(), - ) - ) - - if oformat == "short": - return "\n".join(ainformation) - - ainformation.append("") - ainformation.append( - "{}Primary node:{} {}".format( - ansiprint.purple(), ansiprint.end(), cluster_information["primary_node"] - ) - ) - ainformation.append( - "{}PVC version:{} {}".format( - ansiprint.purple(), - ansiprint.end(), - cluster_information.get("pvc_version", "N/A"), - ) - ) - ainformation.append( - "{}Cluster upstream IP:{} {}".format( - ansiprint.purple(), ansiprint.end(), cluster_information["upstream_ip"] - ) - ) - ainformation.append("") - ainformation.append( - "{}Total nodes:{} {}".format( - ansiprint.purple(), ansiprint.end(), cluster_information["nodes"]["total"] - ) - ) - ainformation.append( - "{}Total VMs:{} {}".format( - ansiprint.purple(), ansiprint.end(), cluster_information["vms"]["total"] - ) - ) - ainformation.append( - "{}Total networks:{} {}".format( - ansiprint.purple(), ansiprint.end(), cluster_information["networks"] - ) - ) - ainformation.append( - "{}Total OSDs:{} {}".format( - ansiprint.purple(), ansiprint.end(), cluster_information["osds"]["total"] - ) - ) - ainformation.append( - "{}Total pools:{} {}".format( - ansiprint.purple(), ansiprint.end(), cluster_information["pools"] - ) - ) - ainformation.append( - "{}Total volumes:{} {}".format( - ansiprint.purple(), ansiprint.end(), cluster_information["volumes"] - ) - ) - ainformation.append( - "{}Total snapshots:{} {}".format( - ansiprint.purple(), ansiprint.end(), cluster_information["snapshots"] - ) - ) - - nodes_string = "{}Nodes:{} {}/{} {}ready,run{}".format( - ansiprint.purple(), - ansiprint.end(), - cluster_information["nodes"].get("run,ready", 0), - cluster_information["nodes"].get("total", 0), - ansiprint.green(), - ansiprint.end(), - ) - for state, count in cluster_information["nodes"].items(): - if state == "total" or state == "run,ready": - continue - - nodes_string += " {}/{} {}{}{}".format( - count, - cluster_information["nodes"]["total"], - ansiprint.yellow(), - state, - ansiprint.end(), - ) - - ainformation.append("") - ainformation.append(nodes_string) - - vms_string = "{}VMs:{} {}/{} {}start{}".format( - ansiprint.purple(), - ansiprint.end(), - cluster_information["vms"].get("start", 0), - cluster_information["vms"].get("total", 0), - ansiprint.green(), - ansiprint.end(), - ) - for state, count in cluster_information["vms"].items(): - if state == "total" or state == "start": - continue - - if state in ["disable", "migrate", "unmigrate", "provision"]: - colour = ansiprint.blue() - else: - colour = ansiprint.yellow() - - vms_string += " {}/{} {}{}{}".format( - count, cluster_information["vms"]["total"], colour, state, ansiprint.end() - ) - - ainformation.append("") - ainformation.append(vms_string) - - if cluster_information["osds"]["total"] > 0: - osds_string = "{}Ceph OSDs:{} {}/{} {}up,in{}".format( - ansiprint.purple(), - ansiprint.end(), - cluster_information["osds"].get("up,in", 0), - cluster_information["osds"].get("total", 0), - ansiprint.green(), - ansiprint.end(), - ) - for state, count in cluster_information["osds"].items(): - if state == "total" or state == "up,in": - continue - - osds_string += " {}/{} {}{}{}".format( - count, - cluster_information["osds"]["total"], - ansiprint.yellow(), - state, - ansiprint.end(), - ) - - ainformation.append("") - ainformation.append(osds_string) - - ainformation.append("") - return "\n".join(ainformation)