Port cluster management functions

cli-client-new/pvc/cli/cli.py

@@ -20,7 +20,9 @@
 ###############################################################################
 
 from functools import wraps
+from json import dump as jdump
 from json import dumps as jdumps
+from json import loads as jloads
 from os import environ, makedirs, path
 from pkg_resources import get_distribution
 
@@ -126,7 +128,12 @@ def connection_req(function):
 
     @wraps(function)
     def validate_connection(*args, **kwargs):
-        if CLI_CONFIG.get("badcfg", None):
+        if CLI_CONFIG.get("badcfg", None) and CLI_CONFIG.get("connection"):
+            echo(
+                f"""Invalid connection "{CLI_CONFIG.get('connection')}" specified; set a valid connection and try again."""
+            )
+            exit(1)
+        elif CLI_CONFIG.get("badcfg", None):
             echo(
                 'No connection specified and no local API configuration found. Use "pvc connection" to add a connection.'
             )
@@ -142,11 +149,11 @@ def connection_req(function):
 
             echo(
                 f'''Using connection "{CLI_CONFIG.get('connection')}" - Host: "{CLI_CONFIG.get('api_host')}"  Scheme: "{CLI_CONFIG.get('api_scheme')}{ssl_verify_msg}"  Prefix: "{CLI_CONFIG.get('api_prefix')}"''',
-                stderr=True,
+                err=True,
             )
             echo(
                 "",
-                stderr=True,
+                err=True,
             )
 
         return function(*args, **kwargs)
@@ -308,12 +315,206 @@ def testing(vm, restart_flag, format_function):
     finish(True, data, format_function)
 
 
+###############################################################################
+# pvc cluster
+###############################################################################
+@click.group(
+    name="cluster",
+    short_help="Manage PVC cluster.",
+    context_settings=CONTEXT_SETTINGS,
+)
+def cli_cluster():
+    """
+    Manage and view status of a PVC cluster.
+    """
+    pass
+
+
+###############################################################################
+# pvc cluster status
+###############################################################################
+@click.command(
+    name="status",
+    short_help="Show cluster status.",
+)
+@format_opt(
+    {
+        "pretty": cli_cluster_status_format_pretty,
+        "short": cli_cluster_status_format_short,
+        "json": lambda d: jdumps(d),
+        "json-pretty": lambda d: jdumps(d, indent=2),
+    }
+)
+@connection_req
+def cli_cluster_status(format_function):
+    """
+    Show information and health about a PVC cluster.
+
+    \b
+    Format options:
+        "pretty": Output all details in a nice colourful format.
+        "short" Output only details about cluster health in a nice colourful format.
+        "json": Output in unformatted JSON.
+        "json-pretty": Output in formatted JSON.
+    """
+
+    retcode, retdata = pvc.lib.cluster.get_info(CLI_CONFIG)
+    finish(retcode, retdata, format_function)
+
+
+###############################################################################
+# pvc cluster init
+###############################################################################
+@click.command(
+    name="init",
+    short_help="Initialize a new cluster.",
+)
+@click.option(
+    "-o",
+    "--overwrite",
+    "overwrite_flag",
+    is_flag=True,
+    default=False,
+    help="Remove and overwrite any existing data (DANGEROUS)",
+)
+@confirm_opt
+@connection_req
+def cli_cluster_init(overwrite_flag):
+    """
+    Perform initialization of a new PVC cluster.
+
+    If the "-o"/"--overwrite" option is specified, all existing data in the cluster will be deleted
+    before new, empty data is written. THIS IS DANGEROUS. YOU WILL LOSE ALL DATA ON THE CLUSTER. Do
+    not "--overwrite" to an existing cluster unless you are absolutely sure what you are doing.
+
+    It is not advisable to initialize a running cluster as this can cause undefined behaviour.
+    Instead, stop all node daemons first and start the API daemon manually before running this
+    command.
+    """
+
+    echo("Some music while we're Layin' Pipe? https://youtu.be/sw8S_Kv89IU")
+
+    retcode, retmsg = pvc.lib.cluster.initialize(CLI_CONFIG, overwrite_flag)
+    finish(retcode, retmsg)
+
+
+###############################################################################
+# pvc cluster backup
+###############################################################################
+@click.command(
+    name="backup",
+    short_help="Create JSON backup of cluster.",
+)
+@click.option(
+    "-f",
+    "--file",
+    "filename",
+    default=None,
+    type=click.File(mode="w"),
+    help="Write backup data to this file.",
+)
+@connection_req
+def cli_cluster_backup(filename):
+    """
+    Create a JSON-format backup of the cluster Zookeeper state database.
+    """
+
+    retcode, retdata = pvc.lib.cluster.backup(CLI_CONFIG)
+    json_data = jloads(retdata)
+    if retcode and filename is not None:
+        jdump(json_data, filename)
+        finish(retcode, f'''Backup written to file "{filename.name}"''')
+    else:
+        finish(retcode, json_data)
+
+
+###############################################################################
+# pvc cluster restore
+###############################################################################
+@click.command(
+    name="restore",
+    short_help="Restore JSON backup to cluster.",
+)
+@click.option(
+    "-f",
+    "--filename",
+    "filename",
+    required=True,
+    default=None,
+    type=click.File(),
+    help="Read backup data from this file.",
+)
+@confirm_opt
+@connection_req
+def cli_cluster_restore(filename):
+    """
+    Restore a JSON-format backup to the cluster Zookeeper state database.
+
+    All existing data in the cluster will be deleted before the restored data is written. THIS IS
+    DANGEROUS. YOU WILL LOSE ALL (CURRENT) DATA ON THE CLUSTER. Do not restore to an existing
+    cluster unless you are absolutely sure what you are doing.
+
+    It is not advisable to restore to a running cluster as this can cause undefined behaviour.
+    Instead, stop all node daemons first and start the API daemon manually before running this
+    command.
+    """
+
+
+###############################################################################
+# pvc cluster maintenance
+###############################################################################
+@click.group(
+    name="maintenance",
+    short_help="Manage PVC cluster maintenance state.",
+    context_settings=CONTEXT_SETTINGS,
+)
+def cli_cluster_maintenance():
+    """
+    Manage the maintenance mode of a PVC cluster.
+    """
+    pass
+
+
+###############################################################################
+# pvc cluster maintenance on
+###############################################################################
+@click.command(
+    name="on",
+    short_help="Enable cluster maintenance mode.",
+)
+@connection_req
+def cli_cluster_maintenance_on():
+    """
+    Enable maintenance mode on a PVC cluster.
+    """
+
+    retcode, retdata = pvc.lib.cluster.maintenance_mode(CLI_CONFIG, "true")
+    finish(retcode, retdata)
+
+
+###############################################################################
+# pvc cluster maintenance off
+###############################################################################
+@click.command(
+    name="off",
+    short_help="Disable cluster maintenance mode.",
+)
+@connection_req
+def cli_cluster_maintenance_off():
+    """
+    Disable maintenance mode on a PVC cluster.
+    """
+
+    retcode, retdata = pvc.lib.cluster.maintenance_mode(CLI_CONFIG, "false")
+    finish(retcode, retdata)
+
+
 ###############################################################################
 # pvc connection
 ###############################################################################
 @click.group(
     name="connection",
-    short_help="Manage PVC cluster connections.",
+    short_help="Manage PVC API connections.",
     context_settings=CONTEXT_SETTINGS,
 )
 def cli_connection():
@@ -459,7 +660,7 @@ def cli_connection_list(show_keys_flag, format_function):
 
     \b
     Format options:
-        "pretty": Output a nice tabular list of all details.
+        "pretty": Output all details in a a nice tabular list format.
         "raw": Output connection names one per line.
         "json": Output in unformatted JSON.
         "json-pretty": Output in formatted JSON.
@@ -582,11 +783,15 @@ def cli(_connection, _debug, _quiet, _unsafe, _colour):
 
     global CLI_CONFIG
     store_data = get_store(store_path)
-    CLI_CONFIG = get_config(store_data, _connection)
 
-    # There is only one connection and no local connection, so even if nothing was passed, use it
+    # If no connection is specified, use the first connection in the store
-    if len(store_data) == 1 and _connection is None and CLI_CONFIG.get("badcfg", None):
+    if _connection is None:
         CLI_CONFIG = get_config(store_data, list(store_data.keys())[0])
+    # If the connection isn't in the store, mark it bad but pass the value
+    elif _connection not in store_data.keys():
+        CLI_CONFIG = {"badcfg": True, "connection": _connection}
+    else:
+        CLI_CONFIG = get_config(store_data, _connection)
 
     if not CLI_CONFIG.get("badcfg", None):
         CLI_CONFIG["debug"] = _debug
@@ -601,6 +806,14 @@ def cli(_connection, _debug, _quiet, _unsafe, _colour):
 # Click command tree
 ###############################################################################
 
+cli_cluster.add_command(cli_cluster_status)
+cli_cluster.add_command(cli_cluster_init)
+cli_cluster.add_command(cli_cluster_backup)
+cli_cluster.add_command(cli_cluster_restore)
+cli_cluster_maintenance.add_command(cli_cluster_maintenance_on)
+cli_cluster_maintenance.add_command(cli_cluster_maintenance_off)
+cli_cluster.add_command(cli_cluster_maintenance)
+cli.add_command(cli_cluster)
 cli_connection.add_command(cli_connection_add)
 cli_connection.add_command(cli_connection_remove)
 cli_connection.add_command(cli_connection_list)

cli-client-new/pvc/cli/formatters.py

@@ -35,6 +35,203 @@ ansii = {
 }
 
 
+def cli_cluster_status_format_pretty(data):
+    """
+    Pretty format the full output of cli_cluster_status
+    """
+
+    # Normalize data to local variables
+    health = data.get("cluster_health", {}).get("health", -1)
+    messages = data.get("cluster_health", {}).get("messages", None)
+    maintenance = data.get("maintenance", "N/A")
+    primary_node = data.get("primary_node", "N/A")
+    pvc_version = data.get("pvc_version", "N/A")
+    upstream_ip = data.get("upstream_ip", "N/A")
+    total_nodes = data.get("nodes", {}).get("total", 0)
+    total_vms = data.get("vms", {}).get("total", 0)
+    total_networks = data.get("networks", 0)
+    total_osds = data.get("osds", {}).get("total", 0)
+    total_pools = data.get("pools", 0)
+    total_volumes = data.get("volumes", 0)
+    total_snapshots = data.get("snapshots", 0)
+
+    if maintenance == "true" or health == -1:
+        health_colour = ansii["blue"]
+    elif health > 90:
+        health_colour = ansii["green"]
+    elif health > 50:
+        health_colour = ansii["yellow"]
+    else:
+        health_colour = ansii["red"]
+
+    output = list()
+
+    output.append(f"{ansii['bold']}PVC cluster status:{ansii['end']}")
+    output.append("")
+
+    if health != "-1":
+        health = f"{health}%"
+    else:
+        health = "N/A"
+
+    if maintenance == "true":
+        health = f"{health} (maintenance on)"
+
+    output.append(
+        f"{ansii['purple']}Cluster health:{ansii['end']}   {health_colour}{health}{ansii['end']}"
+    )
+
+    if messages is not None and len(messages) > 0:
+        messages = "\n                  ".join(sorted(messages))
+        output.append(f"{ansii['purple']}Health messages:{ansii['end']}  {messages}")
+
+    output.append("")
+
+    output.append(f"{ansii['purple']}Primary node:{ansii['end']}     {primary_node}")
+    output.append(f"{ansii['purple']}PVC version:{ansii['end']}      {pvc_version}")
+    output.append(f"{ansii['purple']}Upstream IP:{ansii['end']}      {upstream_ip}")
+    output.append("")
+
+    node_states = ["run,ready"]
+    node_states.extend(
+        [
+            state
+            for state in data.get("nodes", {}).keys()
+            if state not in ["total", "run,ready"]
+        ]
+    )
+
+    nodes_strings = list()
+    for state in node_states:
+        if state in ["run,ready"]:
+            state_colour = ansii["green"]
+        elif state in ["run,flush", "run,unflush", "run,flushed"]:
+            state_colour = ansii["blue"]
+        elif "dead" in state or "stop" in state:
+            state_colour = ansii["red"]
+        else:
+            state_colour = ansii["yellow"]
+
+        nodes_strings.append(
+            f"{data.get('nodes', {}).get(state)}/{total_nodes} {state_colour}{state}{ansii['end']}"
+        )
+
+    nodes_string = ", ".join(nodes_strings)
+
+    output.append(f"{ansii['purple']}Nodes:{ansii['end']}            {nodes_string}")
+
+    vm_states = ["start", "disable"]
+    vm_states.extend(
+        [
+            state
+            for state in data.get("vms", {}).keys()
+            if state not in ["total", "start", "disable"]
+        ]
+    )
+
+    vms_strings = list()
+    for state in vm_states:
+        if state in ["start"]:
+            state_colour = ansii["green"]
+        elif state in ["migrate", "disable"]:
+            state_colour = ansii["blue"]
+        elif state in ["stop", "fail"]:
+            state_colour = ansii["red"]
+        else:
+            state_colour = ansii["yellow"]
+
+        vms_strings.append(
+            f"{data.get('vms', {}).get(state)}/{total_vms} {state_colour}{state}{ansii['end']}"
+        )
+
+    vms_string = ", ".join(vms_strings)
+
+    output.append(f"{ansii['purple']}VMs:{ansii['end']}              {vms_string}")
+
+    osd_states = ["up,in"]
+    osd_states.extend(
+        [
+            state
+            for state in data.get("osds", {}).keys()
+            if state not in ["total", "up,in"]
+        ]
+    )
+
+    osds_strings = list()
+    for state in osd_states:
+        if state in ["up,in"]:
+            state_colour = ansii["green"]
+        elif state in ["down,out"]:
+            state_colour = ansii["red"]
+        else:
+            state_colour = ansii["yellow"]
+
+        osds_strings.append(
+            f"{data.get('osds', {}).get(state)}/{total_osds} {state_colour}{state}{ansii['end']}"
+        )
+
+    osds_string = " ".join(osds_strings)
+
+    output.append(f"{ansii['purple']}OSDs:{ansii['end']}             {osds_string}")
+
+    output.append(f"{ansii['purple']}Pools:{ansii['end']}            {total_pools}")
+
+    output.append(f"{ansii['purple']}Volumes:{ansii['end']}          {total_volumes}")
+
+    output.append(f"{ansii['purple']}Snapshots:{ansii['end']}        {total_snapshots}")
+
+    output.append(f"{ansii['purple']}Networks:{ansii['end']}         {total_networks}")
+
+    output.append("")
+
+    return "\n".join(output)
+
+
+def cli_cluster_status_format_short(data):
+    """
+    Pretty format the health-only output of cli_cluster_status
+    """
+
+    # Normalize data to local variables
+    health = data.get("cluster_health", {}).get("health", -1)
+    messages = data.get("cluster_health", {}).get("messages", None)
+    maintenance = data.get("maintenance", "N/A")
+
+    if maintenance == "true" or health == -1:
+        health_colour = ansii["blue"]
+    elif health > 90:
+        health_colour = ansii["green"]
+    elif health > 50:
+        health_colour = ansii["yellow"]
+    else:
+        health_colour = ansii["red"]
+
+    output = list()
+
+    output.append(f"{ansii['bold']}PVC cluster status:{ansii['end']}")
+    output.append("")
+
+    if health != "-1":
+        health = f"{health}%"
+    else:
+        health = "N/A"
+
+    if maintenance == "true":
+        health = f"{health} (maintenance on)"
+
+    output.append(
+        f"{ansii['purple']}Cluster health:{ansii['end']}   {health_colour}{health}{ansii['end']}"
+    )
+
+    if messages is not None and len(messages) > 0:
+        messages = "\n                  ".join(sorted(messages))
+        output.append(f"{ansii['purple']}Health messages:{ansii['end']}  {messages}")
+
+    output.append("")
+
+    return "\n".join(output)
+
+
 def cli_connection_list_format_pretty(data):
     """
     Pretty format the output of cli_connection_list

cli-client-new/pvc/cli/helpers.py

@@ -80,7 +80,7 @@ def read_config_from_yaml(cfgfile):
     return cfgfile, host, port, scheme, api_key
 
 
-def get_config(store_data, cluster=None):
+def get_config(store_data, connection=None):
     """
     Load CLI configuration from store data
     """
@@ -88,38 +88,41 @@ def get_config(store_data, cluster=None):
     if store_data is None:
         return {"badcfg": True}
 
-    cluster_details = store_data.get(cluster, None)
+    connection_details = store_data.get(connection, None)
 
-    if not cluster_details:
+    if not connection_details:
-        cluster = "local"
+        connection = "local"
-        cluster_details = DEFAULT_STORE_DATA
+        connection_details = DEFAULT_STORE_DATA
 
-    if cluster_details.get("cfgfile", None) is not None:
+    if connection_details.get("cfgfile", None) is not None:
-        if path.isfile(cluster_details.get("cfgfile", None)):
+        if path.isfile(connection_details.get("cfgfile", None)):
             description, host, port, scheme, api_key = read_config_from_yaml(
-                cluster_details.get("cfgfile", None)
+                connection_details.get("cfgfile", None)
             )
             if None in [description, host, port, scheme]:
                 return {"badcfg": True}
         else:
             return {"badcfg": True}
+        # Rewrite a wildcard listener to use localhost instead
+        if host == "0.0.0.0":
+            host = "127.0.0.1"
     else:
         # This is a static configuration, get the details directly
-        description = cluster_details["description"]
+        description = connection_details["description"]
-        host = cluster_details["host"]
+        host = connection_details["host"]
-        port = cluster_details["port"]
+        port = connection_details["port"]
-        scheme = cluster_details["scheme"]
+        scheme = connection_details["scheme"]
-        api_key = cluster_details["api_key"]
+        api_key = connection_details["api_key"]
 
     config = dict()
     config["debug"] = False
-    config["cluster"] = cluster
+    config["connection"] = connection
     config["description"] = description
     config["api_host"] = f"{host}:{port}"
     config["api_scheme"] = scheme
     config["api_key"] = api_key
     config["api_prefix"] = DEFAULT_API_PREFIX
-    if cluster == "local":
+    if connection == "local":
         config["verify_ssl"] = False
     else:
         config["verify_ssl"] = bool(

cli-client-new/pvc/lib/cluster.py

@@ -21,7 +21,6 @@
 
 import json
 
-import pvc.lib.ansiprint as ansiprint
 from pvc.lib.common import call_api
 
 
@@ -115,199 +114,3 @@ def get_info(config):
         return True, response.json()
     else:
         return False, response.json().get("message", "")
-
-
-def format_info(cluster_information, oformat):
-    if oformat == "json":
-        return json.dumps(cluster_information)
-
-    if oformat == "json-pretty":
-        return json.dumps(cluster_information, indent=4)
-
-    # Plain formatting, i.e. human-readable
-    if (
-        cluster_information.get("maintenance") == "true"
-        or cluster_information.get("cluster_health", {}).get("health", "N/A") == "N/A"
-    ):
-        health_colour = ansiprint.blue()
-    elif cluster_information.get("cluster_health", {}).get("health", 100) > 90:
-        health_colour = ansiprint.green()
-    elif cluster_information.get("cluster_health", {}).get("health", 100) > 50:
-        health_colour = ansiprint.yellow()
-    else:
-        health_colour = ansiprint.red()
-
-    ainformation = []
-
-    ainformation.append(
-        "{}PVC cluster status:{}".format(ansiprint.bold(), ansiprint.end())
-    )
-    ainformation.append("")
-
-    health_text = (
-        f"{cluster_information.get('cluster_health', {}).get('health', 'N/A')}"
-    )
-    if health_text != "N/A":
-        health_text += "%"
-    if cluster_information.get("maintenance") == "true":
-        health_text += " (maintenance on)"
-
-    ainformation.append(
-        "{}Cluster health:{}  {}{}{}".format(
-            ansiprint.purple(),
-            ansiprint.end(),
-            health_colour,
-            health_text,
-            ansiprint.end(),
-        )
-    )
-    if cluster_information.get("cluster_health", {}).get("messages"):
-        health_messages = "\n                 > ".join(
-            sorted(cluster_information["cluster_health"]["messages"])
-        )
-        ainformation.append(
-            "{}Health messages:{} > {}".format(
-                ansiprint.purple(),
-                ansiprint.end(),
-                health_messages,
-            )
-        )
-    else:
-        ainformation.append(
-            "{}Health messages:{} N/A".format(
-                ansiprint.purple(),
-                ansiprint.end(),
-            )
-        )
-
-    if oformat == "short":
-        return "\n".join(ainformation)
-
-    ainformation.append("")
-    ainformation.append(
-        "{}Primary node:{}        {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["primary_node"]
-        )
-    )
-    ainformation.append(
-        "{}PVC version:{}         {}".format(
-            ansiprint.purple(),
-            ansiprint.end(),
-            cluster_information.get("pvc_version", "N/A"),
-        )
-    )
-    ainformation.append(
-        "{}Cluster upstream IP:{} {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["upstream_ip"]
-        )
-    )
-    ainformation.append("")
-    ainformation.append(
-        "{}Total nodes:{}     {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["nodes"]["total"]
-        )
-    )
-    ainformation.append(
-        "{}Total VMs:{}       {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["vms"]["total"]
-        )
-    )
-    ainformation.append(
-        "{}Total networks:{}  {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["networks"]
-        )
-    )
-    ainformation.append(
-        "{}Total OSDs:{}      {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["osds"]["total"]
-        )
-    )
-    ainformation.append(
-        "{}Total pools:{}     {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["pools"]
-        )
-    )
-    ainformation.append(
-        "{}Total volumes:{}   {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["volumes"]
-        )
-    )
-    ainformation.append(
-        "{}Total snapshots:{} {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["snapshots"]
-        )
-    )
-
-    nodes_string = "{}Nodes:{} {}/{} {}ready,run{}".format(
-        ansiprint.purple(),
-        ansiprint.end(),
-        cluster_information["nodes"].get("run,ready", 0),
-        cluster_information["nodes"].get("total", 0),
-        ansiprint.green(),
-        ansiprint.end(),
-    )
-    for state, count in cluster_information["nodes"].items():
-        if state == "total" or state == "run,ready":
-            continue
-
-        nodes_string += " {}/{} {}{}{}".format(
-            count,
-            cluster_information["nodes"]["total"],
-            ansiprint.yellow(),
-            state,
-            ansiprint.end(),
-        )
-
-    ainformation.append("")
-    ainformation.append(nodes_string)
-
-    vms_string = "{}VMs:{} {}/{} {}start{}".format(
-        ansiprint.purple(),
-        ansiprint.end(),
-        cluster_information["vms"].get("start", 0),
-        cluster_information["vms"].get("total", 0),
-        ansiprint.green(),
-        ansiprint.end(),
-    )
-    for state, count in cluster_information["vms"].items():
-        if state == "total" or state == "start":
-            continue
-
-        if state in ["disable", "migrate", "unmigrate", "provision"]:
-            colour = ansiprint.blue()
-        else:
-            colour = ansiprint.yellow()
-
-        vms_string += " {}/{} {}{}{}".format(
-            count, cluster_information["vms"]["total"], colour, state, ansiprint.end()
-        )
-
-    ainformation.append("")
-    ainformation.append(vms_string)
-
-    if cluster_information["osds"]["total"] > 0:
-        osds_string = "{}Ceph OSDs:{} {}/{} {}up,in{}".format(
-            ansiprint.purple(),
-            ansiprint.end(),
-            cluster_information["osds"].get("up,in", 0),
-            cluster_information["osds"].get("total", 0),
-            ansiprint.green(),
-            ansiprint.end(),
-        )
-        for state, count in cluster_information["osds"].items():
-            if state == "total" or state == "up,in":
-                continue
-
-            osds_string += " {}/{} {}{}{}".format(
-                count,
-                cluster_information["osds"]["total"],
-                ansiprint.yellow(),
-                state,
-                ansiprint.end(),
-            )
-
-        ainformation.append("")
-        ainformation.append(osds_string)
-
-    ainformation.append("")
-    return "\n".join(ainformation)