Add VM management commands

TODO: Add proper new formatters as required

cli-client-new/pvc.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+# pvc.py - PVC client command-line interface (stub testing interface)
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from pvc.cli.cli import cli
+
+
+#
+# Main entry point
+#
+def main():
+    return cli(obj={})
+
+
+if __name__ == "__main__":
+    main()

cli-client-new/pvc/cli/formatters.py

@@ -0,0 +1,449 @@
+#!/usr/bin/env python3
+
+# formatters.py - PVC Click CLI output formatters library
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2023 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from pvc.lib.node import format_info as node_format_info
+from pvc.lib.node import format_list as node_format_list
+
+
+# Define colour values for use in formatters
+ansii = {
+    "red": "\033[91m",
+    "blue": "\033[94m",
+    "cyan": "\033[96m",
+    "green": "\033[92m",
+    "yellow": "\033[93m",
+    "purple": "\033[95m",
+    "bold": "\033[1m",
+    "end": "\033[0m",
+}
+
+
+def cli_cluster_status_format_pretty(data):
+    """
+    Pretty format the full output of cli_cluster_status
+    """
+
+    # Normalize data to local variables
+    health = data.get("cluster_health", {}).get("health", -1)
+    messages = data.get("cluster_health", {}).get("messages", None)
+    maintenance = data.get("maintenance", "N/A")
+    primary_node = data.get("primary_node", "N/A")
+    pvc_version = data.get("pvc_version", "N/A")
+    upstream_ip = data.get("upstream_ip", "N/A")
+    total_nodes = data.get("nodes", {}).get("total", 0)
+    total_vms = data.get("vms", {}).get("total", 0)
+    total_networks = data.get("networks", 0)
+    total_osds = data.get("osds", {}).get("total", 0)
+    total_pools = data.get("pools", 0)
+    total_volumes = data.get("volumes", 0)
+    total_snapshots = data.get("snapshots", 0)
+
+    if maintenance == "true" or health == -1:
+        health_colour = ansii["blue"]
+    elif health > 90:
+        health_colour = ansii["green"]
+    elif health > 50:
+        health_colour = ansii["yellow"]
+    else:
+        health_colour = ansii["red"]
+
+    output = list()
+
+    output.append(f"{ansii['bold']}PVC cluster status:{ansii['end']}")
+    output.append("")
+
+    if health != "-1":
+        health = f"{health}%"
+    else:
+        health = "N/A"
+
+    if maintenance == "true":
+        health = f"{health} (maintenance on)"
+
+    output.append(
+        f"{ansii['purple']}Cluster health:{ansii['end']}   {health_colour}{health}{ansii['end']}"
+    )
+
+    if messages is not None and len(messages) > 0:
+        messages = "\n                  ".join(sorted(messages))
+        output.append(f"{ansii['purple']}Health messages:{ansii['end']}  {messages}")
+
+    output.append("")
+
+    output.append(f"{ansii['purple']}Primary node:{ansii['end']}     {primary_node}")
+    output.append(f"{ansii['purple']}PVC version:{ansii['end']}      {pvc_version}")
+    output.append(f"{ansii['purple']}Upstream IP:{ansii['end']}      {upstream_ip}")
+    output.append("")
+
+    node_states = ["run,ready"]
+    node_states.extend(
+        [
+            state
+            for state in data.get("nodes", {}).keys()
+            if state not in ["total", "run,ready"]
+        ]
+    )
+
+    nodes_strings = list()
+    for state in node_states:
+        if state in ["run,ready"]:
+            state_colour = ansii["green"]
+        elif state in ["run,flush", "run,unflush", "run,flushed"]:
+            state_colour = ansii["blue"]
+        elif "dead" in state or "stop" in state:
+            state_colour = ansii["red"]
+        else:
+            state_colour = ansii["yellow"]
+
+        nodes_strings.append(
+            f"{data.get('nodes', {}).get(state)}/{total_nodes} {state_colour}{state}{ansii['end']}"
+        )
+
+    nodes_string = ", ".join(nodes_strings)
+
+    output.append(f"{ansii['purple']}Nodes:{ansii['end']}            {nodes_string}")
+
+    vm_states = ["start", "disable"]
+    vm_states.extend(
+        [
+            state
+            for state in data.get("vms", {}).keys()
+            if state not in ["total", "start", "disable"]
+        ]
+    )
+
+    vms_strings = list()
+    for state in vm_states:
+        if state in ["start"]:
+            state_colour = ansii["green"]
+        elif state in ["migrate", "disable"]:
+            state_colour = ansii["blue"]
+        elif state in ["stop", "fail"]:
+            state_colour = ansii["red"]
+        else:
+            state_colour = ansii["yellow"]
+
+        vms_strings.append(
+            f"{data.get('vms', {}).get(state)}/{total_vms} {state_colour}{state}{ansii['end']}"
+        )
+
+    vms_string = ", ".join(vms_strings)
+
+    output.append(f"{ansii['purple']}VMs:{ansii['end']}              {vms_string}")
+
+    osd_states = ["up,in"]
+    osd_states.extend(
+        [
+            state
+            for state in data.get("osds", {}).keys()
+            if state not in ["total", "up,in"]
+        ]
+    )
+
+    osds_strings = list()
+    for state in osd_states:
+        if state in ["up,in"]:
+            state_colour = ansii["green"]
+        elif state in ["down,out"]:
+            state_colour = ansii["red"]
+        else:
+            state_colour = ansii["yellow"]
+
+        osds_strings.append(
+            f"{data.get('osds', {}).get(state)}/{total_osds} {state_colour}{state}{ansii['end']}"
+        )
+
+    osds_string = " ".join(osds_strings)
+
+    output.append(f"{ansii['purple']}OSDs:{ansii['end']}             {osds_string}")
+
+    output.append(f"{ansii['purple']}Pools:{ansii['end']}            {total_pools}")
+
+    output.append(f"{ansii['purple']}Volumes:{ansii['end']}          {total_volumes}")
+
+    output.append(f"{ansii['purple']}Snapshots:{ansii['end']}        {total_snapshots}")
+
+    output.append(f"{ansii['purple']}Networks:{ansii['end']}         {total_networks}")
+
+    output.append("")
+
+    return "\n".join(output)
+
+
+def cli_cluster_status_format_short(data):
+    """
+    Pretty format the health-only output of cli_cluster_status
+    """
+
+    # Normalize data to local variables
+    health = data.get("cluster_health", {}).get("health", -1)
+    messages = data.get("cluster_health", {}).get("messages", None)
+    maintenance = data.get("maintenance", "N/A")
+
+    if maintenance == "true" or health == -1:
+        health_colour = ansii["blue"]
+    elif health > 90:
+        health_colour = ansii["green"]
+    elif health > 50:
+        health_colour = ansii["yellow"]
+    else:
+        health_colour = ansii["red"]
+
+    output = list()
+
+    output.append(f"{ansii['bold']}PVC cluster status:{ansii['end']}")
+    output.append("")
+
+    if health != "-1":
+        health = f"{health}%"
+    else:
+        health = "N/A"
+
+    if maintenance == "true":
+        health = f"{health} (maintenance on)"
+
+    output.append(
+        f"{ansii['purple']}Cluster health:{ansii['end']}   {health_colour}{health}{ansii['end']}"
+    )
+
+    if messages is not None and len(messages) > 0:
+        messages = "\n                  ".join(sorted(messages))
+        output.append(f"{ansii['purple']}Health messages:{ansii['end']}  {messages}")
+
+    output.append("")
+
+    return "\n".join(output)
+
+
+def cli_connection_list_format_pretty(data):
+    """
+    Pretty format the output of cli_connection_list
+    """
+
+    # Set the fields data
+    fields = {
+        "name": {"header": "Name", "length": len("Name") + 1},
+        "description": {"header": "Description", "length": len("Description") + 1},
+        "address": {"header": "Address", "length": len("Address") + 1},
+        "port": {"header": "Port", "length": len("Port") + 1},
+        "scheme": {"header": "Scheme", "length": len("Scheme") + 1},
+        "api_key": {"header": "API Key", "length": len("API Key") + 1},
+    }
+
+    # Parse each connection and adjust field lengths
+    for connection in data:
+        for field, length in [(f, fields[f]["length"]) for f in fields]:
+            _length = len(str(connection[field]))
+            if _length > length:
+                length = len(str(connection[field])) + 1
+
+            fields[field]["length"] = length
+
+    # Create the output object and define the line format
+    output = list()
+    line = "{bold}{name: <{lname}} {desc: <{ldesc}} {addr: <{laddr}} {port: <{lport}} {schm: <{lschm}} {akey: <{lakey}}{end}"
+
+    # Add the header line
+    output.append(
+        line.format(
+            bold=ansii["bold"],
+            end=ansii["end"],
+            name=fields["name"]["header"],
+            lname=fields["name"]["length"],
+            desc=fields["description"]["header"],
+            ldesc=fields["description"]["length"],
+            addr=fields["address"]["header"],
+            laddr=fields["address"]["length"],
+            port=fields["port"]["header"],
+            lport=fields["port"]["length"],
+            schm=fields["scheme"]["header"],
+            lschm=fields["scheme"]["length"],
+            akey=fields["api_key"]["header"],
+            lakey=fields["api_key"]["length"],
+        )
+    )
+
+    # Add a line per connection
+    for connection in data:
+        output.append(
+            line.format(
+                bold="",
+                end="",
+                name=connection["name"],
+                lname=fields["name"]["length"],
+                desc=connection["description"],
+                ldesc=fields["description"]["length"],
+                addr=connection["address"],
+                laddr=fields["address"]["length"],
+                port=connection["port"],
+                lport=fields["port"]["length"],
+                schm=connection["scheme"],
+                lschm=fields["scheme"]["length"],
+                akey=connection["api_key"],
+                lakey=fields["api_key"]["length"],
+            )
+        )
+
+    return "\n".join(output)
+
+
+def cli_connection_detail_format_pretty(data):
+    """
+    Pretty format the output of cli_connection_detail
+    """
+
+    # Set the fields data
+    fields = {
+        "name": {"header": "Name", "length": len("Name") + 1},
+        "description": {"header": "Description", "length": len("Description") + 1},
+        "health": {"header": "Health", "length": len("Health") + 1},
+        "primary_node": {"header": "Primary", "length": len("Primary") + 1},
+        "pvc_version": {"header": "Version", "length": len("Version") + 1},
+        "nodes": {"header": "Nodes", "length": len("Nodes") + 1},
+        "vms": {"header": "VMs", "length": len("VMs") + 1},
+        "networks": {"header": "Networks", "length": len("Networks") + 1},
+        "osds": {"header": "OSDs", "length": len("OSDs") + 1},
+        "pools": {"header": "Pools", "length": len("Pools") + 1},
+        "volumes": {"header": "Volumes", "length": len("Volumes") + 1},
+        "snapshots": {"header": "Snapshots", "length": len("Snapshots") + 1},
+    }
+
+    # Parse each connection and adjust field lengths
+    for connection in data:
+        for field, length in [(f, fields[f]["length"]) for f in fields]:
+            _length = len(str(connection[field]))
+            if _length > length:
+                length = len(str(connection[field])) + 1
+
+            fields[field]["length"] = length
+
+    # Create the output object and define the line format
+    output = list()
+    line = "{bold}{name: <{lname}} {desc: <{ldesc}} {chlth}{hlth: <{lhlth}}{endc} {prin: <{lprin}} {vers: <{lvers}} {nods: <{lnods}} {vms: <{lvms}} {nets: <{lnets}} {osds: <{losds}} {pols: <{lpols}} {vols: <{lvols}} {snts: <{lsnts}}{end}"
+
+    # Add the header line
+    output.append(
+        line.format(
+            bold=ansii["bold"],
+            end=ansii["end"],
+            chlth="",
+            endc="",
+            name=fields["name"]["header"],
+            lname=fields["name"]["length"],
+            desc=fields["description"]["header"],
+            ldesc=fields["description"]["length"],
+            hlth=fields["health"]["header"],
+            lhlth=fields["health"]["length"],
+            prin=fields["primary_node"]["header"],
+            lprin=fields["primary_node"]["length"],
+            vers=fields["pvc_version"]["header"],
+            lvers=fields["pvc_version"]["length"],
+            nods=fields["nodes"]["header"],
+            lnods=fields["nodes"]["length"],
+            vms=fields["vms"]["header"],
+            lvms=fields["vms"]["length"],
+            nets=fields["networks"]["header"],
+            lnets=fields["networks"]["length"],
+            osds=fields["osds"]["header"],
+            losds=fields["osds"]["length"],
+            pols=fields["pools"]["header"],
+            lpols=fields["pools"]["length"],
+            vols=fields["volumes"]["header"],
+            lvols=fields["volumes"]["length"],
+            snts=fields["snapshots"]["header"],
+            lsnts=fields["snapshots"]["length"],
+        )
+    )
+
+    # Add a line per connection
+    for connection in data:
+        if connection["health"] == "N/A":
+            health_value = "N/A"
+            health_colour = ansii["purple"]
+        else:
+            health_value = f"{connection['health']}%"
+            if connection["maintenance"] == "true":
+                health_colour = ansii["blue"]
+            elif connection["health"] > 90:
+                health_colour = ansii["green"]
+            elif connection["health"] > 50:
+                health_colour = ansii["yellow"]
+            else:
+                health_colour = ansii["red"]
+
+        output.append(
+            line.format(
+                bold="",
+                end="",
+                chlth=health_colour,
+                endc=ansii["end"],
+                name=connection["name"],
+                lname=fields["name"]["length"],
+                desc=connection["description"],
+                ldesc=fields["description"]["length"],
+                hlth=health_value,
+                lhlth=fields["health"]["length"],
+                prin=connection["primary_node"],
+                lprin=fields["primary_node"]["length"],
+                vers=connection["pvc_version"],
+                lvers=fields["pvc_version"]["length"],
+                nods=connection["nodes"],
+                lnods=fields["nodes"]["length"],
+                vms=connection["vms"],
+                lvms=fields["vms"]["length"],
+                nets=connection["networks"],
+                lnets=fields["networks"]["length"],
+                osds=connection["osds"],
+                losds=fields["osds"]["length"],
+                pols=connection["pools"],
+                lpols=fields["pools"]["length"],
+                vols=connection["volumes"],
+                lvols=fields["volumes"]["length"],
+                snts=connection["snapshots"],
+                lsnts=fields["snapshots"]["length"],
+            )
+        )
+
+    return "\n".join(output)
+
+
+def cli_node_info_format_pretty(data):
+    """
+    Pretty format the basic output of cli_node_info
+    """
+
+    return node_format_info(data, long_output=False)
+
+
+def cli_node_info_format_long(data):
+    """
+    Pretty format the full output of cli_node_info
+    """
+
+    return node_format_info(data, long_output=True)
+
+
+def cli_node_list_format_pretty(data):
+    """
+    Pretty format the output of cli_node_list
+    """
+
+    return node_format_list(data)

cli-client-new/pvc/cli/helpers.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+
+# helpers.py - PVC Click CLI helper function library
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2023 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from click import echo as click_echo
+from distutils.util import strtobool
+from json import load as jload
+from json import dump as jdump
+from os import chmod, environ, getpid, path
+from socket import gethostname
+from sys import argv
+from syslog import syslog, openlog, closelog, LOG_AUTH
+from yaml import load as yload
+from yaml import BaseLoader
+
+
+DEFAULT_STORE_DATA = {"cfgfile": "/etc/pvc/pvcapid.yaml"}
+DEFAULT_STORE_FILENAME = "pvc.json"
+DEFAULT_API_PREFIX = "/api/v1"
+DEFAULT_NODE_HOSTNAME = gethostname().split(".")[0]
+
+
+def echo(config, message, newline=True, stderr=False):
+    """
+    Output a message with click.echo respecting our configuration
+    """
+
+    if config.get("colour", False):
+        colour = True
+    else:
+        colour = None
+
+    if config.get("silent", False):
+        pass
+    elif config.get("quiet", False) and stderr:
+        pass
+    else:
+        click_echo(message=message, color=colour, nl=newline, err=stderr)
+
+
+def audit():
+    """
+    Log an audit message to the local syslog AUTH facility
+    """
+
+    args = argv
+    args[0] = "pvc"
+    pid = getpid()
+
+    openlog(facility=LOG_AUTH, ident=f"{args[0]}[{pid}]")
+    syslog(
+        f"""client audit: command "{' '.join(args)}" by user {environ.get('USER', None)}"""
+    )
+    closelog()
+
+
+def read_config_from_yaml(cfgfile):
+    """
+    Read the PVC API configuration from the local API configuration file
+    """
+
+    try:
+        with open(cfgfile) as fh:
+            api_config = yload(fh, Loader=BaseLoader)["pvc"]["api"]
+
+        host = api_config["listen_address"]
+        port = api_config["listen_port"]
+        scheme = "https" if strtobool(api_config["ssl"]["enabled"]) else "http"
+        api_key = (
+            api_config["authentication"]["tokens"][0]["token"]
+            if strtobool(api_config["authentication"]["enabled"])
+            else None
+        )
+    except KeyError:
+        host = None
+        port = None
+        scheme = None
+        api_key = None
+
+    return cfgfile, host, port, scheme, api_key
+
+
+def get_config(store_data, connection=None):
+    """
+    Load CLI configuration from store data
+    """
+
+    if store_data is None:
+        return {"badcfg": True}
+
+    connection_details = store_data.get(connection, None)
+
+    if not connection_details:
+        connection = "local"
+        connection_details = DEFAULT_STORE_DATA
+
+    if connection_details.get("cfgfile", None) is not None:
+        if path.isfile(connection_details.get("cfgfile", None)):
+            description, host, port, scheme, api_key = read_config_from_yaml(
+                connection_details.get("cfgfile", None)
+            )
+            if None in [description, host, port, scheme]:
+                return {"badcfg": True}
+        else:
+            return {"badcfg": True}
+        # Rewrite a wildcard listener to use localhost instead
+        if host == "0.0.0.0":
+            host = "127.0.0.1"
+    else:
+        # This is a static configuration, get the details directly
+        description = connection_details["description"]
+        host = connection_details["host"]
+        port = connection_details["port"]
+        scheme = connection_details["scheme"]
+        api_key = connection_details["api_key"]
+
+    config = dict()
+    config["debug"] = False
+    config["connection"] = connection
+    config["description"] = description
+    config["api_host"] = f"{host}:{port}"
+    config["api_scheme"] = scheme
+    config["api_key"] = api_key
+    config["api_prefix"] = DEFAULT_API_PREFIX
+    if connection == "local":
+        config["verify_ssl"] = False
+    else:
+        config["verify_ssl"] = bool(
+            strtobool(environ.get("PVC_CLIENT_VERIFY_SSL", "True"))
+        )
+
+    return config
+
+
+def get_store(store_path):
+    """
+    Load store information from the store path
+    """
+
+    store_file = f"{store_path}/{DEFAULT_STORE_FILENAME}"
+
+    with open(store_file) as fh:
+        try:
+            store_data = jload(fh)
+            return store_data
+        except Exception:
+            return dict()
+
+
+def update_store(store_path, store_data):
+    """
+    Update store information to the store path, creating it (with sensible permissions) if needed
+    """
+
+    store_file = f"{store_path}/{DEFAULT_STORE_FILENAME}"
+
+    if not path.exists(store_file):
+        with open(store_file, "w") as fh:
+            fh.write("")
+        chmod(store_file, int(environ.get("PVC_CLIENT_DB_PERMS", "600"), 8))
+
+    with open(store_file, "w") as fh:
+        jdump(store_data, fh, sort_keys=True, indent=4)

cli-client-new/pvc/cli/parsers.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+# parsers.py - PVC Click CLI data parser function library
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2023 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from os import path
+from re import sub
+
+from pvc.cli.helpers import read_config_from_yaml, get_config
+
+import pvc.lib.cluster
+
+
+def cli_connection_list_parser(connections_config, show_keys_flag):
+    """
+    Parse connections_config into formatable data for cli_connection_list
+    """
+
+    connections_data = list()
+
+    for connection, details in connections_config.items():
+        if details.get("cfgfile", None) is not None:
+            if path.isfile(details.get("cfgfile")):
+                description, address, port, scheme, api_key = read_config_from_yaml(
+                    details.get("cfgfile")
+                )
+            else:
+                continue
+            if not show_keys_flag and api_key is not None:
+                api_key = sub(r"[a-z0-9]", "x", api_key)
+            connections_data.append(
+                {
+                    "name": connection,
+                    "description": description,
+                    "address": address,
+                    "port": port,
+                    "scheme": scheme,
+                    "api_key": api_key,
+                }
+            )
+        else:
+            if not show_keys_flag:
+                details["api_key"] = sub(r"[a-z0-9]", "x", details["api_key"])
+            connections_data.append(
+                {
+                    "name": connection,
+                    "description": details["description"],
+                    "address": details["host"],
+                    "port": details["port"],
+                    "scheme": details["scheme"],
+                    "api_key": details["api_key"],
+                }
+            )
+
+    return connections_data
+
+
+def cli_connection_detail_parser(connections_config):
+    """
+    Parse connections_config into formatable data for cli_connection_detail
+    """
+    connections_data = list()
+    for connection, details in connections_config.items():
+        cluster_config = get_config(connections_config, connection=connection)
+        if cluster_config.get("badcfg", False):
+            continue
+        # Connect to each API and gather cluster status
+        retcode, retdata = pvc.lib.cluster.get_info(cluster_config)
+        if retcode == 0:
+            # Create dummy data of N/A for all fields
+            connections_data.append(
+                {
+                    "name": cluster_config["connection"],
+                    "description": cluster_config["description"],
+                    "health": "N/A",
+                    "maintenance": "N/A",
+                    "primary_node": "N/A",
+                    "pvc_version": "N/A",
+                    "nodes": "N/A",
+                    "vms": "N/A",
+                    "networks": "N/A",
+                    "osds": "N/A",
+                    "pools": "N/A",
+                    "volumes": "N/A",
+                    "snapshots": "N/A",
+                }
+            )
+        else:
+            # Normalize data into nice formattable version
+            connections_data.append(
+                {
+                    "name": cluster_config["connection"],
+                    "description": cluster_config["description"],
+                    "health": retdata.get("cluster_health", {}).get("health", "N/A"),
+                    "maintenance": retdata.get("maintenance", "N/A"),
+                    "primary_node": retdata.get("primary_node", "N/A"),
+                    "pvc_version": retdata.get("pvc_version", "N/A"),
+                    "nodes": retdata.get("nodes", {}).get("total", "N/A"),
+                    "vms": retdata.get("vms", {}).get("total", "N/A"),
+                    "networks": retdata.get("networks", "N/A"),
+                    "osds": retdata.get("osds", {}).get("total", "N/A"),
+                    "pools": retdata.get("pools", "N/A"),
+                    "volumes": retdata.get("volumes", "N/A"),
+                    "snapshots": retdata.get("snapshots", "N/A"),
+                }
+            )
+
+    return connections_data

cli-client-new/pvc/cli/waiters.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+# waiters.py - PVC Click CLI output waiters library
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2023 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from time import sleep, time
+
+from pvc.cli.helpers import echo
+
+import pvc.lib.node
+
+
+def cli_node_waiter(config, node, state_field, state_value):
+    """
+    Wait for state transitions for cli_node tasks
+
+    {node} is the name of the node
+    {state_field} is the node_info field to check for {state_value}
+    {state_value} is the TRANSITIONAL value that, when no longer set, will terminate waiting
+    """
+
+    # Sleep for this long between API polls
+    sleep_time = 1
+
+    # Print a dot after this many {sleep_time}s
+    dot_time = 5
+
+    t_start = time()
+
+    echo(config, "Waiting...", newline=False)
+    sleep(sleep_time)
+
+    count = 0
+    while True:
+        count += 1
+        try:
+            _retcode, _retdata = pvc.lib.node.node_info(config, node)
+            if _retdata[state_field] != state_value:
+                break
+            else:
+                raise ValueError
+        except Exception:
+            sleep(sleep_time)
+            if count % dot_time == 0:
+                echo(config, ".", newline=False)
+
+    t_end = time()
+    echo(config, f" done. [{int(t_end - t_start)}s]")

cli-client-new/pvc/lib/cluster.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+# cluster.py - PVC CLI client function library, cluster management
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import json
+
+from pvc.lib.common import call_api
+
+
+def initialize(config, overwrite=False):
+    """
+    Initialize the PVC cluster
+
+    API endpoint: GET /api/v1/initialize
+    API arguments: overwrite, yes-i-really-mean-it
+    API schema: {json_data_object}
+    """
+    params = {"yes-i-really-mean-it": "yes", "overwrite": overwrite}
+    response = call_api(config, "post", "/initialize", params=params)
+
+    if response.status_code == 200:
+        retstatus = True
+    else:
+        retstatus = False
+
+    return retstatus, response.json().get("message", "")
+
+
+def backup(config):
+    """
+    Get a JSON backup of the cluster
+
+    API endpoint: GET /api/v1/backup
+    API arguments:
+    API schema: {json_data_object}
+    """
+    response = call_api(config, "get", "/backup")
+
+    if response.status_code == 200:
+        return True, response.json()
+    else:
+        return False, response.json().get("message", "")
+
+
+def restore(config, cluster_data):
+    """
+    Restore a JSON backup to the cluster
+
+    API endpoint: POST /api/v1/restore
+    API arguments: yes-i-really-mean-it
+    API schema: {json_data_object}
+    """
+    cluster_data_json = json.dumps(cluster_data)
+
+    params = {"yes-i-really-mean-it": "yes"}
+    data = {"cluster_data": cluster_data_json}
+    response = call_api(config, "post", "/restore", params=params, data=data)
+
+    if response.status_code == 200:
+        retstatus = True
+    else:
+        retstatus = False
+
+    return retstatus, response.json().get("message", "")
+
+
+def maintenance_mode(config, state):
+    """
+    Enable or disable PVC cluster maintenance mode
+
+    API endpoint: POST /api/v1/status
+    API arguments: {state}={state}
+    API schema: {json_data_object}
+    """
+    params = {"state": state}
+    response = call_api(config, "post", "/status", params=params)
+
+    if response.status_code == 200:
+        retstatus = True
+    else:
+        retstatus = False
+
+    return retstatus, response.json().get("message", "")
+
+
+def get_info(config):
+    """
+    Get status of the PVC cluster
+
+    API endpoint: GET /api/v1/status
+    API arguments:
+    API schema: {json_data_object}
+    """
+    response = call_api(config, "get", "/status")
+
+    if response.status_code == 200:
+        return True, response.json()
+    else:
+        return False, response.json().get("message", "")

cli-client-new/pvc/lib/node.py

@@ -0,0 +1,706 @@
+#!/usr/bin/env python3
+
+# node.py - PVC CLI client function library, node management
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import time
+
+import pvc.lib.ansiprint as ansiprint
+from pvc.lib.common import call_api
+
+
+#
+# Primary functions
+#
+def node_coordinator_state(config, node, action):
+    """
+    Set node coordinator state state (primary/secondary)
+
+    API endpoint: POST /api/v1/node/{node}/coordinator-state
+    API arguments: action={action}
+    API schema: {"message": "{data}"}
+    """
+    params = {"state": action}
+    response = call_api(
+        config,
+        "post",
+        "/node/{node}/coordinator-state".format(node=node),
+        params=params,
+    )
+
+    if response.status_code == 200:
+        retstatus = True
+    else:
+        retstatus = False
+
+    return retstatus, response.json().get("message", "")
+
+
+def node_domain_state(config, node, action):
+    """
+    Set node domain state state (flush/ready)
+
+    API endpoint: POST /api/v1/node/{node}/domain-state
+    API arguments: action={action}, wait={wait}
+    API schema: {"message": "{data}"}
+    """
+    params = {"state": action}
+    response = call_api(
+        config, "post", "/node/{node}/domain-state".format(node=node), params=params
+    )
+
+    if response.status_code == 200:
+        retstatus = True
+    else:
+        retstatus = False
+
+    return retstatus, response.json().get("message", "")
+
+
+def view_node_log(config, node, lines=100):
+    """
+    Return node log lines from the API (and display them in a pager in the main CLI)
+
+    API endpoint: GET /node/{node}/log
+    API arguments: lines={lines}
+    API schema: {"name":"{node}","data":"{node_log}"}
+    """
+    params = {"lines": lines}
+    response = call_api(
+        config, "get", "/node/{node}/log".format(node=node), params=params
+    )
+
+    if response.status_code != 200:
+        return False, response.json().get("message", "")
+
+    node_log = response.json()["data"]
+
+    # Shrink the log buffer to length lines
+    shrunk_log = node_log.split("\n")[-lines:]
+    loglines = "\n".join(shrunk_log)
+
+    return True, loglines
+
+
+def follow_node_log(config, node, lines=10):
+    """
+    Return and follow node log lines from the API
+
+    API endpoint: GET /node/{node}/log
+    API arguments: lines={lines}
+    API schema: {"name":"{nodename}","data":"{node_log}"}
+    """
+    # We always grab 200 to match the follow call, but only _show_ `lines` number
+    params = {"lines": 200}
+    response = call_api(
+        config, "get", "/node/{node}/log".format(node=node), params=params
+    )
+
+    if response.status_code != 200:
+        return False, response.json().get("message", "")
+
+    # Shrink the log buffer to length lines
+    node_log = response.json()["data"]
+    shrunk_log = node_log.split("\n")[-int(lines) :]
+    loglines = "\n".join(shrunk_log)
+
+    # Print the initial data and begin following
+    print(loglines, end="")
+    print("\n", end="")
+
+    while True:
+        # Grab the next line set (200 is a reasonable number of lines per half-second; any more are skipped)
+        try:
+            params = {"lines": 200}
+            response = call_api(
+                config, "get", "/node/{node}/log".format(node=node), params=params
+            )
+            new_node_log = response.json()["data"]
+        except Exception:
+            break
+        # Split the new and old log strings into constitutent lines
+        old_node_loglines = node_log.split("\n")
+        new_node_loglines = new_node_log.split("\n")
+
+        # Set the node log to the new log value for the next iteration
+        node_log = new_node_log
+
+        # Get the difference between the two sets of lines
+        old_node_loglines_set = set(old_node_loglines)
+        diff_node_loglines = [
+            x for x in new_node_loglines if x not in old_node_loglines_set
+        ]
+
+        # If there's a difference, print it out
+        if len(diff_node_loglines) > 0:
+            print("\n".join(diff_node_loglines), end="")
+            print("\n", end="")
+
+        # Wait half a second
+        time.sleep(0.5)
+
+    return True, ""
+
+
+def node_info(config, node):
+    """
+    Get information about node
+
+    API endpoint: GET /api/v1/node/{node}
+    API arguments:
+    API schema: {json_data_object}
+    """
+    response = call_api(config, "get", "/node/{node}".format(node=node))
+
+    if response.status_code == 200:
+        if isinstance(response.json(), list) and len(response.json()) != 1:
+            # No exact match, return not found
+            return False, "Node not found."
+        else:
+            # Return a single instance if the response is a list
+            if isinstance(response.json(), list):
+                return True, response.json()[0]
+            # This shouldn't happen, but is here just in case
+            else:
+                return True, response.json()
+    else:
+        return False, response.json().get("message", "")
+
+
+def node_list(
+    config, limit, target_daemon_state, target_coordinator_state, target_domain_state
+):
+    """
+    Get list information about nodes (limited by {limit})
+
+    API endpoint: GET /api/v1/node
+    API arguments: limit={limit}
+    API schema: [{json_data_object},{json_data_object},etc.]
+    """
+    params = dict()
+    if limit:
+        params["limit"] = limit
+    if target_daemon_state:
+        params["daemon_state"] = target_daemon_state
+    if target_coordinator_state:
+        params["coordinator_state"] = target_coordinator_state
+    if target_domain_state:
+        params["domain_state"] = target_domain_state
+
+    response = call_api(config, "get", "/node", params=params)
+
+    if response.status_code == 200:
+        return True, response.json()
+    else:
+        return False, response.json().get("message", "")
+
+
+#
+# Output display functions
+#
+def getOutputColours(node_information):
+    node_health = node_information.get("health", "N/A")
+    if isinstance(node_health, int):
+        if node_health <= 50:
+            health_colour = ansiprint.red()
+        elif node_health <= 90:
+            health_colour = ansiprint.yellow()
+        elif node_health <= 100:
+            health_colour = ansiprint.green()
+        else:
+            health_colour = ansiprint.blue()
+    else:
+        health_colour = ansiprint.blue()
+
+    if node_information["daemon_state"] == "run":
+        daemon_state_colour = ansiprint.green()
+    elif node_information["daemon_state"] == "stop":
+        daemon_state_colour = ansiprint.red()
+    elif node_information["daemon_state"] == "shutdown":
+        daemon_state_colour = ansiprint.yellow()
+    elif node_information["daemon_state"] == "init":
+        daemon_state_colour = ansiprint.yellow()
+    elif node_information["daemon_state"] == "dead":
+        daemon_state_colour = ansiprint.red() + ansiprint.bold()
+    else:
+        daemon_state_colour = ansiprint.blue()
+
+    if node_information["coordinator_state"] == "primary":
+        coordinator_state_colour = ansiprint.green()
+    elif node_information["coordinator_state"] == "secondary":
+        coordinator_state_colour = ansiprint.blue()
+    else:
+        coordinator_state_colour = ansiprint.cyan()
+
+    if node_information["domain_state"] == "ready":
+        domain_state_colour = ansiprint.green()
+    else:
+        domain_state_colour = ansiprint.blue()
+
+    if node_information["memory"]["allocated"] > node_information["memory"]["total"]:
+        mem_allocated_colour = ansiprint.yellow()
+    else:
+        mem_allocated_colour = ""
+
+    if node_information["memory"]["provisioned"] > node_information["memory"]["total"]:
+        mem_provisioned_colour = ansiprint.yellow()
+    else:
+        mem_provisioned_colour = ""
+
+    return (
+        health_colour,
+        daemon_state_colour,
+        coordinator_state_colour,
+        domain_state_colour,
+        mem_allocated_colour,
+        mem_provisioned_colour,
+    )
+
+
+def format_info(node_information, long_output):
+    (
+        health_colour,
+        daemon_state_colour,
+        coordinator_state_colour,
+        domain_state_colour,
+        mem_allocated_colour,
+        mem_provisioned_colour,
+    ) = getOutputColours(node_information)
+
+    # Format a nice output; do this line-by-line then concat the elements at the end
+    ainformation = []
+    # Basic information
+    ainformation.append(
+        "{}Name:{}                  {}".format(
+            ansiprint.purple(),
+            ansiprint.end(),
+            node_information["name"],
+        )
+    )
+    ainformation.append(
+        "{}PVC Version:{}           {}".format(
+            ansiprint.purple(),
+            ansiprint.end(),
+            node_information["pvc_version"],
+        )
+    )
+
+    node_health = node_information.get("health", "N/A")
+    if isinstance(node_health, int):
+        node_health_text = f"{node_health}%"
+    else:
+        node_health_text = node_health
+    ainformation.append(
+        "{}Health:{}                {}{}{}".format(
+            ansiprint.purple(),
+            ansiprint.end(),
+            health_colour,
+            node_health_text,
+            ansiprint.end(),
+        )
+    )
+
+    node_health_details = node_information.get("health_details", [])
+    if long_output:
+        node_health_messages = "\n                       ".join(
+            [f"{plugin['name']}: {plugin['message']}" for plugin in node_health_details]
+        )
+    else:
+        node_health_messages = "\n                       ".join(
+            [
+                f"{plugin['name']}: {plugin['message']}"
+                for plugin in node_health_details
+                if int(plugin.get("health_delta", 0)) > 0
+            ]
+        )
+
+    if len(node_health_messages) > 0:
+        ainformation.append(
+            "{}Health Plugin Details:{} {}".format(
+                ansiprint.purple(), ansiprint.end(), node_health_messages
+            )
+        )
+    ainformation.append("")
+
+    ainformation.append(
+        "{}Daemon State:{}          {}{}{}".format(
+            ansiprint.purple(),
+            ansiprint.end(),
+            daemon_state_colour,
+            node_information["daemon_state"],
+            ansiprint.end(),
+        )
+    )
+    ainformation.append(
+        "{}Coordinator State:{}     {}{}{}".format(
+            ansiprint.purple(),
+            ansiprint.end(),
+            coordinator_state_colour,
+            node_information["coordinator_state"],
+            ansiprint.end(),
+        )
+    )
+    ainformation.append(
+        "{}Domain State:{}          {}{}{}".format(
+            ansiprint.purple(),
+            ansiprint.end(),
+            domain_state_colour,
+            node_information["domain_state"],
+            ansiprint.end(),
+        )
+    )
+    if long_output:
+        ainformation.append("")
+        ainformation.append(
+            "{}Architecture:{}          {}".format(
+                ansiprint.purple(), ansiprint.end(), node_information["arch"]
+            )
+        )
+        ainformation.append(
+            "{}Operating System:{}      {}".format(
+                ansiprint.purple(), ansiprint.end(), node_information["os"]
+            )
+        )
+        ainformation.append(
+            "{}Kernel Version:{}        {}".format(
+                ansiprint.purple(), ansiprint.end(), node_information["kernel"]
+            )
+        )
+    ainformation.append("")
+    ainformation.append(
+        "{}Active VM Count:{}       {}".format(
+            ansiprint.purple(), ansiprint.end(), node_information["domains_count"]
+        )
+    )
+    ainformation.append(
+        "{}Host CPUs:{}             {}".format(
+            ansiprint.purple(), ansiprint.end(), node_information["vcpu"]["total"]
+        )
+    )
+    ainformation.append(
+        "{}vCPUs:{}                 {}".format(
+            ansiprint.purple(), ansiprint.end(), node_information["vcpu"]["allocated"]
+        )
+    )
+    ainformation.append(
+        "{}Load:{}                  {}".format(
+            ansiprint.purple(), ansiprint.end(), node_information["load"]
+        )
+    )
+    ainformation.append(
+        "{}Total RAM (MiB):{}       {}".format(
+            ansiprint.purple(), ansiprint.end(), node_information["memory"]["total"]
+        )
+    )
+    ainformation.append(
+        "{}Used RAM (MiB):{}        {}".format(
+            ansiprint.purple(), ansiprint.end(), node_information["memory"]["used"]
+        )
+    )
+    ainformation.append(
+        "{}Free RAM (MiB):{}        {}".format(
+            ansiprint.purple(), ansiprint.end(), node_information["memory"]["free"]
+        )
+    )
+    ainformation.append(
+        "{}Allocated RAM (MiB):{}   {}{}{}".format(
+            ansiprint.purple(),
+            ansiprint.end(),
+            mem_allocated_colour,
+            node_information["memory"]["allocated"],
+            ansiprint.end(),
+        )
+    )
+    ainformation.append(
+        "{}Provisioned RAM (MiB):{} {}{}{}".format(
+            ansiprint.purple(),
+            ansiprint.end(),
+            mem_provisioned_colour,
+            node_information["memory"]["provisioned"],
+            ansiprint.end(),
+        )
+    )
+
+    # Join it all together
+    ainformation.append("")
+    return "\n".join(ainformation)
+
+
+def format_list(node_list):
+    if node_list == "Node not found.":
+        return node_list
+
+    node_list_output = []
+
+    # Determine optimal column widths
+    node_name_length = 5
+    pvc_version_length = 8
+    health_length = 7
+    daemon_state_length = 7
+    coordinator_state_length = 12
+    domain_state_length = 7
+    domains_count_length = 4
+    cpu_count_length = 6
+    load_length = 5
+    mem_total_length = 6
+    mem_used_length = 5
+    mem_free_length = 5
+    mem_alloc_length = 6
+    mem_prov_length = 5
+    for node_information in node_list:
+        # node_name column
+        _node_name_length = len(node_information["name"]) + 1
+        if _node_name_length > node_name_length:
+            node_name_length = _node_name_length
+        # node_pvc_version column
+        _pvc_version_length = len(node_information.get("pvc_version", "N/A")) + 1
+        if _pvc_version_length > pvc_version_length:
+            pvc_version_length = _pvc_version_length
+        # node_health column
+        node_health = node_information.get("health", "N/A")
+        if isinstance(node_health, int):
+            node_health_text = f"{node_health}%"
+        else:
+            node_health_text = node_health
+        _health_length = len(node_health_text) + 1
+        if _health_length > health_length:
+            health_length = _health_length
+        # daemon_state column
+        _daemon_state_length = len(node_information["daemon_state"]) + 1
+        if _daemon_state_length > daemon_state_length:
+            daemon_state_length = _daemon_state_length
+        # coordinator_state column
+        _coordinator_state_length = len(node_information["coordinator_state"]) + 1
+        if _coordinator_state_length > coordinator_state_length:
+            coordinator_state_length = _coordinator_state_length
+        # domain_state column
+        _domain_state_length = len(node_information["domain_state"]) + 1
+        if _domain_state_length > domain_state_length:
+            domain_state_length = _domain_state_length
+        # domains_count column
+        _domains_count_length = len(str(node_information["domains_count"])) + 1
+        if _domains_count_length > domains_count_length:
+            domains_count_length = _domains_count_length
+        # cpu_count column
+        _cpu_count_length = len(str(node_information["cpu_count"])) + 1
+        if _cpu_count_length > cpu_count_length:
+            cpu_count_length = _cpu_count_length
+        # load column
+        _load_length = len(str(node_information["load"])) + 1
+        if _load_length > load_length:
+            load_length = _load_length
+        # mem_total column
+        _mem_total_length = len(str(node_information["memory"]["total"])) + 1
+        if _mem_total_length > mem_total_length:
+            mem_total_length = _mem_total_length
+        # mem_used column
+        _mem_used_length = len(str(node_information["memory"]["used"])) + 1
+        if _mem_used_length > mem_used_length:
+            mem_used_length = _mem_used_length
+        # mem_free column
+        _mem_free_length = len(str(node_information["memory"]["free"])) + 1
+        if _mem_free_length > mem_free_length:
+            mem_free_length = _mem_free_length
+        # mem_alloc column
+        _mem_alloc_length = len(str(node_information["memory"]["allocated"])) + 1
+        if _mem_alloc_length > mem_alloc_length:
+            mem_alloc_length = _mem_alloc_length
+
+        # mem_prov column
+        _mem_prov_length = len(str(node_information["memory"]["provisioned"])) + 1
+        if _mem_prov_length > mem_prov_length:
+            mem_prov_length = _mem_prov_length
+
+    # Format the string (header)
+    node_list_output.append(
+        "{bold}{node_header: <{node_header_length}} {state_header: <{state_header_length}} {resource_header: <{resource_header_length}} {memory_header: <{memory_header_length}}{end_bold}".format(
+            node_header_length=node_name_length
+            + pvc_version_length
+            + health_length
+            + 2,
+            state_header_length=daemon_state_length
+            + coordinator_state_length
+            + domain_state_length
+            + 2,
+            resource_header_length=domains_count_length
+            + cpu_count_length
+            + load_length
+            + 2,
+            memory_header_length=mem_total_length
+            + mem_used_length
+            + mem_free_length
+            + mem_alloc_length
+            + mem_prov_length
+            + 4,
+            bold=ansiprint.bold(),
+            end_bold=ansiprint.end(),
+            node_header="Nodes "
+            + "".join(
+                [
+                    "-"
+                    for _ in range(
+                        6, node_name_length + pvc_version_length + health_length + 1
+                    )
+                ]
+            ),
+            state_header="States "
+            + "".join(
+                [
+                    "-"
+                    for _ in range(
+                        7,
+                        daemon_state_length
+                        + coordinator_state_length
+                        + domain_state_length
+                        + 1,
+                    )
+                ]
+            ),
+            resource_header="Resources "
+            + "".join(
+                [
+                    "-"
+                    for _ in range(
+                        10, domains_count_length + cpu_count_length + load_length + 1
+                    )
+                ]
+            ),
+            memory_header="Memory (M) "
+            + "".join(
+                [
+                    "-"
+                    for _ in range(
+                        11,
+                        mem_total_length
+                        + mem_used_length
+                        + mem_free_length
+                        + mem_alloc_length
+                        + mem_prov_length
+                        + 3,
+                    )
+                ]
+            ),
+        )
+    )
+
+    node_list_output.append(
+        "{bold}{node_name: <{node_name_length}} {node_pvc_version: <{pvc_version_length}} {node_health: <{health_length}} \
+{daemon_state_colour}{node_daemon_state: <{daemon_state_length}}{end_colour} {coordinator_state_colour}{node_coordinator_state: <{coordinator_state_length}}{end_colour} {domain_state_colour}{node_domain_state: <{domain_state_length}}{end_colour} \
+{node_domains_count: <{domains_count_length}} {node_cpu_count: <{cpu_count_length}} {node_load: <{load_length}} \
+{node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {node_mem_allocated: <{mem_alloc_length}} {node_mem_provisioned: <{mem_prov_length}}{end_bold}".format(
+            node_name_length=node_name_length,
+            pvc_version_length=pvc_version_length,
+            health_length=health_length,
+            daemon_state_length=daemon_state_length,
+            coordinator_state_length=coordinator_state_length,
+            domain_state_length=domain_state_length,
+            domains_count_length=domains_count_length,
+            cpu_count_length=cpu_count_length,
+            load_length=load_length,
+            mem_total_length=mem_total_length,
+            mem_used_length=mem_used_length,
+            mem_free_length=mem_free_length,
+            mem_alloc_length=mem_alloc_length,
+            mem_prov_length=mem_prov_length,
+            bold=ansiprint.bold(),
+            end_bold=ansiprint.end(),
+            daemon_state_colour="",
+            coordinator_state_colour="",
+            domain_state_colour="",
+            end_colour="",
+            node_name="Name",
+            node_pvc_version="Version",
+            node_health="Health",
+            node_daemon_state="Daemon",
+            node_coordinator_state="Coordinator",
+            node_domain_state="Domain",
+            node_domains_count="VMs",
+            node_cpu_count="vCPUs",
+            node_load="Load",
+            node_mem_total="Total",
+            node_mem_used="Used",
+            node_mem_free="Free",
+            node_mem_allocated="Alloc",
+            node_mem_provisioned="Prov",
+        )
+    )
+
+    # Format the string (elements)
+    for node_information in sorted(node_list, key=lambda n: n["name"]):
+        (
+            health_colour,
+            daemon_state_colour,
+            coordinator_state_colour,
+            domain_state_colour,
+            mem_allocated_colour,
+            mem_provisioned_colour,
+        ) = getOutputColours(node_information)
+
+        node_health = node_information.get("health", "N/A")
+        if isinstance(node_health, int):
+            node_health_text = f"{node_health}%"
+        else:
+            node_health_text = node_health
+
+        node_list_output.append(
+            "{bold}{node_name: <{node_name_length}} {node_pvc_version: <{pvc_version_length}} {health_colour}{node_health: <{health_length}}{end_colour} \
+{daemon_state_colour}{node_daemon_state: <{daemon_state_length}}{end_colour} {coordinator_state_colour}{node_coordinator_state: <{coordinator_state_length}}{end_colour} {domain_state_colour}{node_domain_state: <{domain_state_length}}{end_colour} \
+{node_domains_count: <{domains_count_length}} {node_cpu_count: <{cpu_count_length}} {node_load: <{load_length}} \
+{node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {mem_allocated_colour}{node_mem_allocated: <{mem_alloc_length}}{end_colour} {mem_provisioned_colour}{node_mem_provisioned: <{mem_prov_length}}{end_colour}{end_bold}".format(
+                node_name_length=node_name_length,
+                pvc_version_length=pvc_version_length,
+                health_length=health_length,
+                daemon_state_length=daemon_state_length,
+                coordinator_state_length=coordinator_state_length,
+                domain_state_length=domain_state_length,
+                domains_count_length=domains_count_length,
+                cpu_count_length=cpu_count_length,
+                load_length=load_length,
+                mem_total_length=mem_total_length,
+                mem_used_length=mem_used_length,
+                mem_free_length=mem_free_length,
+                mem_alloc_length=mem_alloc_length,
+                mem_prov_length=mem_prov_length,
+                bold="",
+                end_bold="",
+                health_colour=health_colour,
+                daemon_state_colour=daemon_state_colour,
+                coordinator_state_colour=coordinator_state_colour,
+                domain_state_colour=domain_state_colour,
+                mem_allocated_colour=mem_allocated_colour,
+                mem_provisioned_colour=mem_allocated_colour,
+                end_colour=ansiprint.end(),
+                node_name=node_information["name"],
+                node_pvc_version=node_information.get("pvc_version", "N/A"),
+                node_health=node_health_text,
+                node_daemon_state=node_information["daemon_state"],
+                node_coordinator_state=node_information["coordinator_state"],
+                node_domain_state=node_information["domain_state"],
+                node_domains_count=node_information["domains_count"],
+                node_cpu_count=node_information["vcpu"]["allocated"],
+                node_load=node_information["load"],
+                node_mem_total=node_information["memory"]["total"],
+                node_mem_used=node_information["memory"]["used"],
+                node_mem_free=node_information["memory"]["free"],
+                node_mem_allocated=node_information["memory"]["allocated"],
+                node_mem_provisioned=node_information["memory"]["provisioned"],
+            )
+        )
+
+    return "\n".join(node_list_output)

cli-client-new/setup.py

@@ -0,0 +1,20 @@
+from setuptools import setup
+
+setup(
+    name="pvc",
+    version="0.9.63",
+    packages=["pvc", "pvc.lib"],
+    install_requires=[
+        "Click",
+        "PyYAML",
+        "lxml",
+        "colorama",
+        "requests",
+        "requests-toolbelt",
+    ],
+    entry_points={
+        "console_scripts": [
+            "pvc = pvc.pvc:cli",
+        ],
+    },
+)

client-cli/pvc/lib/ansiprint.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+
+# ansiprint.py - Printing function for formatted messages
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import datetime
+
+
+# ANSII colours for output
+def red():
+    return "\033[91m"
+
+
+def blue():
+    return "\033[94m"
+
+
+def cyan():
+    return "\033[96m"
+
+
+def green():
+    return "\033[92m"
+
+
+def yellow():
+    return "\033[93m"
+
+
+def purple():
+    return "\033[95m"
+
+
+def bold():
+    return "\033[1m"
+
+
+def end():
+    return "\033[0m"
+
+
+# Print function
+def echo(message, prefix, state):
+    # Get the date
+    date = "{} - ".format(datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S.%f"))
+    endc = end()
+
+    # Continuation
+    if state == "c":
+        date = ""
+        colour = ""
+        prompt = "    "
+    # OK
+    elif state == "o":
+        colour = green()
+        prompt = ">>> "
+    # Error
+    elif state == "e":
+        colour = red()
+        prompt = ">>> "
+    # Warning
+    elif state == "w":
+        colour = yellow()
+        prompt = ">>> "
+    # Tick
+    elif state == "t":
+        colour = purple()
+        prompt = ">>> "
+    # Information
+    elif state == "i":
+        colour = blue()
+        prompt = ">>> "
+    else:
+        colour = bold()
+        prompt = ">>> "
+
+    # Append space to prefix
+    if prefix != "":
+        prefix = prefix + " "
+
+    print(colour + prompt + endc + date + prefix + message)

client-cli/pvc/lib/common.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+
+# common.py - PVC CLI client function library, Common functions
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import os
+import math
+import time
+import requests
+import click
+from urllib3 import disable_warnings
+
+
+def format_bytes(size_bytes):
+    byte_unit_matrix = {
+        "B": 1,
+        "K": 1024,
+        "M": 1024 * 1024,
+        "G": 1024 * 1024 * 1024,
+        "T": 1024 * 1024 * 1024 * 1024,
+        "P": 1024 * 1024 * 1024 * 1024 * 1024,
+    }
+    human_bytes = "0B"
+    for unit in sorted(byte_unit_matrix, key=byte_unit_matrix.get):
+        formatted_bytes = int(math.ceil(size_bytes / byte_unit_matrix[unit]))
+        if formatted_bytes < 10000:
+            human_bytes = "{}{}".format(formatted_bytes, unit)
+            break
+    return human_bytes
+
+
+def format_metric(integer):
+    integer_unit_matrix = {
+        "": 1,
+        "K": 1000,
+        "M": 1000 * 1000,
+        "B": 1000 * 1000 * 1000,
+        "T": 1000 * 1000 * 1000 * 1000,
+        "Q": 1000 * 1000 * 1000 * 1000 * 1000,
+    }
+    human_integer = "0"
+    for unit in sorted(integer_unit_matrix, key=integer_unit_matrix.get):
+        formatted_integer = int(math.ceil(integer / integer_unit_matrix[unit]))
+        if formatted_integer < 10000:
+            human_integer = "{}{}".format(formatted_integer, unit)
+            break
+    return human_integer
+
+
+class UploadProgressBar(object):
+    def __init__(self, filename, end_message="", end_nl=True):
+        file_size = os.path.getsize(filename)
+        file_size_human = format_bytes(file_size)
+        click.echo("Uploading file (total size {})...".format(file_size_human))
+
+        self.length = file_size
+        self.time_last = int(round(time.time() * 1000)) - 1000
+        self.bytes_last = 0
+        self.bytes_diff = 0
+        self.is_end = False
+
+        self.end_message = end_message
+        self.end_nl = end_nl
+        if not self.end_nl:
+            self.end_suffix = " "
+        else:
+            self.end_suffix = ""
+
+        self.bar = click.progressbar(length=self.length, show_eta=True)
+
+    def update(self, monitor):
+        bytes_cur = monitor.bytes_read
+        self.bytes_diff += bytes_cur - self.bytes_last
+        if self.bytes_last == bytes_cur:
+            self.is_end = True
+        self.bytes_last = bytes_cur
+
+        time_cur = int(round(time.time() * 1000))
+        if (time_cur - 1000) > self.time_last:
+            self.time_last = time_cur
+            self.bar.update(self.bytes_diff)
+            self.bytes_diff = 0
+
+        if self.is_end:
+            self.bar.update(self.bytes_diff)
+            self.bytes_diff = 0
+            click.echo()
+            click.echo()
+            if self.end_message:
+                click.echo(self.end_message + self.end_suffix, nl=self.end_nl)
+
+
+class ErrorResponse(requests.Response):
+    def __init__(self, json_data, status_code):
+        self.json_data = json_data
+        self.status_code = status_code
+
+    def json(self):
+        return self.json_data
+
+
+def call_api(
+    config,
+    operation,
+    request_uri,
+    headers={},
+    params=None,
+    data=None,
+    files=None,
+):
+    # Set the connect timeout to 2 seconds but extremely long (48 hour) data timeout
+    timeout = (2.05, 172800)
+
+    # Craft the URI
+    uri = "{}://{}{}{}".format(
+        config["api_scheme"], config["api_host"], config["api_prefix"], request_uri
+    )
+
+    # Craft the authentication header if required
+    if config["api_key"]:
+        headers["X-Api-Key"] = config["api_key"]
+
+    # Determine the request type and hit the API
+    disable_warnings()
+    try:
+        if operation == "get":
+            response = requests.get(
+                uri,
+                timeout=timeout,
+                headers=headers,
+                params=params,
+                data=data,
+                verify=config["verify_ssl"],
+            )
+        if operation == "post":
+            response = requests.post(
+                uri,
+                timeout=timeout,
+                headers=headers,
+                params=params,
+                data=data,
+                files=files,
+                verify=config["verify_ssl"],
+            )
+        if operation == "put":
+            response = requests.put(
+                uri,
+                timeout=timeout,
+                headers=headers,
+                params=params,
+                data=data,
+                files=files,
+                verify=config["verify_ssl"],
+            )
+        if operation == "patch":
+            response = requests.patch(
+                uri,
+                timeout=timeout,
+                headers=headers,
+                params=params,
+                data=data,
+                verify=config["verify_ssl"],
+            )
+        if operation == "delete":
+            response = requests.delete(
+                uri,
+                timeout=timeout,
+                headers=headers,
+                params=params,
+                data=data,
+                verify=config["verify_ssl"],
+            )
+    except Exception as e:
+        message = "Failed to connect to the API: {}".format(e)
+        response = ErrorResponse({"message": message}, 500)
+
+    # Display debug output
+    if config["debug"]:
+        click.echo("API endpoint: {}".format(uri), err=True)
+        click.echo("Response code: {}".format(response.status_code), err=True)
+        click.echo("Response headers: {}".format(response.headers), err=True)
+        click.echo(err=True)
+
+    # Return the response object
+    return response

client-cli/pvc/lib/zkhandler.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+# zkhandler.py - Secure versioned ZooKeeper updates
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import uuid
+
+
+# Exists function
+def exists(zk_conn, key):
+    stat = zk_conn.exists(key)
+    if stat:
+        return True
+    else:
+        return False
+
+
+# Child list function
+def listchildren(zk_conn, key):
+    children = zk_conn.get_children(key)
+    return children
+
+
+# Delete key function
+def deletekey(zk_conn, key, recursive=True):
+    zk_conn.delete(key, recursive=recursive)
+
+
+# Data read function
+def readdata(zk_conn, key):
+    data_raw = zk_conn.get(key)
+    data = data_raw[0].decode("utf8")
+    return data
+
+
+# Data write function
+def writedata(zk_conn, kv):
+    # Start up a transaction
+    zk_transaction = zk_conn.transaction()
+
+    # Proceed one KV pair at a time
+    for key in sorted(kv):
+        data = kv[key]
+
+        # Check if this key already exists or not
+        if not zk_conn.exists(key):
+            # We're creating a new key
+            zk_transaction.create(key, str(data).encode("utf8"))
+        else:
+            # We're updating a key with version validation
+            orig_data = zk_conn.get(key)
+            version = orig_data[1].version
+
+            # Set what we expect the new version to be
+            new_version = version + 1
+
+            # Update the data
+            zk_transaction.set_data(key, str(data).encode("utf8"))
+
+            # Set up the check
+            try:
+                zk_transaction.check(key, new_version)
+            except TypeError:
+                print('Zookeeper key "{}" does not match expected version'.format(key))
+                return False
+
+    # Commit the transaction
+    try:
+        zk_transaction.commit()
+        return True
+    except Exception:
+        return False
+
+
+# Write lock function
+def writelock(zk_conn, key):
+    lock_id = str(uuid.uuid1())
+    lock = zk_conn.WriteLock("{}".format(key), lock_id)
+    return lock
+
+
+# Read lock function
+def readlock(zk_conn, key):
+    lock_id = str(uuid.uuid1())
+    lock = zk_conn.ReadLock("{}".format(key), lock_id)
+    return lock

client-cli/pvc/pvc.py

@@ -37,13 +37,13 @@ from distutils.util import strtobool
 
 from functools import wraps
 
-import pvc.cli_lib.ansiprint as ansiprint
-import pvc.cli_lib.cluster as pvc_cluster
-import pvc.cli_lib.node as pvc_node
-import pvc.cli_lib.vm as pvc_vm
-import pvc.cli_lib.network as pvc_network
-import pvc.cli_lib.ceph as pvc_ceph
-import pvc.cli_lib.provisioner as pvc_provisioner
+import pvc.lib.ansiprint as ansiprint
+import pvc.lib.cluster as pvc_cluster
+import pvc.lib.node as pvc_node
+import pvc.lib.vm as pvc_vm
+import pvc.lib.network as pvc_network
+import pvc.lib.ceph as pvc_ceph
+import pvc.lib.provisioner as pvc_provisioner
 
 
 myhostname = socket.gethostname().split(".")[0]

client-cli/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup
 setup(
     name="pvc",
     version="0.9.63",
-    packages=["pvc", "pvc.cli_lib"],
+    packages=["pvc", "pvc.lib"],
     install_requires=[
         "Click",
         "PyYAML",

daemon-common/node.py

@@ -127,16 +127,14 @@ def getNodeInformation(zkhandler, node_name):
 def secondary_node(zkhandler, node):
     # Verify node is valid
     if not common.verifyNode(zkhandler, node):
-        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(
-            node
-        )
+        return False, "ERROR: No node named {} is present in the cluster.".format(node)
 
     # Ensure node is a coordinator
     daemon_mode = zkhandler.read(("node.mode", node))
     if daemon_mode == "hypervisor":
         return (
             False,
-            'ERROR: Cannot change coordinator mode on non-coordinator node "{}"'.format(
+            "ERROR: Cannot change coordinator state on non-coordinator node {}".format(
                 node
             ),
         )
@@ -144,14 +142,14 @@ def secondary_node(zkhandler, node):
     # Ensure node is in run daemonstate
     daemon_state = zkhandler.read(("node.state.daemon", node))
     if daemon_state != "run":
-        return False, 'ERROR: Node "{}" is not active'.format(node)
+        return False, "ERROR: Node {} is not active".format(node)
 
     # Get current state
     current_state = zkhandler.read(("node.state.router", node))
     if current_state == "secondary":
-        return True, 'Node "{}" is already in secondary coordinator mode.'.format(node)
+        return True, "Node {} is already in secondary coordinator state.".format(node)
 
-    retmsg = "Setting node {} in secondary coordinator mode.".format(node)
+    retmsg = "Setting node {} in secondary coordinator state.".format(node)
     zkhandler.write([("base.config.primary_node", "none")])
 
     return True, retmsg
@@ -160,16 +158,14 @@ def secondary_node(zkhandler, node):
 def primary_node(zkhandler, node):
     # Verify node is valid
     if not common.verifyNode(zkhandler, node):
-        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(
-            node
-        )
+        return False, "ERROR: No node named {} is present in the cluster.".format(node)
 
     # Ensure node is a coordinator
     daemon_mode = zkhandler.read(("node.mode", node))
     if daemon_mode == "hypervisor":
         return (
             False,
-            'ERROR: Cannot change coordinator mode on non-coordinator node "{}"'.format(
+            "ERROR: Cannot change coordinator state on non-coordinator node {}".format(
                 node
             ),
         )
@@ -177,14 +173,14 @@ def primary_node(zkhandler, node):
     # Ensure node is in run daemonstate
     daemon_state = zkhandler.read(("node.state.daemon", node))
     if daemon_state != "run":
-        return False, 'ERROR: Node "{}" is not active'.format(node)
+        return False, "ERROR: Node {} is not active".format(node)
 
     # Get current state
     current_state = zkhandler.read(("node.state.router", node))
     if current_state == "primary":
-        return True, 'Node "{}" is already in primary coordinator mode.'.format(node)
+        return True, "Node {} is already in primary coordinator state.".format(node)
 
-    retmsg = "Setting node {} in primary coordinator mode.".format(node)
+    retmsg = "Setting node {} in primary coordinator state.".format(node)
     zkhandler.write([("base.config.primary_node", node)])
 
     return True, retmsg
@@ -193,14 +189,12 @@ def primary_node(zkhandler, node):
 def flush_node(zkhandler, node, wait=False):
     # Verify node is valid
     if not common.verifyNode(zkhandler, node):
-        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(
-            node
-        )
+        return False, "ERROR: No node named {} is present in the cluster.".format(node)
 
     if zkhandler.read(("node.state.domain", node)) == "flushed":
-        return True, "Hypervisor {} is already flushed.".format(node)
+        return True, "Node {} is already flushed.".format(node)
 
-    retmsg = "Flushing hypervisor {} of running VMs.".format(node)
+    retmsg = "Removing node {} from active service.".format(node)
 
     # Add the new domain to Zookeeper
     zkhandler.write([(("node.state.domain", node), "flush")])
@@ -208,7 +202,7 @@ def flush_node(zkhandler, node, wait=False):
     if wait:
         while zkhandler.read(("node.state.domain", node)) == "flush":
             time.sleep(1)
-        retmsg = "Flushed hypervisor {} of running VMs.".format(node)
+        retmsg = "Removed node {} from active service.".format(node)
 
     return True, retmsg
 
@@ -216,14 +210,12 @@ def flush_node(zkhandler, node, wait=False):
 def ready_node(zkhandler, node, wait=False):
     # Verify node is valid
     if not common.verifyNode(zkhandler, node):
-        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(
-            node
-        )
+        return False, "ERROR: No node named {} is present in the cluster.".format(node)
 
     if zkhandler.read(("node.state.domain", node)) == "ready":
-        return True, "Hypervisor {} is already ready.".format(node)
+        return True, "Node {} is already ready.".format(node)
 
-    retmsg = "Restoring hypervisor {} to active service.".format(node)
+    retmsg = "Restoring node {} to active service.".format(node)
 
     # Add the new domain to Zookeeper
     zkhandler.write([(("node.state.domain", node), "unflush")])
@@ -231,7 +223,7 @@ def ready_node(zkhandler, node, wait=False):
     if wait:
         while zkhandler.read(("node.state.domain", node)) == "unflush":
             time.sleep(1)
-        retmsg = "Restored hypervisor {} to active service.".format(node)
+        retmsg = "Restored node {} to active service.".format(node)
 
     return True, retmsg
 
@@ -239,9 +231,7 @@ def ready_node(zkhandler, node, wait=False):
 def get_node_log(zkhandler, node, lines=2000):
     # Verify node is valid
     if not common.verifyNode(zkhandler, node):
-        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(
-            node
-        )
+        return False, "ERROR: No node named {} is present in the cluster.".format(node)
 
     # Get the data from ZK
     node_log = zkhandler.read(("logs.messages", node))
@@ -259,14 +249,12 @@ def get_node_log(zkhandler, node, lines=2000):
 def get_info(zkhandler, node):
     # Verify node is valid
     if not common.verifyNode(zkhandler, node):
-        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(
-            node
-        )
+        return False, "ERROR: No node named {} is present in the cluster.".format(node)
 
     # Get information about node in a pretty format
     node_information = getNodeInformation(zkhandler, node)
     if not node_information:
-        return False, 'ERROR: Could not get information about node "{}".'.format(node)
+        return False, "ERROR: Could not get information about node {}.".format(node)
 
     return True, node_information