Port cluster management functions

This commit is contained in:
Joshua Boniface 2023-05-02 22:31:44 -04:00
parent e294e1c087
commit 59c9d89986
4 changed files with 436 additions and 220 deletions

View File

@ -20,7 +20,9 @@
############################################################################### ###############################################################################
from functools import wraps from functools import wraps
from json import dump as jdump
from json import dumps as jdumps from json import dumps as jdumps
from json import loads as jloads
from os import environ, makedirs, path from os import environ, makedirs, path
from pkg_resources import get_distribution from pkg_resources import get_distribution
@ -126,7 +128,12 @@ def connection_req(function):
@wraps(function) @wraps(function)
def validate_connection(*args, **kwargs): def validate_connection(*args, **kwargs):
if CLI_CONFIG.get("badcfg", None): if CLI_CONFIG.get("badcfg", None) and CLI_CONFIG.get("connection"):
echo(
f"""Invalid connection "{CLI_CONFIG.get('connection')}" specified; set a valid connection and try again."""
)
exit(1)
elif CLI_CONFIG.get("badcfg", None):
echo( echo(
'No connection specified and no local API configuration found. Use "pvc connection" to add a connection.' 'No connection specified and no local API configuration found. Use "pvc connection" to add a connection.'
) )
@ -142,11 +149,11 @@ def connection_req(function):
echo( echo(
f'''Using connection "{CLI_CONFIG.get('connection')}" - Host: "{CLI_CONFIG.get('api_host')}" Scheme: "{CLI_CONFIG.get('api_scheme')}{ssl_verify_msg}" Prefix: "{CLI_CONFIG.get('api_prefix')}"''', f'''Using connection "{CLI_CONFIG.get('connection')}" - Host: "{CLI_CONFIG.get('api_host')}" Scheme: "{CLI_CONFIG.get('api_scheme')}{ssl_verify_msg}" Prefix: "{CLI_CONFIG.get('api_prefix')}"''',
stderr=True, err=True,
) )
echo( echo(
"", "",
stderr=True, err=True,
) )
return function(*args, **kwargs) return function(*args, **kwargs)
@ -308,12 +315,206 @@ def testing(vm, restart_flag, format_function):
finish(True, data, format_function) finish(True, data, format_function)
###############################################################################
# pvc cluster
###############################################################################
@click.group(
name="cluster",
short_help="Manage PVC cluster.",
context_settings=CONTEXT_SETTINGS,
)
def cli_cluster():
"""
Manage and view status of a PVC cluster.
"""
pass
###############################################################################
# pvc cluster status
###############################################################################
@click.command(
name="status",
short_help="Show cluster status.",
)
@format_opt(
{
"pretty": cli_cluster_status_format_pretty,
"short": cli_cluster_status_format_short,
"json": lambda d: jdumps(d),
"json-pretty": lambda d: jdumps(d, indent=2),
}
)
@connection_req
def cli_cluster_status(format_function):
"""
Show information and health about a PVC cluster.
\b
Format options:
"pretty": Output all details in a nice colourful format.
"short" Output only details about cluster health in a nice colourful format.
"json": Output in unformatted JSON.
"json-pretty": Output in formatted JSON.
"""
retcode, retdata = pvc.lib.cluster.get_info(CLI_CONFIG)
finish(retcode, retdata, format_function)
###############################################################################
# pvc cluster init
###############################################################################
@click.command(
name="init",
short_help="Initialize a new cluster.",
)
@click.option(
"-o",
"--overwrite",
"overwrite_flag",
is_flag=True,
default=False,
help="Remove and overwrite any existing data (DANGEROUS)",
)
@confirm_opt
@connection_req
def cli_cluster_init(overwrite_flag):
"""
Perform initialization of a new PVC cluster.
If the "-o"/"--overwrite" option is specified, all existing data in the cluster will be deleted
before new, empty data is written. THIS IS DANGEROUS. YOU WILL LOSE ALL DATA ON THE CLUSTER. Do
not "--overwrite" to an existing cluster unless you are absolutely sure what you are doing.
It is not advisable to initialize a running cluster as this can cause undefined behaviour.
Instead, stop all node daemons first and start the API daemon manually before running this
command.
"""
echo("Some music while we're Layin' Pipe? https://youtu.be/sw8S_Kv89IU")
retcode, retmsg = pvc.lib.cluster.initialize(CLI_CONFIG, overwrite_flag)
finish(retcode, retmsg)
###############################################################################
# pvc cluster backup
###############################################################################
@click.command(
name="backup",
short_help="Create JSON backup of cluster.",
)
@click.option(
"-f",
"--file",
"filename",
default=None,
type=click.File(mode="w"),
help="Write backup data to this file.",
)
@connection_req
def cli_cluster_backup(filename):
"""
Create a JSON-format backup of the cluster Zookeeper state database.
"""
retcode, retdata = pvc.lib.cluster.backup(CLI_CONFIG)
json_data = jloads(retdata)
if retcode and filename is not None:
jdump(json_data, filename)
finish(retcode, f'''Backup written to file "{filename.name}"''')
else:
finish(retcode, json_data)
###############################################################################
# pvc cluster restore
###############################################################################
@click.command(
name="restore",
short_help="Restore JSON backup to cluster.",
)
@click.option(
"-f",
"--filename",
"filename",
required=True,
default=None,
type=click.File(),
help="Read backup data from this file.",
)
@confirm_opt
@connection_req
def cli_cluster_restore(filename):
"""
Restore a JSON-format backup to the cluster Zookeeper state database.
All existing data in the cluster will be deleted before the restored data is written. THIS IS
DANGEROUS. YOU WILL LOSE ALL (CURRENT) DATA ON THE CLUSTER. Do not restore to an existing
cluster unless you are absolutely sure what you are doing.
It is not advisable to restore to a running cluster as this can cause undefined behaviour.
Instead, stop all node daemons first and start the API daemon manually before running this
command.
"""
###############################################################################
# pvc cluster maintenance
###############################################################################
@click.group(
name="maintenance",
short_help="Manage PVC cluster maintenance state.",
context_settings=CONTEXT_SETTINGS,
)
def cli_cluster_maintenance():
"""
Manage the maintenance mode of a PVC cluster.
"""
pass
###############################################################################
# pvc cluster maintenance on
###############################################################################
@click.command(
name="on",
short_help="Enable cluster maintenance mode.",
)
@connection_req
def cli_cluster_maintenance_on():
"""
Enable maintenance mode on a PVC cluster.
"""
retcode, retdata = pvc.lib.cluster.maintenance_mode(CLI_CONFIG, "true")
finish(retcode, retdata)
###############################################################################
# pvc cluster maintenance off
###############################################################################
@click.command(
name="off",
short_help="Disable cluster maintenance mode.",
)
@connection_req
def cli_cluster_maintenance_off():
"""
Disable maintenance mode on a PVC cluster.
"""
retcode, retdata = pvc.lib.cluster.maintenance_mode(CLI_CONFIG, "false")
finish(retcode, retdata)
############################################################################### ###############################################################################
# pvc connection # pvc connection
############################################################################### ###############################################################################
@click.group( @click.group(
name="connection", name="connection",
short_help="Manage PVC cluster connections.", short_help="Manage PVC API connections.",
context_settings=CONTEXT_SETTINGS, context_settings=CONTEXT_SETTINGS,
) )
def cli_connection(): def cli_connection():
@ -459,7 +660,7 @@ def cli_connection_list(show_keys_flag, format_function):
\b \b
Format options: Format options:
"pretty": Output a nice tabular list of all details. "pretty": Output all details in a a nice tabular list format.
"raw": Output connection names one per line. "raw": Output connection names one per line.
"json": Output in unformatted JSON. "json": Output in unformatted JSON.
"json-pretty": Output in formatted JSON. "json-pretty": Output in formatted JSON.
@ -582,11 +783,15 @@ def cli(_connection, _debug, _quiet, _unsafe, _colour):
global CLI_CONFIG global CLI_CONFIG
store_data = get_store(store_path) store_data = get_store(store_path)
CLI_CONFIG = get_config(store_data, _connection)
# There is only one connection and no local connection, so even if nothing was passed, use it # If no connection is specified, use the first connection in the store
if len(store_data) == 1 and _connection is None and CLI_CONFIG.get("badcfg", None): if _connection is None:
CLI_CONFIG = get_config(store_data, list(store_data.keys())[0]) CLI_CONFIG = get_config(store_data, list(store_data.keys())[0])
# If the connection isn't in the store, mark it bad but pass the value
elif _connection not in store_data.keys():
CLI_CONFIG = {"badcfg": True, "connection": _connection}
else:
CLI_CONFIG = get_config(store_data, _connection)
if not CLI_CONFIG.get("badcfg", None): if not CLI_CONFIG.get("badcfg", None):
CLI_CONFIG["debug"] = _debug CLI_CONFIG["debug"] = _debug
@ -601,6 +806,14 @@ def cli(_connection, _debug, _quiet, _unsafe, _colour):
# Click command tree # Click command tree
############################################################################### ###############################################################################
cli_cluster.add_command(cli_cluster_status)
cli_cluster.add_command(cli_cluster_init)
cli_cluster.add_command(cli_cluster_backup)
cli_cluster.add_command(cli_cluster_restore)
cli_cluster_maintenance.add_command(cli_cluster_maintenance_on)
cli_cluster_maintenance.add_command(cli_cluster_maintenance_off)
cli_cluster.add_command(cli_cluster_maintenance)
cli.add_command(cli_cluster)
cli_connection.add_command(cli_connection_add) cli_connection.add_command(cli_connection_add)
cli_connection.add_command(cli_connection_remove) cli_connection.add_command(cli_connection_remove)
cli_connection.add_command(cli_connection_list) cli_connection.add_command(cli_connection_list)

View File

@ -35,6 +35,203 @@ ansii = {
} }
def cli_cluster_status_format_pretty(data):
"""
Pretty format the full output of cli_cluster_status
"""
# Normalize data to local variables
health = data.get("cluster_health", {}).get("health", -1)
messages = data.get("cluster_health", {}).get("messages", None)
maintenance = data.get("maintenance", "N/A")
primary_node = data.get("primary_node", "N/A")
pvc_version = data.get("pvc_version", "N/A")
upstream_ip = data.get("upstream_ip", "N/A")
total_nodes = data.get("nodes", {}).get("total", 0)
total_vms = data.get("vms", {}).get("total", 0)
total_networks = data.get("networks", 0)
total_osds = data.get("osds", {}).get("total", 0)
total_pools = data.get("pools", 0)
total_volumes = data.get("volumes", 0)
total_snapshots = data.get("snapshots", 0)
if maintenance == "true" or health == -1:
health_colour = ansii["blue"]
elif health > 90:
health_colour = ansii["green"]
elif health > 50:
health_colour = ansii["yellow"]
else:
health_colour = ansii["red"]
output = list()
output.append(f"{ansii['bold']}PVC cluster status:{ansii['end']}")
output.append("")
if health != "-1":
health = f"{health}%"
else:
health = "N/A"
if maintenance == "true":
health = f"{health} (maintenance on)"
output.append(
f"{ansii['purple']}Cluster health:{ansii['end']} {health_colour}{health}{ansii['end']}"
)
if messages is not None and len(messages) > 0:
messages = "\n ".join(sorted(messages))
output.append(f"{ansii['purple']}Health messages:{ansii['end']} {messages}")
output.append("")
output.append(f"{ansii['purple']}Primary node:{ansii['end']} {primary_node}")
output.append(f"{ansii['purple']}PVC version:{ansii['end']} {pvc_version}")
output.append(f"{ansii['purple']}Upstream IP:{ansii['end']} {upstream_ip}")
output.append("")
node_states = ["run,ready"]
node_states.extend(
[
state
for state in data.get("nodes", {}).keys()
if state not in ["total", "run,ready"]
]
)
nodes_strings = list()
for state in node_states:
if state in ["run,ready"]:
state_colour = ansii["green"]
elif state in ["run,flush", "run,unflush", "run,flushed"]:
state_colour = ansii["blue"]
elif "dead" in state or "stop" in state:
state_colour = ansii["red"]
else:
state_colour = ansii["yellow"]
nodes_strings.append(
f"{data.get('nodes', {}).get(state)}/{total_nodes} {state_colour}{state}{ansii['end']}"
)
nodes_string = ", ".join(nodes_strings)
output.append(f"{ansii['purple']}Nodes:{ansii['end']} {nodes_string}")
vm_states = ["start", "disable"]
vm_states.extend(
[
state
for state in data.get("vms", {}).keys()
if state not in ["total", "start", "disable"]
]
)
vms_strings = list()
for state in vm_states:
if state in ["start"]:
state_colour = ansii["green"]
elif state in ["migrate", "disable"]:
state_colour = ansii["blue"]
elif state in ["stop", "fail"]:
state_colour = ansii["red"]
else:
state_colour = ansii["yellow"]
vms_strings.append(
f"{data.get('vms', {}).get(state)}/{total_vms} {state_colour}{state}{ansii['end']}"
)
vms_string = ", ".join(vms_strings)
output.append(f"{ansii['purple']}VMs:{ansii['end']} {vms_string}")
osd_states = ["up,in"]
osd_states.extend(
[
state
for state in data.get("osds", {}).keys()
if state not in ["total", "up,in"]
]
)
osds_strings = list()
for state in osd_states:
if state in ["up,in"]:
state_colour = ansii["green"]
elif state in ["down,out"]:
state_colour = ansii["red"]
else:
state_colour = ansii["yellow"]
osds_strings.append(
f"{data.get('osds', {}).get(state)}/{total_osds} {state_colour}{state}{ansii['end']}"
)
osds_string = " ".join(osds_strings)
output.append(f"{ansii['purple']}OSDs:{ansii['end']} {osds_string}")
output.append(f"{ansii['purple']}Pools:{ansii['end']} {total_pools}")
output.append(f"{ansii['purple']}Volumes:{ansii['end']} {total_volumes}")
output.append(f"{ansii['purple']}Snapshots:{ansii['end']} {total_snapshots}")
output.append(f"{ansii['purple']}Networks:{ansii['end']} {total_networks}")
output.append("")
return "\n".join(output)
def cli_cluster_status_format_short(data):
"""
Pretty format the health-only output of cli_cluster_status
"""
# Normalize data to local variables
health = data.get("cluster_health", {}).get("health", -1)
messages = data.get("cluster_health", {}).get("messages", None)
maintenance = data.get("maintenance", "N/A")
if maintenance == "true" or health == -1:
health_colour = ansii["blue"]
elif health > 90:
health_colour = ansii["green"]
elif health > 50:
health_colour = ansii["yellow"]
else:
health_colour = ansii["red"]
output = list()
output.append(f"{ansii['bold']}PVC cluster status:{ansii['end']}")
output.append("")
if health != "-1":
health = f"{health}%"
else:
health = "N/A"
if maintenance == "true":
health = f"{health} (maintenance on)"
output.append(
f"{ansii['purple']}Cluster health:{ansii['end']} {health_colour}{health}{ansii['end']}"
)
if messages is not None and len(messages) > 0:
messages = "\n ".join(sorted(messages))
output.append(f"{ansii['purple']}Health messages:{ansii['end']} {messages}")
output.append("")
return "\n".join(output)
def cli_connection_list_format_pretty(data): def cli_connection_list_format_pretty(data):
""" """
Pretty format the output of cli_connection_list Pretty format the output of cli_connection_list

View File

@ -80,7 +80,7 @@ def read_config_from_yaml(cfgfile):
return cfgfile, host, port, scheme, api_key return cfgfile, host, port, scheme, api_key
def get_config(store_data, cluster=None): def get_config(store_data, connection=None):
""" """
Load CLI configuration from store data Load CLI configuration from store data
""" """
@ -88,38 +88,41 @@ def get_config(store_data, cluster=None):
if store_data is None: if store_data is None:
return {"badcfg": True} return {"badcfg": True}
cluster_details = store_data.get(cluster, None) connection_details = store_data.get(connection, None)
if not cluster_details: if not connection_details:
cluster = "local" connection = "local"
cluster_details = DEFAULT_STORE_DATA connection_details = DEFAULT_STORE_DATA
if cluster_details.get("cfgfile", None) is not None: if connection_details.get("cfgfile", None) is not None:
if path.isfile(cluster_details.get("cfgfile", None)): if path.isfile(connection_details.get("cfgfile", None)):
description, host, port, scheme, api_key = read_config_from_yaml( description, host, port, scheme, api_key = read_config_from_yaml(
cluster_details.get("cfgfile", None) connection_details.get("cfgfile", None)
) )
if None in [description, host, port, scheme]: if None in [description, host, port, scheme]:
return {"badcfg": True} return {"badcfg": True}
else: else:
return {"badcfg": True} return {"badcfg": True}
# Rewrite a wildcard listener to use localhost instead
if host == "0.0.0.0":
host = "127.0.0.1"
else: else:
# This is a static configuration, get the details directly # This is a static configuration, get the details directly
description = cluster_details["description"] description = connection_details["description"]
host = cluster_details["host"] host = connection_details["host"]
port = cluster_details["port"] port = connection_details["port"]
scheme = cluster_details["scheme"] scheme = connection_details["scheme"]
api_key = cluster_details["api_key"] api_key = connection_details["api_key"]
config = dict() config = dict()
config["debug"] = False config["debug"] = False
config["cluster"] = cluster config["connection"] = connection
config["description"] = description config["description"] = description
config["api_host"] = f"{host}:{port}" config["api_host"] = f"{host}:{port}"
config["api_scheme"] = scheme config["api_scheme"] = scheme
config["api_key"] = api_key config["api_key"] = api_key
config["api_prefix"] = DEFAULT_API_PREFIX config["api_prefix"] = DEFAULT_API_PREFIX
if cluster == "local": if connection == "local":
config["verify_ssl"] = False config["verify_ssl"] = False
else: else:
config["verify_ssl"] = bool( config["verify_ssl"] = bool(

View File

@ -21,7 +21,6 @@
import json import json
import pvc.lib.ansiprint as ansiprint
from pvc.lib.common import call_api from pvc.lib.common import call_api
@ -115,199 +114,3 @@ def get_info(config):
return True, response.json() return True, response.json()
else: else:
return False, response.json().get("message", "") return False, response.json().get("message", "")
def format_info(cluster_information, oformat):
if oformat == "json":
return json.dumps(cluster_information)
if oformat == "json-pretty":
return json.dumps(cluster_information, indent=4)
# Plain formatting, i.e. human-readable
if (
cluster_information.get("maintenance") == "true"
or cluster_information.get("cluster_health", {}).get("health", "N/A") == "N/A"
):
health_colour = ansiprint.blue()
elif cluster_information.get("cluster_health", {}).get("health", 100) > 90:
health_colour = ansiprint.green()
elif cluster_information.get("cluster_health", {}).get("health", 100) > 50:
health_colour = ansiprint.yellow()
else:
health_colour = ansiprint.red()
ainformation = []
ainformation.append(
"{}PVC cluster status:{}".format(ansiprint.bold(), ansiprint.end())
)
ainformation.append("")
health_text = (
f"{cluster_information.get('cluster_health', {}).get('health', 'N/A')}"
)
if health_text != "N/A":
health_text += "%"
if cluster_information.get("maintenance") == "true":
health_text += " (maintenance on)"
ainformation.append(
"{}Cluster health:{} {}{}{}".format(
ansiprint.purple(),
ansiprint.end(),
health_colour,
health_text,
ansiprint.end(),
)
)
if cluster_information.get("cluster_health", {}).get("messages"):
health_messages = "\n > ".join(
sorted(cluster_information["cluster_health"]["messages"])
)
ainformation.append(
"{}Health messages:{} > {}".format(
ansiprint.purple(),
ansiprint.end(),
health_messages,
)
)
else:
ainformation.append(
"{}Health messages:{} N/A".format(
ansiprint.purple(),
ansiprint.end(),
)
)
if oformat == "short":
return "\n".join(ainformation)
ainformation.append("")
ainformation.append(
"{}Primary node:{} {}".format(
ansiprint.purple(), ansiprint.end(), cluster_information["primary_node"]
)
)
ainformation.append(
"{}PVC version:{} {}".format(
ansiprint.purple(),
ansiprint.end(),
cluster_information.get("pvc_version", "N/A"),
)
)
ainformation.append(
"{}Cluster upstream IP:{} {}".format(
ansiprint.purple(), ansiprint.end(), cluster_information["upstream_ip"]
)
)
ainformation.append("")
ainformation.append(
"{}Total nodes:{} {}".format(
ansiprint.purple(), ansiprint.end(), cluster_information["nodes"]["total"]
)
)
ainformation.append(
"{}Total VMs:{} {}".format(
ansiprint.purple(), ansiprint.end(), cluster_information["vms"]["total"]
)
)
ainformation.append(
"{}Total networks:{} {}".format(
ansiprint.purple(), ansiprint.end(), cluster_information["networks"]
)
)
ainformation.append(
"{}Total OSDs:{} {}".format(
ansiprint.purple(), ansiprint.end(), cluster_information["osds"]["total"]
)
)
ainformation.append(
"{}Total pools:{} {}".format(
ansiprint.purple(), ansiprint.end(), cluster_information["pools"]
)
)
ainformation.append(
"{}Total volumes:{} {}".format(
ansiprint.purple(), ansiprint.end(), cluster_information["volumes"]
)
)
ainformation.append(
"{}Total snapshots:{} {}".format(
ansiprint.purple(), ansiprint.end(), cluster_information["snapshots"]
)
)
nodes_string = "{}Nodes:{} {}/{} {}ready,run{}".format(
ansiprint.purple(),
ansiprint.end(),
cluster_information["nodes"].get("run,ready", 0),
cluster_information["nodes"].get("total", 0),
ansiprint.green(),
ansiprint.end(),
)
for state, count in cluster_information["nodes"].items():
if state == "total" or state == "run,ready":
continue
nodes_string += " {}/{} {}{}{}".format(
count,
cluster_information["nodes"]["total"],
ansiprint.yellow(),
state,
ansiprint.end(),
)
ainformation.append("")
ainformation.append(nodes_string)
vms_string = "{}VMs:{} {}/{} {}start{}".format(
ansiprint.purple(),
ansiprint.end(),
cluster_information["vms"].get("start", 0),
cluster_information["vms"].get("total", 0),
ansiprint.green(),
ansiprint.end(),
)
for state, count in cluster_information["vms"].items():
if state == "total" or state == "start":
continue
if state in ["disable", "migrate", "unmigrate", "provision"]:
colour = ansiprint.blue()
else:
colour = ansiprint.yellow()
vms_string += " {}/{} {}{}{}".format(
count, cluster_information["vms"]["total"], colour, state, ansiprint.end()
)
ainformation.append("")
ainformation.append(vms_string)
if cluster_information["osds"]["total"] > 0:
osds_string = "{}Ceph OSDs:{} {}/{} {}up,in{}".format(
ansiprint.purple(),
ansiprint.end(),
cluster_information["osds"].get("up,in", 0),
cluster_information["osds"].get("total", 0),
ansiprint.green(),
ansiprint.end(),
)
for state, count in cluster_information["osds"].items():
if state == "total" or state == "up,in":
continue
osds_string += " {}/{} {}{}{}".format(
count,
cluster_information["osds"]["total"],
ansiprint.yellow(),
state,
ansiprint.end(),
)
ainformation.append("")
ainformation.append(osds_string)
ainformation.append("")
return "\n".join(ainformation)