parallelvirtualcluster
/
pvc
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 111 Wiki Activity

Add node management commands

This commit is contained in:
Joshua Boniface 2023-05-05 02:04:36 -04:00
parent 653b95ee25
commit 776daac267
5 changed files with 619 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

546
cli-client-new/pvc/cli/cli.py
View File

@ -27,9 +27,14 @@ from os import environ, makedirs, path
from pkg_resources import get_distribution
from pvc.cli.helpers import *
from pvc.cli.waiters import *
from pvc.cli.parsers import *
from pvc.cli.formatters import *
import pvc.lib.cluster
import pvc.lib.node
import pvc.lib.provisioner
import click
@ -68,19 +73,6 @@ if not IS_COMPLETION:
###############################################################################
def echo(message, newline=True, err=False):
"""
Output a message with click.echo respecting our configuration
"""
if CLI_CONFIG.get("colour", False):
colour = True
else:
colour = None
click.echo(message=message, color=colour, nl=newline, err=err)
def finish(success=True, data=None, formatter=None):
"""
Output data to the terminal and exit based on code (T/F or integer code)
@ -88,9 +80,9 @@ def finish(success=True, data=None, formatter=None):
if data is not None:
if formatter is not None:
echo(formatter(data))
echo(CLI_CONFIG, formatter(data))
else:
echo(data)
echo(CLI_CONFIG, data)
# Allow passing
if isinstance(success, int):
@ -111,7 +103,7 @@ def version(ctx, param, value):
return
version = get_distribution("pvc").version
echo(f"Parallel Virtual Cluster CLI client version {version}")
echo(CLI_CONFIG, f"Parallel Virtual Cluster CLI client version {version}")
ctx.exit()
@ -130,30 +122,31 @@ def connection_req(function):
def validate_connection(*args, **kwargs):
if CLI_CONFIG.get("badcfg", None) and CLI_CONFIG.get("connection"):
echo(
f"""Invalid connection "{CLI_CONFIG.get('connection')}" specified; set a valid connection and try again."""
CLI_CONFIG,
f"""Invalid connection "{CLI_CONFIG.get('connection')}" specified; set a valid connection and try again.""",
)
exit(1)
elif CLI_CONFIG.get("badcfg", None):
echo(
'No connection specified and no local API configuration found. Use "pvc connection" to add a connection.'
CLI_CONFIG,
'No connection specified and no local API configuration found. Use "pvc connection" to add a connection.',
)
exit(1)
if not CLI_CONFIG.get("quiet", False):
if CLI_CONFIG.get("api_scheme") == "https" and not CLI_CONFIG.get(
"verify_ssl"
):
if CLI_CONFIG.get("api_scheme") == "https" and not CLI_CONFIG.get("verify_ssl"):
ssl_verify_msg = " (unverified)"
else:
ssl_verify_msg = ""
echo(
CLI_CONFIG,
f'''Using connection "{CLI_CONFIG.get('connection')}" - Host: "{CLI_CONFIG.get('api_host')}" Scheme: "{CLI_CONFIG.get('api_scheme')}{ssl_verify_msg}" Prefix: "{CLI_CONFIG.get('api_prefix')}"''',
err=True,
stderr=True,
)
echo(
CLI_CONFIG,
"",
err=True,
stderr=True,
)
return function(*args, **kwargs)
@ -195,7 +188,7 @@ def restart_opt(function):
f"Restart VM {kwargs.get('vm')}", prompt_suffix="? ", abort=True
)
except Exception:
echo("Changes will be applied on next VM start/restart.")
echo(CLI_CONFIG, "Changes will be applied on next VM start/restart.")
kwargs["restart_flag"] = False
return function(*args, **kwargs)
@ -251,12 +244,13 @@ def format_opt(formats, default_format="pretty"):
"""
Click Option Decorator with argument:
Wraps a Click command that can output in multiple formats; {formats} defines a dictionary of
formatting functions for the command with keys as valid format types
formatting functions for the command with keys as valid format types.
e.g. { "json": lambda d: json.dumps(d), "pretty": format_function_pretty, ... }
Injects a "format_function" argument into the function for this purpose.
"""
if default_format not in formats.keys():
echo(f"Fatal code error: {default_format} not in {formats.keys()}")
echo(CLI_CONFIG, f"Fatal code error: {default_format} not in {formats.keys()}")
exit(255)
def format_decorator(function):
@ -302,9 +296,9 @@ def format_opt(formats, default_format="pretty"):
)
# Always in format {arguments}, {options}, {flags}, {format_function}
def testing(vm, restart_flag, format_function):
echo(vm)
echo(restart_flag)
echo(format_function)
echo(CLI_CONFIG, vm)
echo(CLI_CONFIG, restart_flag)
echo(CLI_CONFIG, format_function)
data = {
"athing": "value",
@ -320,12 +314,12 @@ def testing(vm, restart_flag, format_function):
###############################################################################
@click.group(
name="cluster",
short_help="Manage PVC cluster.",
short_help="Manage PVC clusters.",
context_settings=CONTEXT_SETTINGS,
)
def cli_cluster():
"""
Manage and view status of a PVC cluster.
Manage and view the status of a PVC cluster.
"""
pass
@ -346,7 +340,9 @@ def cli_cluster():
}
)
@connection_req
def cli_cluster_status(format_function):
def cli_cluster_status(
format_function,
):
"""
Show information and health about a PVC cluster.
@ -379,7 +375,9 @@ def cli_cluster_status(format_function):
)
@confirm_opt
@connection_req
def cli_cluster_init(overwrite_flag):
def cli_cluster_init(
overwrite_flag,
):
"""
Perform initialization of a new PVC cluster.
@ -392,7 +390,7 @@ def cli_cluster_init(overwrite_flag):
command.
"""
echo("Some music while we're Layin' Pipe? https://youtu.be/sw8S_Kv89IU")
echo(CLI_CONFIG, "Some music while we're Layin' Pipe? https://youtu.be/sw8S_Kv89IU")
retcode, retmsg = pvc.lib.cluster.initialize(CLI_CONFIG, overwrite_flag)
finish(retcode, retmsg)
@ -414,7 +412,9 @@ def cli_cluster_init(overwrite_flag):
help="Write backup data to this file.",
)
@connection_req
def cli_cluster_backup(filename):
def cli_cluster_backup(
filename,
):
"""
Create a JSON-format backup of the cluster Zookeeper state database.
"""
@ -446,7 +446,9 @@ def cli_cluster_backup(filename):
)
@confirm_opt
@connection_req
def cli_cluster_restore(filename):
def cli_cluster_restore(
filename,
):
"""
Restore a JSON-format backup to the cluster Zookeeper state database.
@ -509,6 +511,408 @@ def cli_cluster_maintenance_off():
finish(retcode, retdata)
###############################################################################
# pvc node
###############################################################################
@click.group(
name="node",
short_help="Manage PVC nodes.",
context_settings=CONTEXT_SETTINGS,
)
def cli_node():
"""
Manage and view the status of nodes in a PVC cluster.
"""
pass
###############################################################################
# pvc node primary
###############################################################################
@click.command(
name="primary",
short_help="Set node as primary coordinator.",
)
@click.argument("node")
@click.option(
"-w",
"--wait",
"wait_flag",
default=False,
show_default=True,
is_flag=True,
help="Block waiting for state transition",
)
@connection_req
def cli_node_primary(
node,
wait_flag,
):
"""
Set NODE in primary coordinator state, making it the primary coordinator for the cluster.
"""
# Handle active provisioner task warnings
_, tasks_retdata = pvc.lib.provisioner.task_status(CLI_CONFIG, None)
if len(tasks_retdata) > 0:
echo(
CLI_CONFIG,
f"""\
NOTE: There are currently {len(tasks_retdata)} active or queued provisioner tasks.
These jobs will continue executing, but their status visibility will be lost until
the current primary node returns to primary state.
""",
)
retcode, retdata = pvc.lib.node.node_coordinator_state(CLI_CONFIG, node, "primary")
if not retcode or "already" in retdata:
finish(retcode, retdata)
if wait_flag:
echo(CLI_CONFIG, retdata)
cli_node_waiter(CLI_CONFIG, node, "coordinator_state", "takeover")
retdata = f"Set node {node} in primary coordinator state."
finish(retcode, retdata)
###############################################################################
# pvc node secondary
###############################################################################
@click.command(
name="secondary",
short_help="Set node as secondary coordinator.",
)
@click.argument("node")
@click.option(
"-w",
"--wait",
"wait_flag",
default=False,
show_default=True,
is_flag=True,
help="Block waiting for state transition",
)
@connection_req
def cli_node_secondary(
node,
wait_flag,
):
"""
Set NODE in secondary coordinator state, making another active node the primary node for the cluster.
"""
# Handle active provisioner task warnings
_, tasks_retdata = pvc.lib.provisioner.task_status(CLI_CONFIG, None)
if len(tasks_retdata) > 0:
echo(
CLI_CONFIG,
f"""\
NOTE: There are currently {len(tasks_retdata)} active or queued provisioner tasks.
These jobs will continue executing, but their status visibility will be lost until
the current primary node returns to primary state.
""",
)
retcode, retdata = pvc.lib.node.node_coordinator_state(
CLI_CONFIG, node, "secondary"
)
if not retcode or "already" in retdata:
finish(retcode, retdata)
if wait_flag:
echo(CLI_CONFIG, retdata)
cli_node_waiter(CLI_CONFIG, node, "coordinator_state", "relinquish")
retdata = f"Set node {node} in secondary coordinator state."
finish(retcode, retdata)
###############################################################################
# pvc node flush
###############################################################################
@click.command(
name="flush",
short_help="Take node out of service.",
)
@click.argument("node")
@click.option(
"-w",
"--wait",
"wait_flag",
default=False,
show_default=True,
is_flag=True,
help="Block waiting for state transition",
)
@connection_req
def cli_node_flush(
node,
wait_flag,
):
"""
Take NODE out of service, migrating all VMs on it to other nodes.
"""
retcode, retdata = pvc.lib.node.node_domain_state(CLI_CONFIG, node, "flush")
if not retcode or "already" in retdata:
finish(retcode, retdata)
if wait_flag:
echo(CLI_CONFIG, retdata)
cli_node_waiter(CLI_CONFIG, node, "domain_state", "flush")
retdata = f"Removed node {node} from active service."
finish(retcode, retdata)
###############################################################################
# pvc node ready
###############################################################################
@click.command(
name="ready",
short_help="Restore node to service.",
)
@click.argument("node")
@click.option(
"-w",
"--wait",
"wait_flag",
default=False,
show_default=True,
is_flag=True,
help="Block waiting for state transition",
)
@connection_req
def cli_node_ready(
node,
wait_flag,
):
"""
Restore NODE to service, returning all previous VMs to it from other nodes.
"""
retcode, retdata = pvc.lib.node.node_domain_state(CLI_CONFIG, node, "ready")
if not retcode or "already" in retdata:
finish(retcode, retdata)
if wait_flag:
echo(CLI_CONFIG, retdata)
cli_node_waiter(CLI_CONFIG, node, "domain_state", "unflush")
retdata = f"Restored node {node} to active service."
finish(retcode, retdata)
###############################################################################
# pvc node log
###############################################################################
@click.command(
name="log",
short_help="View node daemon logs.",
)
@click.argument("node")
@click.option(
"-l",
"--lines",
"lines",
default=None,
show_default=False,
help="Display this many log lines from the end of the log buffer. [default: 1000; with follow: 10]",
)
@click.option(
"-f",
"--follow",
"follow_flag",
is_flag=True,
default=False,
help="Follow the live changes of the log buffer.",
)
@connection_req
def cli_node_log(
node,
lines,
follow_flag,
):
"""
Show daemon logs of NODE, either in the local $PAGER tool or following the current output.
If "-f"/"--follow" is used, log output may be delayed by up to 1-2 seconds relative to the
live system due to API refresh delays. Logs will display in batches with each API refresh.
With "--follow", the default "--lines" value is 10, otherwise it is 1000 unless "--lines" is
specified with another value.
The maximum number of lines is limited only by the systemd journal of the node, though values
above ~5000 may cause performance problems.
"""
# Set the default lines value based on the follow option
if lines is None:
if follow_flag:
lines = 10
else:
lines = 1000
if follow_flag:
# This command blocks following the logs until cancelled
retcode, retmsg = pvc.lib.node.follow_node_log(CLI_CONFIG, node, lines)
retmsg = ""
else:
retcode, retmsg = pvc.lib.node.view_node_log(CLI_CONFIG, node, lines)
click.echo_via_pager(retmsg)
retmsg = ""
finish(retcode, retmsg)
###############################################################################
# pvc node info
###############################################################################
@click.command(
name="info",
short_help="Show details of node.",
)
@click.argument("node", default=DEFAULT_NODE_HOSTNAME)
@format_opt(
{
"pretty": cli_node_info_format_pretty,
"long": cli_node_info_format_long,
"json": lambda d: jdumps(d),
"json-pretty": lambda d: jdumps(d, indent=2),
}
)
@connection_req
def cli_node_info(
node,
format_function,
):
"""
Show information about NODE. If a node is not specified, defaults to this host.
\b
Format options:
"pretty": Output basic details in a nice colourful format.
"long" Output full details including all health plugins in a nice colourful format.
"json": Output in unformatted JSON.
"json-pretty": Output in formatted JSON.
"""
retcode, retdata = pvc.lib.node.node_info(CLI_CONFIG, node)
finish(retcode, retdata, format_function)
###############################################################################
# pvc node list
###############################################################################
@click.command(
name="list",
short_help="List all nodes.",
)
@click.argument("limit", default=None, required=False)
@click.option(
"-ds",
"--daemon-state",
"daemon_state_filter",
default=None,
help="Limit list to nodes in the specified daemon state.",
)
@click.option(
"-ds",
"--coordinator-state",
"coordinator_state_filter",
default=None,
help="Limit list to nodes in the specified coordinator state.",
)
@click.option(
"-ds",
"--domain-state",
"domain_state_filter",
default=None,
help="Limit list to nodes in the specified domain state.",
)
@format_opt(
{
"pretty": cli_node_list_format_pretty,
"raw": lambda d: "\n".join([c["name"] for c in d]),
"json": lambda d: jdumps(d),
"json-pretty": lambda d: jdumps(d, indent=2),
}
)
@connection_req
def cli_node_list(
limit,
daemon_state_filter,
coordinator_state_filter,
domain_state_filter,
format_function,
):
"""
List all nodes, optionally only nodes matching regex LIMIT.
\b
Format options:
"pretty": Output all details in a nice tabular list format.
"raw": Output node names one per line.
"json": Output in unformatted JSON.
"json-pretty": Output in formatted JSON.
"""
retcode, retdata = pvc.lib.node.node_list(
CLI_CONFIG,
limit,
daemon_state_filter,
coordinator_state_filter,
domain_state_filter,
)
finish(retcode, retdata, format_function)
###############################################################################
# pvc
###############################################################################
###############################################################################
# pvc
###############################################################################
###############################################################################
# pvc
###############################################################################
###############################################################################
# pvc
###############################################################################
###############################################################################
# pvc
###############################################################################
###############################################################################
# pvc
###############################################################################
###############################################################################
# pvc
###############################################################################
###############################################################################
# pvc
###############################################################################
###############################################################################
# pvc
###############################################################################
###############################################################################
# pvc connection
###############################################################################
@ -572,7 +976,14 @@ def cli_connection():
default=False,
help="Whether or not to use SSL for the API connection. [default: False]",
)
def cli_connection_add(name, description, address, port, api_key, ssl_flag):
def cli_connection_add(
name,
description,
address,
port,
api_key,
ssl_flag,
):
"""
Add the PVC connection NAME to the database of the local CLI client.
@ -611,7 +1022,9 @@ def cli_connection_add(name, description, address, port, api_key, ssl_flag):
short_help="Remove connections from the client database.",
)
@click.argument("name")
def cli_connection_remove(name):
def cli_connection_remove(
name,
):
"""
Remove the PVC connection NAME from the database of the local CLI client.
"""
@ -654,13 +1067,16 @@ def cli_connection_remove(name):
"json-pretty": lambda d: jdumps(d, indent=2),
}
)
def cli_connection_list(show_keys_flag, format_function):
def cli_connection_list(
show_keys_flag,
format_function,
):
"""
List all PVC connections in the database of the local CLI client.
\b
Format options:
"pretty": Output all details in a a nice tabular list format.
"pretty": Output all details in a nice tabular list format.
"raw": Output connection names one per line.
"json": Output in unformatted JSON.
"json-pretty": Output in formatted JSON.
@ -685,7 +1101,9 @@ def cli_connection_list(show_keys_flag, format_function):
"json-pretty": lambda d: jdumps(d, indent=2),
}
)
def cli_connection_detail(format_function):
def cli_connection_detail(
format_function,
):
"""
List the status and information of all PVC cluster in the database of the local CLI client.
@ -696,11 +1114,16 @@ def cli_connection_detail(format_function):
"json-pretty": Output in formatted JSON.
"""
echo("Gathering information from all clusters... ", newline=False, err=True)
echo(
CLI_CONFIG,
"Gathering information from all clusters... ",
newline=False,
stderr=True,
)
connections_config = get_store(store_path)
connections_data = cli_connection_detail_parser(connections_config)
echo("done.", err=True)
echo("", err=True)
echo(CLI_CONFIG, "done.", stderr=True)
echo(CLI_CONFIG, "", stderr=True)
finish(True, connections_data, format_function)
@ -732,7 +1155,16 @@ def cli_connection_detail(format_function):
envvar="PVC_QUIET",
is_flag=True,
default=False,
help="Suppress connection connection information.",
help="Suppress information sent to stderr.",
)
@click.option(
"-s",
"--silent",
"_silent",
envvar="PVC_SILENT",
is_flag=True,
default=False,
help="Suppress information sent to stdout and stderr.",
)
@click.option(
"-u",
@ -760,7 +1192,14 @@ def cli_connection_detail(format_function):
is_eager=True,
help="Show CLI version and exit.",
)
def cli(_connection, _debug, _quiet, _unsafe, _colour):
def cli(
_connection,
_debug,
_quiet,
_silent,
_unsafe,
_colour,
):
"""
Parallel Virtual Cluster CLI management tool
@ -770,7 +1209,9 @@ def cli(_connection, _debug, _quiet, _unsafe, _colour):
"PVC_DEBUG": Enable additional debugging details instead of using --debug/-v
"PVC_QUIET": Suppress stderr connection output from client instead of using --quiet/-q
"PVC_QUIET": Suppress stderr output from client instead of using --quiet/-q
"PVC_SILENT": Suppress stdout and stderr output from client instead of using --silent/-s
"PVC_UNSAFE": Always suppress confirmations instead of needing --unsafe/-u or --yes/-y; USE WITH EXTREME CARE
@ -798,6 +1239,7 @@ def cli(_connection, _debug, _quiet, _unsafe, _colour):
CLI_CONFIG["unsafe"] = _unsafe
CLI_CONFIG["colour"] = _colour
CLI_CONFIG["quiet"] = _quiet
CLI_CONFIG["silent"] = _silent
audit()
@ -806,6 +1248,14 @@ def cli(_connection, _debug, _quiet, _unsafe, _colour):
# Click command tree
###############################################################################
cli_node.add_command(cli_node_primary)
cli_node.add_command(cli_node_secondary)
cli_node.add_command(cli_node_flush)
cli_node.add_command(cli_node_ready)
cli_node.add_command(cli_node_log)
cli_node.add_command(cli_node_info)
cli_node.add_command(cli_node_list)
cli.add_command(cli_node)
cli_cluster.add_command(cli_cluster_status)
cli_cluster.add_command(cli_cluster_init)
cli_cluster.add_command(cli_cluster_backup)

27
cli-client-new/pvc/cli/formatters.py
View File

@ -19,7 +19,8 @@
#
###############################################################################
# import colorama
from pvc.lib.node import format_info as node_format_info
from pvc.lib.node import format_list as node_format_list
# Define colour values for use in formatters
@ -422,3 +423,27 @@ def cli_connection_detail_format_pretty(data):
)
return "\n".join(output)
def cli_node_info_format_pretty(data):
"""
Pretty format the basic output of cli_node_info
"""
return node_format_info(data, long_output=False)
def cli_node_info_format_long(data):
"""
Pretty format the full output of cli_node_info
"""
return node_format_info(data, long_output=True)
def cli_node_list_format_pretty(data):
"""
Pretty format the output of cli_node_list
"""
return node_format_list(data)

21
cli-client-new/pvc/cli/helpers.py
View File

@ -19,7 +19,7 @@
#
###############################################################################
from click import echo
from click import echo as click_echo
from distutils.util import strtobool
from json import load as jload
from json import dump as jdump
@ -37,6 +37,24 @@ DEFAULT_API_PREFIX = "/api/v1"
DEFAULT_NODE_HOSTNAME = gethostname().split(".")[0]
def echo(config, message, newline=True, stderr=False):
"""
Output a message with click.echo respecting our configuration
"""
if config.get("colour", False):
colour = True
else:
colour = None
if config.get("silent", False):
pass
elif config.get("quiet", False) and stderr:
pass
else:
click_echo(message=message, color=colour, nl=newline, err=stderr)
def audit():
"""
Log an audit message to the local syslog AUTH facility
@ -71,7 +89,6 @@ def read_config_from_yaml(cfgfile):
else None
)
except KeyError:
echo("Invalid API YAML found, ignoring.")
host = None
port = None
scheme = None

64
cli-client-new/pvc/cli/waiters.py Normal file
View File

@ -0,0 +1,64 @@
#!/usr/bin/env python3
# waiters.py - PVC Click CLI output waiters library
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2023 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
from time import sleep, time
from pvc.cli.helpers import echo
import pvc.lib.node
def cli_node_waiter(config, node, state_field, state_value):
"""
Wait for state transitions for cli_node tasks
{node} is the name of the node
{state_field} is the node_info field to check for {state_value}
{state_value} is the TRANSITIONAL value that, when no longer set, will terminate waiting
"""
# Sleep for this long between API polls
sleep_time = 1
# Print a dot after this many {sleep_time}s
dot_time = 5
t_start = time()
echo(config, "Waiting...", newline=False)
sleep(sleep_time)
count = 0
while True:
count += 1
try:
_retcode, _retdata = pvc.lib.node.node_info(config, node)
if _retdata[state_field] != state_value:
break
else:
raise ValueError
except Exception:
sleep(sleep_time)
if count % dot_time == 0:
echo(config, ".", newline=False)
t_end = time()
echo(config, f" done. [{int(t_end - t_start)}s]")

12
cli-client-new/pvc/lib/node.py
View File

@ -52,7 +52,7 @@ def node_coordinator_state(config, node, action):
return retstatus, response.json().get("message", "")
def node_domain_state(config, node, action, wait):
def node_domain_state(config, node, action):
"""
Set node domain state state (flush/ready)
@ -60,7 +60,7 @@ def node_domain_state(config, node, action, wait):
API arguments: action={action}, wait={wait}
API schema: {"message": "{data}"}
"""
params = {"state": action, "wait": str(wait).lower()}
params = {"state": action}
response = call_api(
config, "post", "/node/{node}/domain-state".format(node=node), params=params
)
@ -442,13 +442,7 @@ def format_info(node_information, long_output):
return "\n".join(ainformation)
def format_list(node_list, raw):
if raw:
ainformation = list()
for node in sorted(item["name"] for item in node_list):
ainformation.append(node)
return "\n".join(ainformation)
def format_list(node_list):
node_list_output = []
# Determine optimal column widths