Add provisioner management commands

TODO: Add proper new formatters as required

.flake8

@@ -3,9 +3,7 @@
 #   * W503 (line break before binary operator): Black moves these to new lines
 #   * E501 (line too long): Long lines are a fact of life in comment blocks; Black handles active instances of this
 #   * E203 (whitespace before ':'): Black recommends this as disabled
-#   * F403 (import * used; unable to detect undefined names): We use a wildcard for helpers
-#   * F405 (possibly undefined name): We use a wildcard for helpers
-ignore = W503, E501, F403, F405
+ignore = W503, E501
 extend-ignore = E203
 # We exclude the Debian, migrations, and provisioner examples
 exclude = debian,api-daemon/migrations/versions,api-daemon/provisioner/examples,node-daemon/monitoring

.version

@@ -1 +1 @@
-0.9.70
+0.9.63

CHANGELOG.md

@@ -1,53 +1,5 @@
 ## PVC Changelog
 
-###### [v0.9.70](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.70)
-
-  * [Node Daemon] Fixes several compatibility issues for Debian 12 "Bookworm"
-
-###### [v0.9.69](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.69)
-
-  * [Node Daemon] Ensures that system load is always 2 decimal places on Bookworm
-  * [Node Daemon] Fixes bug blocking primary takeover at DNS Aggregator start if Patroni is down
-
-###### [v0.9.68](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.68)
-
-  * [CLI] Fixes another bug with network info view
-
-###### [v0.9.67](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.67)
-
-  * [CLI] Fixes several more bugs in the refactored CLI
-
-###### [v0.9.66](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.66)
-
-  * [CLI] Fixes a missing YAML import in CLI
-
-###### [v0.9.65](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.65)
-
-  * [CLI] Fixes a bug in the node list filtering command
-  * [CLI] Fixes a bug/default when no connection is specified
-
-###### [v0.9.64](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.64)
-
-  **Breaking Change [CLI]**: The CLI client root commands have been reorganized. The following commands have changed:
-
-   * `pvc cluster` -> `pvc connection` (all subcommands)
-   * `pvc task` -> `pvc cluster` (all subcommands)
-   * `pvc maintenance` -> `pvc cluster maintenance`
-   * `pvc status` -> `pvc cluster status`
-
-Ensure you have updated to the latest version of the PVC Ansible repository before deploying this version or using PVC Ansible oneshot playbooks for management.
-
-  **Breaking Change [CLI]**: The `--restart` option for VM configuration changes now has an explicit `--no-restart` to disable restarting, or a prompt if neither is specified; `--unsafe` no longer bypasses this prompt which was a bug. Applies to most `vm <cmd> set` commands like `vm vcpu set`, `vm memory set`, etc. All instances also feature restart confirmation afterwards, which, if `--restart` is provided, will prompt for confirmation unless `--yes` or `--unsafe` is specified.
-
-  **Breaking Change [CLI]**: The `--long` option previously on some `info` commands no longer exists; use `-f long`/`--format long` instead.
-
-  * [CLI] Significantly refactors the CLI client code for consistency and cleanliness
-  * [CLI] Implements `-f`/`--format` options for all `list` and `info` commands in a consistent way
-  * [CLI] Changes the behaviour of VM modification options with "--restart" to provide a "--no-restart"; defaults to a prompt if neither is specified and ignores the "--unsafe" global entirely
-  * [API] Fixes several bugs in the 3-debootstrap.py provisioner example script
-  * [Node] Fixes some bugs around VM shutdown on node flush
-  * [Documentation] Adds mentions of Ganeti and Harvester
-
 ###### [v0.9.63](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.63)
 
   * Mentions Ganeti in the docs

api-daemon/provisioner/examples/script/3-debootstrap.py

@@ -441,7 +441,7 @@ class VMBuilderScript(VMBuilder):
 
         # The directory we mounted things on earlier during prepare(); this could very well
         # be exposed as a module-level variable if you so choose
-        temp_dir = "/tmp/target"
+        temporary_directory = "/tmp/target"
 
         # Use these convenient aliases for later (avoiding lots of "self.vm_data" everywhere)
         vm_name = self.vm_name
@@ -469,8 +469,6 @@ class VMBuilderScript(VMBuilder):
                 "grub-pc",
                 "cloud-init",
                 "python3-cffi-backend",
-                "acpid",
-                "acpi-support-base",
                 "wget",
             ]
 
@@ -484,17 +482,17 @@ class VMBuilderScript(VMBuilder):
 
         # Perform a debootstrap installation
         print(
-            f"Installing system with debootstrap: debootstrap --include={','.join(deb_packages)} {deb_release} {temp_dir} {deb_mirror}"
+            f"Installing system with debootstrap: debootstrap --include={','.join(deb_packages)} {deb_release} {temporary_directory} {deb_mirror}"
         )
         os.system(
-            f"debootstrap --include={','.join(deb_packages)} {deb_release} {temp_dir} {deb_mirror}"
+            f"debootstrap --include={','.join(deb_packages)} {deb_release} {temporary_directory} {deb_mirror}"
         )
 
         # Bind mount the devfs so we can grub-install later
-        os.system("mount --bind /dev {}/dev".format(temp_dir))
+        os.system("mount --bind /dev {}/dev".format(temporary_directory))
 
         # Create an fstab entry for each volume
-        fstab_file = "{}/etc/fstab".format(temp_dir)
+        fstab_file = "{}/etc/fstab".format(temporary_directory)
         # The volume ID starts at zero and increments by one for each volume in the fixed-order
         # volume list. This lets us work around the insanity of Libvirt IDs not matching guest IDs,
         # while still letting us have some semblance of control here without enforcing things
@@ -539,13 +537,13 @@ class VMBuilderScript(VMBuilder):
             volume_id += 1
 
         # Write the hostname; you could also take an FQDN argument for this as an example
-        hostname_file = "{}/etc/hostname".format(temp_dir)
+        hostname_file = "{}/etc/hostname".format(temporary_directory)
         with open(hostname_file, "w") as fh:
             fh.write("{}".format(vm_name))
 
         # Fix the cloud-init.target since it's broken by default in Debian 11
         cloudinit_target_file = "{}/etc/systemd/system/cloud-init.target".format(
-            temp_dir
+            temporary_directory
         )
         with open(cloudinit_target_file, "w") as fh:
             # We lose our indent on these raw blocks to preserve the apperance of the files
@@ -559,7 +557,7 @@ After=multi-user.target
             fh.write(data)
 
         # Write the cloud-init configuration
-        ci_cfg_file = "{}/etc/cloud/cloud.cfg".format(temp_dir)
+        ci_cfg_file = "{}/etc/cloud/cloud.cfg".format(temporary_directory)
         with open(ci_cfg_file, "w") as fh:
             fh.write(
                 """
@@ -620,15 +618,15 @@ After=multi-user.target
                      - arches: [default]
                        failsafe:
                          primary: {deb_mirror}
-                """.format(
-                    deb_mirror=deb_mirror
-                )
-            )
+                """
+            ).format(deb_mirror=deb_mirror)
 
         # Due to device ordering within the Libvirt XML configuration, the first Ethernet interface
         # will always be on PCI bus ID 2, hence the name "ens2".
         # Write a DHCP stanza for ens2
-        ens2_network_file = "{}/etc/network/interfaces.d/ens2".format(temp_dir)
+        ens2_network_file = "{}/etc/network/interfaces.d/ens2".format(
+            temporary_directory
+        )
         with open(ens2_network_file, "w") as fh:
             data = """auto ens2
 iface ens2 inet dhcp
@@ -636,7 +634,7 @@ iface ens2 inet dhcp
             fh.write(data)
 
         # Write the DHCP config for ens2
-        dhclient_file = "{}/etc/dhcp/dhclient.conf".format(temp_dir)
+        dhclient_file = "{}/etc/dhcp/dhclient.conf".format(temporary_directory)
         with open(dhclient_file, "w") as fh:
             # We can use fstrings too, since PVC will always have Python 3.6+, though
             # using format() might be preferable for clarity in some situations
@@ -656,7 +654,7 @@ interface "ens2" {{
             fh.write(data)
 
         # Write the GRUB configuration
-        grubcfg_file = "{}/etc/default/grub".format(temp_dir)
+        grubcfg_file = "{}/etc/default/grub".format(temporary_directory)
         with open(grubcfg_file, "w") as fh:
             data = """# Written by the PVC provisioner
 GRUB_DEFAULT=0
@@ -673,7 +671,7 @@ GRUB_DISABLE_LINUX_UUID=false
             fh.write(data)
 
         # Do some tasks inside the chroot using the provided context manager
-        with chroot(temp_dir):
+        with chroot(temporary_directory):
             # Install and update GRUB
             os.system(
                 "grub-install --force /dev/rbd/{}/{}_{}".format(
@@ -706,17 +704,16 @@ GRUB_DISABLE_LINUX_UUID=false
         """
 
         # Run any imports first
-        import os
         from pvcapid.vmbuilder import open_zk
         from pvcapid.Daemon import config
         import daemon_lib.common as pvc_common
         import daemon_lib.ceph as pvc_ceph
 
-        # Set the temp_dir we used in the prepare() and install() steps
+        # Set the tempdir we used in the prepare() and install() steps
         temp_dir = "/tmp/target"
 
         # Unmount the bound devfs
-        os.system("umount {}/dev".format(temp_dir))
+        os.system("umount {}/dev".format(temporary_directory))
 
         # Use this construct for reversing the list, as the normal reverse() messes with the list
         for volume in list(reversed(self.vm_data["volumes"])):

api-daemon/pvcapid-manage.py

@@ -19,7 +19,8 @@
 #
 ###############################################################################
 
-from flask_migrate import Migrate, MigrateCommand, Manager
+from flask_migrate import Migrate, MigrateCommand
+from flask_script import Manager
 
 from pvcapid.flaskapi import app, db
 from pvcapid.models import *  # noqa F401,F403

api-daemon/pvcapid/Daemon.py

@@ -27,7 +27,7 @@ from ssl import SSLContext, TLSVersion
 from distutils.util import strtobool as dustrtobool
 
 # Daemon version
-version = "0.9.70"
+version = "0.9.63"
 
 # API version
 API_VERSION = 1.0

cli-client-new/pvc/cli/formatters.py

@@ -21,33 +21,6 @@
 
 from pvc.lib.node import format_info as node_format_info
 from pvc.lib.node import format_list as node_format_list
-from pvc.lib.vm import format_vm_tags as vm_format_tags
-from pvc.lib.vm import format_vm_vcpus as vm_format_vcpus
-from pvc.lib.vm import format_vm_memory as vm_format_memory
-from pvc.lib.vm import format_vm_networks as vm_format_networks
-from pvc.lib.vm import format_vm_volumes as vm_format_volumes
-from pvc.lib.vm import format_info as vm_format_info
-from pvc.lib.vm import format_list as vm_format_list
-from pvc.lib.network import format_info as network_format_info
-from pvc.lib.network import format_list as network_format_list
-from pvc.lib.network import format_list_dhcp as network_format_dhcp_list
-from pvc.lib.network import format_list_acl as network_format_acl_list
-from pvc.lib.network import format_list_sriov_pf as network_format_sriov_pf_list
-from pvc.lib.network import format_info_sriov_vf as network_format_sriov_vf_info
-from pvc.lib.network import format_list_sriov_vf as network_format_sriov_vf_list
-from pvc.lib.storage import format_raw_output as storage_format_raw
-from pvc.lib.storage import format_info_benchmark as storage_format_benchmark_info
-from pvc.lib.storage import format_list_benchmark as storage_format_benchmark_list
-from pvc.lib.storage import format_list_osd as storage_format_osd_list
-from pvc.lib.storage import format_list_pool as storage_format_pool_list
-from pvc.lib.storage import format_list_volume as storage_format_volume_list
-from pvc.lib.storage import format_list_snapshot as storage_format_snapshot_list
-from pvc.lib.provisioner import format_list_template as provisioner_format_template_list
-from pvc.lib.provisioner import format_list_userdata as provisioner_format_userdata_list
-from pvc.lib.provisioner import format_list_script as provisioner_format_script_list
-from pvc.lib.provisioner import format_list_ova as provisioner_format_ova_list
-from pvc.lib.provisioner import format_list_profile as provisioner_format_profile_list
-from pvc.lib.provisioner import format_list_task as provisioner_format_task_status
 
 
 # Define colour values for use in formatters
@@ -63,7 +36,7 @@ ansii = {
 }
 
 
-def cli_cluster_status_format_pretty(CLI_CONFIG, data):
+def cli_cluster_status_format_pretty(data):
     """
     Pretty format the full output of cli_cluster_status
     """
@@ -159,8 +132,6 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data):
 
     vms_strings = list()
     for state in vm_states:
-        if data.get("vms", {}).get(state) is None:
-            continue
         if state in ["start"]:
             state_colour = ansii["green"]
         elif state in ["migrate", "disable"]:
@@ -217,7 +188,7 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data):
     return "\n".join(output)
 
 
-def cli_cluster_status_format_short(CLI_CONFIG, data):
+def cli_cluster_status_format_short(data):
     """
     Pretty format the health-only output of cli_cluster_status
     """
@@ -262,7 +233,7 @@ def cli_cluster_status_format_short(CLI_CONFIG, data):
     return "\n".join(output)
 
 
-def cli_connection_list_format_pretty(CLI_CONFIG, data):
+def cli_connection_list_format_pretty(data):
     """
     Pretty format the output of cli_connection_list
     """
@@ -334,7 +305,7 @@ def cli_connection_list_format_pretty(CLI_CONFIG, data):
     return "\n".join(output)
 
 
-def cli_connection_detail_format_pretty(CLI_CONFIG, data):
+def cli_connection_detail_format_pretty(data):
     """
     Pretty format the output of cli_connection_detail
     """
@@ -454,281 +425,25 @@ def cli_connection_detail_format_pretty(CLI_CONFIG, data):
     return "\n".join(output)
 
 
-def cli_node_info_format_pretty(CLI_CONFIG, data):
+def cli_node_info_format_pretty(data):
     """
     Pretty format the basic output of cli_node_info
     """
 
-    return node_format_info(CLI_CONFIG, data, long_output=False)
+    return node_format_info(data, long_output=False)
 
 
-def cli_node_info_format_long(CLI_CONFIG, data):
+def cli_node_info_format_long(data):
     """
     Pretty format the full output of cli_node_info
     """
 
-    return node_format_info(CLI_CONFIG, data, long_output=True)
+    return node_format_info(data, long_output=True)
 
 
-def cli_node_list_format_pretty(CLI_CONFIG, data):
+def cli_node_list_format_pretty(data):
     """
     Pretty format the output of cli_node_list
     """
 
-    return node_format_list(CLI_CONFIG, data)
-
-
-def cli_vm_tag_get_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_vm_tag_get
-    """
-
-    return vm_format_tags(CLI_CONFIG, data)
-
-
-def cli_vm_vcpu_get_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_vm_vcpu_get
-    """
-
-    return vm_format_vcpus(CLI_CONFIG, data)
-
-
-def cli_vm_memory_get_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_vm_memory_get
-    """
-
-    return vm_format_memory(CLI_CONFIG, data)
-
-
-def cli_vm_network_get_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_vm_network_get
-    """
-
-    return vm_format_networks(CLI_CONFIG, data)
-
-
-def cli_vm_volume_get_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_vm_volume_get
-    """
-
-    return vm_format_volumes(CLI_CONFIG, data)
-
-
-def cli_vm_info_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the basic output of cli_vm_info
-    """
-
-    return vm_format_info(CLI_CONFIG, data, long_output=False)
-
-
-def cli_vm_info_format_long(CLI_CONFIG, data):
-    """
-    Pretty format the full output of cli_vm_info
-    """
-
-    return vm_format_info(CLI_CONFIG, data, long_output=True)
-
-
-def cli_vm_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_vm_list
-    """
-
-    return vm_format_list(CLI_CONFIG, data)
-
-
-def cli_network_info_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the full output of cli_network_info
-    """
-
-    return network_format_info(CLI_CONFIG, data, long_output=True)
-
-
-def cli_network_info_format_long(CLI_CONFIG, data):
-    """
-    Pretty format the full output of cli_network_info
-    """
-
-    return network_format_info(CLI_CONFIG, data, long_output=True)
-
-
-def cli_network_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_network_list
-    """
-
-    return network_format_list(CLI_CONFIG, data)
-
-
-def cli_network_dhcp_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_network_dhcp_list
-    """
-
-    return network_format_dhcp_list(CLI_CONFIG, data)
-
-
-def cli_network_acl_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_network_acl_list
-    """
-
-    return network_format_acl_list(CLI_CONFIG, data)
-
-
-def cli_network_sriov_pf_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_network_sriov_pf_list
-    """
-
-    return network_format_sriov_pf_list(CLI_CONFIG, data)
-
-
-def cli_network_sriov_vf_info_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_network_sriov_vf_info
-    """
-
-    return network_format_sriov_vf_info(CLI_CONFIG, data)
-
-
-def cli_network_sriov_vf_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_network_sriov_vf_list
-    """
-
-    return network_format_sriov_vf_list(CLI_CONFIG, data)
-
-
-def cli_storage_status_format_raw(CLI_CONFIG, data):
-    """
-    Direct format the output of cli_storage_status
-    """
-
-    return storage_format_raw(CLI_CONFIG, data)
-
-
-def cli_storage_util_format_raw(CLI_CONFIG, data):
-    """
-    Direct format the output of cli_storage_util
-    """
-
-    return storage_format_raw(CLI_CONFIG, data)
-
-
-def cli_storage_benchmark_info_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_storage_benchmark_info
-    """
-
-    return storage_format_benchmark_info(CLI_CONFIG, data)
-
-
-def cli_storage_benchmark_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_storage_benchmark_list
-    """
-
-    return storage_format_benchmark_list(CLI_CONFIG, data)
-
-
-def cli_storage_osd_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_storage_osd_list
-    """
-
-    return storage_format_osd_list(CLI_CONFIG, data)
-
-
-def cli_storage_pool_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_storage_pool_list
-    """
-
-    return storage_format_pool_list(CLI_CONFIG, data)
-
-
-def cli_storage_volume_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_storage_volume_list
-    """
-
-    return storage_format_volume_list(CLI_CONFIG, data)
-
-
-def cli_storage_snapshot_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_storage_snapshot_list
-    """
-
-    return storage_format_snapshot_list(CLI_CONFIG, data)
-
-
-def cli_provisioner_template_system_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_provisioner_template_system_list
-    """
-
-    return provisioner_format_template_list(CLI_CONFIG, data, template_type="system")
-
-
-def cli_provisioner_template_network_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_provisioner_template_network_list
-    """
-
-    return provisioner_format_template_list(CLI_CONFIG, data, template_type="network")
-
-
-def cli_provisioner_template_storage_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_provisioner_template_storage_list
-    """
-
-    return provisioner_format_template_list(CLI_CONFIG, data, template_type="storage")
-
-
-def cli_provisioner_userdata_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_provisioner_userdata_list
-    """
-
-    return provisioner_format_userdata_list(CLI_CONFIG, data)
-
-
-def cli_provisioner_script_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_provisioner_script_list
-    """
-
-    return provisioner_format_script_list(CLI_CONFIG, data)
-
-
-def cli_provisioner_ova_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_provisioner_ova_list
-    """
-
-    return provisioner_format_ova_list(CLI_CONFIG, data)
-
-
-def cli_provisioner_profile_list_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_provisioner_profile_list
-    """
-
-    return provisioner_format_profile_list(CLI_CONFIG, data)
-
-
-def cli_provisioner_status_format_pretty(CLI_CONFIG, data):
-    """
-    Pretty format the output of cli_provisioner_status
-    """
-
-    return provisioner_format_task_status(CLI_CONFIG, data)
+    return node_format_list(data)

cli-client-new/pvc/cli/helpers.py

@@ -20,7 +20,6 @@
 ###############################################################################
 
 from click import echo as click_echo
-from click import progressbar
 from distutils.util import strtobool
 from json import load as jload
 from json import dump as jdump
@@ -28,12 +27,9 @@ from os import chmod, environ, getpid, path
 from socket import gethostname
 from sys import argv
 from syslog import syslog, openlog, closelog, LOG_AUTH
-from time import sleep
 from yaml import load as yload
 from yaml import BaseLoader
 
-import pvc.lib.provisioner
-
 
 DEFAULT_STORE_DATA = {"cfgfile": "/etc/pvc/pvcapid.yaml"}
 DEFAULT_STORE_FILENAME = "pvc.json"
@@ -182,60 +178,3 @@ def update_store(store_path, store_data):
 
     with open(store_file, "w") as fh:
         jdump(store_data, fh, sort_keys=True, indent=4)
-
-
-def wait_for_provisioner(CLI_CONFIG, task_id):
-    """
-    Wait for a provisioner task to complete
-    """
-
-    echo(CLI_CONFIG, f"Task ID: {task_id}")
-    echo(CLI_CONFIG, "")
-
-    # Wait for the task to start
-    echo(CLI_CONFIG, "Waiting for task to start...", newline=False)
-    while True:
-        sleep(1)
-        task_status = pvc.lib.provisioner.task_status(
-            CLI_CONFIG, task_id, is_watching=True
-        )
-        if task_status.get("state") != "PENDING":
-            break
-        echo(".", newline=False)
-    echo(CLI_CONFIG, " done.")
-    echo(CLI_CONFIG, "")
-
-    # Start following the task state, updating progress as we go
-    total_task = task_status.get("total")
-    with progressbar(length=total_task, show_eta=False) as bar:
-        last_task = 0
-        maxlen = 0
-        while True:
-            sleep(1)
-            if task_status.get("state") != "RUNNING":
-                break
-            if task_status.get("current") > last_task:
-                current_task = int(task_status.get("current"))
-                bar.update(current_task - last_task)
-                last_task = current_task
-                # The extensive spaces at the end cause this to overwrite longer previous messages
-                curlen = len(str(task_status.get("status")))
-                if curlen > maxlen:
-                    maxlen = curlen
-                lendiff = maxlen - curlen
-                overwrite_whitespace = " " * lendiff
-                echo(
-                    CLI_CONFIG,
-                    "  " + task_status.get("status") + overwrite_whitespace,
-                    newline=False,
-                )
-            task_status = pvc.lib.provisioner.task_status(
-                CLI_CONFIG, task_id, is_watching=True
-            )
-        if task_status.get("state") == "SUCCESS":
-            bar.update(total_task - last_task)
-
-    echo(CLI_CONFIG, "")
-    retdata = task_status.get("state") + ": " + task_status.get("status")
-
-    return retdata

cli-client-new/pvc/lib/cluster.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+# cluster.py - PVC CLI client function library, cluster management
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import json
+
+from pvc.lib.common import call_api
+
+
+def initialize(config, overwrite=False):
+    """
+    Initialize the PVC cluster
+
+    API endpoint: GET /api/v1/initialize
+    API arguments: overwrite, yes-i-really-mean-it
+    API schema: {json_data_object}
+    """
+    params = {"yes-i-really-mean-it": "yes", "overwrite": overwrite}
+    response = call_api(config, "post", "/initialize", params=params)
+
+    if response.status_code == 200:
+        retstatus = True
+    else:
+        retstatus = False
+
+    return retstatus, response.json().get("message", "")
+
+
+def backup(config):
+    """
+    Get a JSON backup of the cluster
+
+    API endpoint: GET /api/v1/backup
+    API arguments:
+    API schema: {json_data_object}
+    """
+    response = call_api(config, "get", "/backup")
+
+    if response.status_code == 200:
+        return True, response.json()
+    else:
+        return False, response.json().get("message", "")
+
+
+def restore(config, cluster_data):
+    """
+    Restore a JSON backup to the cluster
+
+    API endpoint: POST /api/v1/restore
+    API arguments: yes-i-really-mean-it
+    API schema: {json_data_object}
+    """
+    cluster_data_json = json.dumps(cluster_data)
+
+    params = {"yes-i-really-mean-it": "yes"}
+    data = {"cluster_data": cluster_data_json}
+    response = call_api(config, "post", "/restore", params=params, data=data)
+
+    if response.status_code == 200:
+        retstatus = True
+    else:
+        retstatus = False
+
+    return retstatus, response.json().get("message", "")
+
+
+def maintenance_mode(config, state):
+    """
+    Enable or disable PVC cluster maintenance mode
+
+    API endpoint: POST /api/v1/status
+    API arguments: {state}={state}
+    API schema: {json_data_object}
+    """
+    params = {"state": state}
+    response = call_api(config, "post", "/status", params=params)
+
+    if response.status_code == 200:
+        retstatus = True
+    else:
+        retstatus = False
+
+    return retstatus, response.json().get("message", "")
+
+
+def get_info(config):
+    """
+    Get status of the PVC cluster
+
+    API endpoint: GET /api/v1/status
+    API arguments:
+    API schema: {json_data_object}
+    """
+    response = call_api(config, "get", "/status")
+
+    if response.status_code == 200:
+        return True, response.json()
+    else:
+        return False, response.json().get("message", "")

cli-client-new/pvc/lib/node.py

@@ -52,7 +52,7 @@ def node_coordinator_state(config, node, action):
     return retstatus, response.json().get("message", "")
 
 
-def node_domain_state(config, node, action, wait):
+def node_domain_state(config, node, action):
     """
     Set node domain state state (flush/ready)
 
@@ -60,7 +60,7 @@ def node_domain_state(config, node, action, wait):
     API arguments: action={action}, wait={wait}
     API schema: {"message": "{data}"}
     """
-    params = {"state": action, "wait": str(wait).lower()}
+    params = {"state": action}
     response = call_api(
         config, "post", "/node/{node}/domain-state".format(node=node), params=params
     )
@@ -442,12 +442,9 @@ def format_info(node_information, long_output):
     return "\n".join(ainformation)
 
 
-def format_list(node_list, raw):
-    if raw:
-        ainformation = list()
-        for node in sorted(item["name"] for item in node_list):
-            ainformation.append(node)
-        return "\n".join(ainformation)
+def format_list(node_list):
+    if node_list == "Node not found.":
+        return node_list
 
     node_list_output = []
 

client-cli-old/pvc.py

@@ -1,33 +0,0 @@
-#!/usr/bin/env python3
-
-# pvc.py - PVC client command-line interface (stub testing interface)
-# Part of the Parallel Virtual Cluster (PVC) system
-#
-#    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, version 3.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-###############################################################################
-
-import pvc.pvc
-
-
-#
-# Main entry point
-#
-def main():
-    return pvc.pvc.cli(obj={})
-
-
-if __name__ == "__main__":
-    main()

client-cli-old/pvc/lib/cluster.py

@@ -1,313 +0,0 @@
-#!/usr/bin/env python3
-
-# cluster.py - PVC CLI client function library, cluster management
-# Part of the Parallel Virtual Cluster (PVC) system
-#
-#    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, version 3.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-###############################################################################
-
-import json
-
-import pvc.lib.ansiprint as ansiprint
-from pvc.lib.common import call_api
-
-
-def initialize(config, overwrite=False):
-    """
-    Initialize the PVC cluster
-
-    API endpoint: GET /api/v1/initialize
-    API arguments: overwrite, yes-i-really-mean-it
-    API schema: {json_data_object}
-    """
-    params = {"yes-i-really-mean-it": "yes", "overwrite": overwrite}
-    response = call_api(config, "post", "/initialize", params=params)
-
-    if response.status_code == 200:
-        retstatus = True
-    else:
-        retstatus = False
-
-    return retstatus, response.json().get("message", "")
-
-
-def backup(config):
-    """
-    Get a JSON backup of the cluster
-
-    API endpoint: GET /api/v1/backup
-    API arguments:
-    API schema: {json_data_object}
-    """
-    response = call_api(config, "get", "/backup")
-
-    if response.status_code == 200:
-        return True, response.json()
-    else:
-        return False, response.json().get("message", "")
-
-
-def restore(config, cluster_data):
-    """
-    Restore a JSON backup to the cluster
-
-    API endpoint: POST /api/v1/restore
-    API arguments: yes-i-really-mean-it
-    API schema: {json_data_object}
-    """
-    cluster_data_json = json.dumps(cluster_data)
-
-    params = {"yes-i-really-mean-it": "yes"}
-    data = {"cluster_data": cluster_data_json}
-    response = call_api(config, "post", "/restore", params=params, data=data)
-
-    if response.status_code == 200:
-        retstatus = True
-    else:
-        retstatus = False
-
-    return retstatus, response.json().get("message", "")
-
-
-def maintenance_mode(config, state):
-    """
-    Enable or disable PVC cluster maintenance mode
-
-    API endpoint: POST /api/v1/status
-    API arguments: {state}={state}
-    API schema: {json_data_object}
-    """
-    params = {"state": state}
-    response = call_api(config, "post", "/status", params=params)
-
-    if response.status_code == 200:
-        retstatus = True
-    else:
-        retstatus = False
-
-    return retstatus, response.json().get("message", "")
-
-
-def get_info(config):
-    """
-    Get status of the PVC cluster
-
-    API endpoint: GET /api/v1/status
-    API arguments:
-    API schema: {json_data_object}
-    """
-    response = call_api(config, "get", "/status")
-
-    if response.status_code == 200:
-        return True, response.json()
-    else:
-        return False, response.json().get("message", "")
-
-
-def format_info(cluster_information, oformat):
-    if oformat == "json":
-        return json.dumps(cluster_information)
-
-    if oformat == "json-pretty":
-        return json.dumps(cluster_information, indent=4)
-
-    # Plain formatting, i.e. human-readable
-    if (
-        cluster_information.get("maintenance") == "true"
-        or cluster_information.get("cluster_health", {}).get("health", "N/A") == "N/A"
-    ):
-        health_colour = ansiprint.blue()
-    elif cluster_information.get("cluster_health", {}).get("health", 100) > 90:
-        health_colour = ansiprint.green()
-    elif cluster_information.get("cluster_health", {}).get("health", 100) > 50:
-        health_colour = ansiprint.yellow()
-    else:
-        health_colour = ansiprint.red()
-
-    ainformation = []
-
-    ainformation.append(
-        "{}PVC cluster status:{}".format(ansiprint.bold(), ansiprint.end())
-    )
-    ainformation.append("")
-
-    health_text = (
-        f"{cluster_information.get('cluster_health', {}).get('health', 'N/A')}"
-    )
-    if health_text != "N/A":
-        health_text += "%"
-    if cluster_information.get("maintenance") == "true":
-        health_text += " (maintenance on)"
-
-    ainformation.append(
-        "{}Cluster health:{}  {}{}{}".format(
-            ansiprint.purple(),
-            ansiprint.end(),
-            health_colour,
-            health_text,
-            ansiprint.end(),
-        )
-    )
-    if cluster_information.get("cluster_health", {}).get("messages"):
-        health_messages = "\n                 > ".join(
-            sorted(cluster_information["cluster_health"]["messages"])
-        )
-        ainformation.append(
-            "{}Health messages:{} > {}".format(
-                ansiprint.purple(),
-                ansiprint.end(),
-                health_messages,
-            )
-        )
-    else:
-        ainformation.append(
-            "{}Health messages:{} N/A".format(
-                ansiprint.purple(),
-                ansiprint.end(),
-            )
-        )
-
-    if oformat == "short":
-        return "\n".join(ainformation)
-
-    ainformation.append("")
-    ainformation.append(
-        "{}Primary node:{}        {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["primary_node"]
-        )
-    )
-    ainformation.append(
-        "{}PVC version:{}         {}".format(
-            ansiprint.purple(),
-            ansiprint.end(),
-            cluster_information.get("pvc_version", "N/A"),
-        )
-    )
-    ainformation.append(
-        "{}Cluster upstream IP:{} {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["upstream_ip"]
-        )
-    )
-    ainformation.append("")
-    ainformation.append(
-        "{}Total nodes:{}     {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["nodes"]["total"]
-        )
-    )
-    ainformation.append(
-        "{}Total VMs:{}       {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["vms"]["total"]
-        )
-    )
-    ainformation.append(
-        "{}Total networks:{}  {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["networks"]
-        )
-    )
-    ainformation.append(
-        "{}Total OSDs:{}      {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["osds"]["total"]
-        )
-    )
-    ainformation.append(
-        "{}Total pools:{}     {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["pools"]
-        )
-    )
-    ainformation.append(
-        "{}Total volumes:{}   {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["volumes"]
-        )
-    )
-    ainformation.append(
-        "{}Total snapshots:{} {}".format(
-            ansiprint.purple(), ansiprint.end(), cluster_information["snapshots"]
-        )
-    )
-
-    nodes_string = "{}Nodes:{} {}/{} {}ready,run{}".format(
-        ansiprint.purple(),
-        ansiprint.end(),
-        cluster_information["nodes"].get("run,ready", 0),
-        cluster_information["nodes"].get("total", 0),
-        ansiprint.green(),
-        ansiprint.end(),
-    )
-    for state, count in cluster_information["nodes"].items():
-        if state == "total" or state == "run,ready":
-            continue
-
-        nodes_string += " {}/{} {}{}{}".format(
-            count,
-            cluster_information["nodes"]["total"],
-            ansiprint.yellow(),
-            state,
-            ansiprint.end(),
-        )
-
-    ainformation.append("")
-    ainformation.append(nodes_string)
-
-    vms_string = "{}VMs:{} {}/{} {}start{}".format(
-        ansiprint.purple(),
-        ansiprint.end(),
-        cluster_information["vms"].get("start", 0),
-        cluster_information["vms"].get("total", 0),
-        ansiprint.green(),
-        ansiprint.end(),
-    )
-    for state, count in cluster_information["vms"].items():
-        if state == "total" or state == "start":
-            continue
-
-        if state in ["disable", "migrate", "unmigrate", "provision"]:
-            colour = ansiprint.blue()
-        else:
-            colour = ansiprint.yellow()
-
-        vms_string += " {}/{} {}{}{}".format(
-            count, cluster_information["vms"]["total"], colour, state, ansiprint.end()
-        )
-
-    ainformation.append("")
-    ainformation.append(vms_string)
-
-    if cluster_information["osds"]["total"] > 0:
-        osds_string = "{}Ceph OSDs:{} {}/{} {}up,in{}".format(
-            ansiprint.purple(),
-            ansiprint.end(),
-            cluster_information["osds"].get("up,in", 0),
-            cluster_information["osds"].get("total", 0),
-            ansiprint.green(),
-            ansiprint.end(),
-        )
-        for state, count in cluster_information["osds"].items():
-            if state == "total" or state == "up,in":
-                continue
-
-            osds_string += " {}/{} {}{}{}".format(
-                count,
-                cluster_information["osds"]["total"],
-                ansiprint.yellow(),
-                state,
-                ansiprint.end(),
-            )
-
-        ainformation.append("")
-        ainformation.append(osds_string)
-
-    ainformation.append("")
-    return "\n".join(ainformation)

client-cli/pvc/lib/ceph.py

@@ -21,14 +21,14 @@
 
 import math
 
-from json import loads
+from json import dumps, loads
 from requests_toolbelt.multipart.encoder import (
     MultipartEncoder,
     MultipartEncoderMonitor,
 )
 
-import pvc.lib.ansiprint as ansiprint
-from pvc.lib.common import UploadProgressBar, call_api
+import pvc.cli_lib.ansiprint as ansiprint
+from pvc.cli_lib.common import UploadProgressBar, call_api
 
 #
 # Supplemental functions
@@ -143,7 +143,7 @@ def ceph_util(config):
         return False, response.json().get("message", "")
 
 
-def format_raw_output(config, status_data):
+def format_raw_output(status_data):
     ainformation = list()
     ainformation.append(
         "{bold}Ceph cluster {stype} (primary node {end}{blue}{primary}{end}{bold}){end}\n".format(
@@ -379,7 +379,7 @@ def getOutputColoursOSD(osd_information):
     return osd_up_flag, osd_up_colour, osd_in_flag, osd_in_colour
 
 
-def format_list_osd(config, osd_list):
+def format_list_osd(osd_list):
     # Handle empty list
     if not osd_list:
         osd_list = list()
@@ -835,7 +835,7 @@ def ceph_pool_set_pgs(config, pool, pgs):
     return retstatus, response.json().get("message", "")
 
 
-def format_list_pool(config, pool_list):
+def format_list_pool(pool_list):
     # Handle empty list
     if not pool_list:
         pool_list = list()
@@ -1318,7 +1318,7 @@ def ceph_volume_clone(config, pool, volume, new_volume):
     return retstatus, response.json().get("message", "")
 
 
-def format_list_volume(config, volume_list):
+def format_list_volume(volume_list):
     # Handle empty list
     if not volume_list:
         volume_list = list()
@@ -1596,7 +1596,7 @@ def ceph_snapshot_modify(config, pool, volume, snapshot, new_name=None):
     return retstatus, response.json().get("message", "")
 
 
-def format_list_snapshot(config, snapshot_list):
+def format_list_snapshot(snapshot_list):
     # Handle empty list
     if not snapshot_list:
         snapshot_list = list()
@@ -1981,7 +1981,7 @@ def format_list_benchmark(config, benchmark_information):
     return "\n".join(benchmark_list_output)
 
 
-def format_info_benchmark(config, benchmark_information):
+def format_info_benchmark(config, oformat, benchmark_information):
     # This matrix is a list of the possible format functions for a benchmark result
     # It is extensable in the future should newer formats be required.
     benchmark_matrix = {
@@ -1991,7 +1991,12 @@ def format_info_benchmark(config, benchmark_information):
 
     benchmark_version = benchmark_information[0]["test_format"]
 
-    return benchmark_matrix[benchmark_version](config, benchmark_information[0])
+    if oformat == "json-pretty":
+        return dumps(benchmark_information, indent=4)
+    elif oformat == "json":
+        return dumps(benchmark_information)
+    else:
+        return benchmark_matrix[benchmark_version](config, benchmark_information[0])
 
 
 def format_info_benchmark_legacy(config, benchmark_information):

client-cli/pvc/lib/cluster.py

@@ -21,7 +21,8 @@
 
 import json
 
-from pvc.lib.common import call_api
+import pvc.cli_lib.ansiprint as ansiprint
+from pvc.cli_lib.common import call_api
 
 
 def initialize(config, overwrite=False):
@@ -114,3 +115,199 @@ def get_info(config):
         return True, response.json()
     else:
         return False, response.json().get("message", "")
+
+
+def format_info(cluster_information, oformat):
+    if oformat == "json":
+        return json.dumps(cluster_information)
+
+    if oformat == "json-pretty":
+        return json.dumps(cluster_information, indent=4)
+
+    # Plain formatting, i.e. human-readable
+    if (
+        cluster_information.get("maintenance") == "true"
+        or cluster_information.get("cluster_health", {}).get("health", "N/A") == "N/A"
+    ):
+        health_colour = ansiprint.blue()
+    elif cluster_information.get("cluster_health", {}).get("health", 100) > 90:
+        health_colour = ansiprint.green()
+    elif cluster_information.get("cluster_health", {}).get("health", 100) > 50:
+        health_colour = ansiprint.yellow()
+    else:
+        health_colour = ansiprint.red()
+
+    ainformation = []
+
+    ainformation.append(
+        "{}PVC cluster status:{}".format(ansiprint.bold(), ansiprint.end())
+    )
+    ainformation.append("")
+
+    health_text = (
+        f"{cluster_information.get('cluster_health', {}).get('health', 'N/A')}"
+    )
+    if health_text != "N/A":
+        health_text += "%"
+    if cluster_information.get("maintenance") == "true":
+        health_text += " (maintenance on)"
+
+    ainformation.append(
+        "{}Cluster health:{}  {}{}{}".format(
+            ansiprint.purple(),
+            ansiprint.end(),
+            health_colour,
+            health_text,
+            ansiprint.end(),
+        )
+    )
+    if cluster_information.get("cluster_health", {}).get("messages"):
+        health_messages = "\n                 > ".join(
+            sorted(cluster_information["cluster_health"]["messages"])
+        )
+        ainformation.append(
+            "{}Health messages:{} > {}".format(
+                ansiprint.purple(),
+                ansiprint.end(),
+                health_messages,
+            )
+        )
+    else:
+        ainformation.append(
+            "{}Health messages:{} N/A".format(
+                ansiprint.purple(),
+                ansiprint.end(),
+            )
+        )
+
+    if oformat == "short":
+        return "\n".join(ainformation)
+
+    ainformation.append("")
+    ainformation.append(
+        "{}Primary node:{}        {}".format(
+            ansiprint.purple(), ansiprint.end(), cluster_information["primary_node"]
+        )
+    )
+    ainformation.append(
+        "{}PVC version:{}         {}".format(
+            ansiprint.purple(),
+            ansiprint.end(),
+            cluster_information.get("pvc_version", "N/A"),
+        )
+    )
+    ainformation.append(
+        "{}Cluster upstream IP:{} {}".format(
+            ansiprint.purple(), ansiprint.end(), cluster_information["upstream_ip"]
+        )
+    )
+    ainformation.append("")
+    ainformation.append(
+        "{}Total nodes:{}     {}".format(
+            ansiprint.purple(), ansiprint.end(), cluster_information["nodes"]["total"]
+        )
+    )
+    ainformation.append(
+        "{}Total VMs:{}       {}".format(
+            ansiprint.purple(), ansiprint.end(), cluster_information["vms"]["total"]
+        )
+    )
+    ainformation.append(
+        "{}Total networks:{}  {}".format(
+            ansiprint.purple(), ansiprint.end(), cluster_information["networks"]
+        )
+    )
+    ainformation.append(
+        "{}Total OSDs:{}      {}".format(
+            ansiprint.purple(), ansiprint.end(), cluster_information["osds"]["total"]
+        )
+    )
+    ainformation.append(
+        "{}Total pools:{}     {}".format(
+            ansiprint.purple(), ansiprint.end(), cluster_information["pools"]
+        )
+    )
+    ainformation.append(
+        "{}Total volumes:{}   {}".format(
+            ansiprint.purple(), ansiprint.end(), cluster_information["volumes"]
+        )
+    )
+    ainformation.append(
+        "{}Total snapshots:{} {}".format(
+            ansiprint.purple(), ansiprint.end(), cluster_information["snapshots"]
+        )
+    )
+
+    nodes_string = "{}Nodes:{} {}/{} {}ready,run{}".format(
+        ansiprint.purple(),
+        ansiprint.end(),
+        cluster_information["nodes"].get("run,ready", 0),
+        cluster_information["nodes"].get("total", 0),
+        ansiprint.green(),
+        ansiprint.end(),
+    )
+    for state, count in cluster_information["nodes"].items():
+        if state == "total" or state == "run,ready":
+            continue
+
+        nodes_string += " {}/{} {}{}{}".format(
+            count,
+            cluster_information["nodes"]["total"],
+            ansiprint.yellow(),
+            state,
+            ansiprint.end(),
+        )
+
+    ainformation.append("")
+    ainformation.append(nodes_string)
+
+    vms_string = "{}VMs:{} {}/{} {}start{}".format(
+        ansiprint.purple(),
+        ansiprint.end(),
+        cluster_information["vms"].get("start", 0),
+        cluster_information["vms"].get("total", 0),
+        ansiprint.green(),
+        ansiprint.end(),
+    )
+    for state, count in cluster_information["vms"].items():
+        if state == "total" or state == "start":
+            continue
+
+        if state in ["disable", "migrate", "unmigrate", "provision"]:
+            colour = ansiprint.blue()
+        else:
+            colour = ansiprint.yellow()
+
+        vms_string += " {}/{} {}{}{}".format(
+            count, cluster_information["vms"]["total"], colour, state, ansiprint.end()
+        )
+
+    ainformation.append("")
+    ainformation.append(vms_string)
+
+    if cluster_information["osds"]["total"] > 0:
+        osds_string = "{}Ceph OSDs:{} {}/{} {}up,in{}".format(
+            ansiprint.purple(),
+            ansiprint.end(),
+            cluster_information["osds"].get("up,in", 0),
+            cluster_information["osds"].get("total", 0),
+            ansiprint.green(),
+            ansiprint.end(),
+        )
+        for state, count in cluster_information["osds"].items():
+            if state == "total" or state == "up,in":
+                continue
+
+            osds_string += " {}/{} {}{}{}".format(
+                count,
+                cluster_information["osds"]["total"],
+                ansiprint.yellow(),
+                state,
+                ansiprint.end(),
+            )
+
+        ainformation.append("")
+        ainformation.append(osds_string)
+
+    ainformation.append("")
+    return "\n".join(ainformation)

client-cli/pvc/lib/network.py

@@ -20,8 +20,8 @@
 ###############################################################################
 
 import re
-import pvc.lib.ansiprint as ansiprint
-from pvc.lib.common import call_api
+import pvc.cli_lib.ansiprint as ansiprint
+from pvc.cli_lib.common import call_api
 
 
 def isValidMAC(macaddr):
@@ -542,16 +542,11 @@ def net_sriov_vf_info(config, node, vf):
             return False, "VF not found."
         else:
             # Return a single instance if the response is a list
-            data = dict()
-            data["node"] = node
             if isinstance(response.json(), list):
-                data = dict()
-                data["vf_information"] = response.json()[0]
-                return True, data
+                return True, response.json()[0]
             # This shouldn't happen, but is here just in case
             else:
-                data["vf_information"] = response.json()
-                return True, data
+                return True, response.json()
     else:
         return False, response.json().get("message", "")
 
@@ -700,7 +695,7 @@ def format_info(config, network_information, long_output):
                 ainformation.append("")
                 if retcode:
                     dhcp4_reservations_string = format_list_dhcp(
-                        config, dhcp4_reservations_list
+                        dhcp4_reservations_list
                     )
                     for line in dhcp4_reservations_string.split("\n"):
                         ainformation.append(line)
@@ -719,7 +714,7 @@ def format_info(config, network_information, long_output):
                 )
                 ainformation.append("")
                 if retcode:
-                    firewall_rules_string = format_list_acl(config, firewall_rules_list)
+                    firewall_rules_string = format_list_acl(firewall_rules_list)
                     for line in firewall_rules_string.split("\n"):
                         ainformation.append(line)
                 else:
@@ -893,7 +888,7 @@ def format_list(config, network_list):
     return "\n".join(network_list_output)
 
 
-def format_list_dhcp(config, dhcp_lease_list):
+def format_list_dhcp(dhcp_lease_list):
     dhcp_lease_list_output = []
 
     # Determine optimal column widths
@@ -992,7 +987,7 @@ def format_list_dhcp(config, dhcp_lease_list):
     return "\n".join(dhcp_lease_list_output)
 
 
-def format_list_acl(config, acl_list):
+def format_list_acl(acl_list):
     # Handle when we get an empty entry
     if not acl_list:
         acl_list = list()
@@ -1091,7 +1086,7 @@ def format_list_acl(config, acl_list):
     return "\n".join(acl_list_output)
 
 
-def format_list_sriov_pf(config, pf_list):
+def format_list_sriov_pf(pf_list):
     # The maximum column width of the VFs column
     max_vfs_length = 70
 
@@ -1211,7 +1206,7 @@ def format_list_sriov_pf(config, pf_list):
     return "\n".join(pf_list_output)
 
 
-def format_list_sriov_vf(config, vf_list):
+def format_list_sriov_vf(vf_list):
     # Handle when we get an empty entry
     if not vf_list:
         vf_list = list()
@@ -1343,13 +1338,10 @@ def format_list_sriov_vf(config, vf_list):
     return "\n".join(vf_list_output)
 
 
-def format_info_sriov_vf(config, data):
-    if not data or not data["vf_information"]:
+def format_info_sriov_vf(config, vf_information, node):
+    if not vf_information:
         return "No VF found"
 
-    node = data["node"]
-    vf_information = data["vf_information"]
-
     # Get information on the using VM if applicable
     if vf_information["usage"]["used"] == "True" and vf_information["usage"]["domain"]:
         vm_information = call_api(

client-cli/pvc/lib/node.py

@@ -21,8 +21,8 @@
 
 import time
 
-import pvc.lib.ansiprint as ansiprint
-from pvc.lib.common import call_api
+import pvc.cli_lib.ansiprint as ansiprint
+from pvc.cli_lib.common import call_api
 
 
 #
@@ -52,7 +52,7 @@ def node_coordinator_state(config, node, action):
     return retstatus, response.json().get("message", "")
 
 
-def node_domain_state(config, node, action):
+def node_domain_state(config, node, action, wait):
     """
     Set node domain state state (flush/ready)
 
@@ -60,7 +60,7 @@ def node_domain_state(config, node, action):
     API arguments: action={action}, wait={wait}
     API schema: {"message": "{data}"}
     """
-    params = {"state": action}
+    params = {"state": action, "wait": str(wait).lower()}
     response = call_api(
         config, "post", "/node/{node}/domain-state".format(node=node), params=params
     )
@@ -273,7 +273,7 @@ def getOutputColours(node_information):
     )
 
 
-def format_info(config, node_information, long_output):
+def format_info(node_information, long_output):
     (
         health_colour,
         daemon_state_colour,
@@ -442,9 +442,12 @@ def format_info(config, node_information, long_output):
     return "\n".join(ainformation)
 
 
-def format_list(config, node_list):
-    if node_list == "Node not found.":
-        return node_list
+def format_list(node_list, raw):
+    if raw:
+        ainformation = list()
+        for node in sorted(item["name"] for item in node_list):
+            ainformation.append(node)
+        return "\n".join(ainformation)
 
     node_list_output = []
 

client-cli/pvc/lib/provisioner.py

@@ -24,8 +24,8 @@ from requests_toolbelt.multipart.encoder import (
     MultipartEncoderMonitor,
 )
 
-import pvc.lib.ansiprint as ansiprint
-from pvc.lib.common import UploadProgressBar, call_api
+import pvc.cli_lib.ansiprint as ansiprint
+from pvc.cli_lib.common import UploadProgressBar, call_api
 from ast import literal_eval
 
 
@@ -750,11 +750,24 @@ def task_status(config, task_id=None, is_watching=False):
         if response.status_code == 200:
             retvalue = True
             respjson = response.json()
+
             if is_watching:
-                # Just return the raw JSON to the watching process instead of including value
+                # Just return the raw JSON to the watching process instead of formatting it
                 return respjson
+
+            job_state = respjson["state"]
+            if job_state == "RUNNING":
+                retdata = "Job state: RUNNING\nStage: {}/{}\nStatus: {}".format(
+                    respjson["current"], respjson["total"], respjson["status"]
+                )
+            elif job_state == "FAILED":
+                retdata = "Job state: FAILED\nStatus: {}".format(respjson["status"])
+            elif job_state == "COMPLETED":
+                retdata = "Job state: COMPLETED\nStatus: {}".format(respjson["status"])
             else:
-                return retvalue, respjson
+                retdata = "Job state: {}\nStatus: {}".format(
+                    respjson["state"], respjson["status"]
+                )
         else:
             retvalue = False
             retdata = response.json().get("message", "")
@@ -801,7 +814,7 @@ def task_status(config, task_id=None, is_watching=False):
 #
 # Format functions
 #
-def format_list_template(config, template_data, template_type=None):
+def format_list_template(template_data, template_type=None):
     """
     Format the returned template template
 
@@ -1317,12 +1330,7 @@ def format_list_template_storage(template_template):
     return "\n".join(template_list_output)
 
 
-def format_list_userdata(config, userdata_data):
-    if not config.get("long_output"):
-        lines = 4
-    else:
-        lines = None
-
+def format_list_userdata(userdata_data, lines=None):
     if isinstance(userdata_data, dict):
         userdata_data = [userdata_data]
 
@@ -1424,12 +1432,7 @@ def format_list_userdata(config, userdata_data):
     return "\n".join(userdata_list_output)
 
 
-def format_list_script(config, script_data):
-    if not config.get("long_output"):
-        lines = 4
-    else:
-        lines = None
-
+def format_list_script(script_data, lines=None):
     if isinstance(script_data, dict):
         script_data = [script_data]
 
@@ -1528,7 +1531,7 @@ def format_list_script(config, script_data):
     return "\n".join(script_list_output)
 
 
-def format_list_ova(config, ova_data):
+def format_list_ova(ova_data):
     if isinstance(ova_data, dict):
         ova_data = [ova_data]
 
@@ -1675,7 +1678,7 @@ def format_list_ova(config, ova_data):
     return "\n".join(ova_list_output)
 
 
-def format_list_profile(config, profile_data):
+def format_list_profile(profile_data):
     if isinstance(profile_data, dict):
         profile_data = [profile_data]
 
@@ -1864,23 +1867,7 @@ def format_list_profile(config, profile_data):
     return "\n".join(profile_list_output)
 
 
-def format_list_task(config, task_data):
-    if not isinstance(task_data, list):
-        job_state = task_data["state"]
-        if job_state == "RUNNING":
-            retdata = "Job state: RUNNING\nStage: {}/{}\nStatus: {}".format(
-                task_data["current"], task_data["total"], task_data["status"]
-            )
-        elif job_state == "FAILED":
-            retdata = "Job state: FAILED\nStatus: {}".format(task_data["status"])
-        elif job_state == "COMPLETED":
-            retdata = "Job state: COMPLETED\nStatus: {}".format(task_data["status"])
-        else:
-            retdata = "Job state: {}\nStatus: {}".format(
-                task_data["state"], task_data["status"]
-            )
-        return retdata
-
+def format_list_task(task_data):
     task_list_output = []
 
     # Determine optimal column widths

client-cli/pvc/lib/vm.py

@@ -22,8 +22,8 @@
 import time
 import re
 
-import pvc.lib.ansiprint as ansiprint
-from pvc.lib.common import call_api, format_bytes, format_metric
+import pvc.cli_lib.ansiprint as ansiprint
+from pvc.cli_lib.common import call_api, format_bytes, format_metric
 
 
 #
@@ -286,18 +286,20 @@ def vm_tag_set(config, vm, action, tag, protected=False):
     return retstatus, response.json().get("message", "")
 
 
-def format_vm_tags(config, data):
+def format_vm_tags(config, name, tags):
     """
     Format the output of a tags dictionary in a nice table
     """
-
-    tags = data.get("tags", [])
-
     if len(tags) < 1:
         return "No tags found."
 
     output_list = []
 
+    name_length = 5
+    _name_length = len(name) + 1
+    if _name_length > name_length:
+        name_length = _name_length
+
     tags_name_length = 4
     tags_type_length = 5
     tags_protected_length = 10
@@ -493,38 +495,44 @@ def vm_vcpus_get(config, vm):
     except Exception:
         return False, "ERROR: Failed to parse XML data."
 
-    data = dict()
-    data["name"] = vm
-    data["vcpus"] = int(parsed_xml.vcpu.text)
-    data["sockets"] = parsed_xml.cpu.topology.attrib.get("sockets")
-    data["cores"] = parsed_xml.cpu.topology.attrib.get("cores")
-    data["threads"] = parsed_xml.cpu.topology.attrib.get("threads")
+    vm_vcpus = int(parsed_xml.vcpu.text)
+    vm_sockets = parsed_xml.cpu.topology.attrib.get("sockets")
+    vm_cores = parsed_xml.cpu.topology.attrib.get("cores")
+    vm_threads = parsed_xml.cpu.topology.attrib.get("threads")
 
-    return True, data
+    return True, (vm_vcpus, (vm_sockets, vm_cores, vm_threads))
 
 
-def format_vm_vcpus(config, data):
+def format_vm_vcpus(config, name, vcpus):
     """
     Format the output of a vCPU value in a nice table
     """
     output_list = []
 
+    name_length = 5
+    _name_length = len(name) + 1
+    if _name_length > name_length:
+        name_length = _name_length
+
     vcpus_length = 6
     sockets_length = 8
     cores_length = 6
     threads_length = 8
 
     output_list.append(
-        "{bold}{vcpus: <{vcpus_length}}  \
+        "{bold}{name: <{name_length}}  \
+{vcpus: <{vcpus_length}}   \
 {sockets: <{sockets_length}} \
 {cores: <{cores_length}} \
 {threads: <{threads_length}}{end_bold}".format(
+            name_length=name_length,
             vcpus_length=vcpus_length,
             sockets_length=sockets_length,
             cores_length=cores_length,
             threads_length=threads_length,
             bold=ansiprint.bold(),
             end_bold=ansiprint.end(),
+            name="Name",
             vcpus="vCPUs",
             sockets="Sockets",
             cores="Cores",
@@ -532,20 +540,23 @@ def format_vm_vcpus(config, data):
         )
     )
     output_list.append(
-        "{bold}{vcpus: <{vcpus_length}}  \
+        "{bold}{name: <{name_length}}  \
+{vcpus: <{vcpus_length}}   \
 {sockets: <{sockets_length}} \
 {cores: <{cores_length}} \
 {threads: <{threads_length}}{end_bold}".format(
+            name_length=name_length,
             vcpus_length=vcpus_length,
             sockets_length=sockets_length,
             cores_length=cores_length,
             threads_length=threads_length,
             bold="",
             end_bold="",
-            vcpus=data["vcpus"],
-            sockets=data["sockets"],
-            cores=data["cores"],
-            threads=data["threads"],
+            name=name,
+            vcpus=vcpus[0],
+            sockets=vcpus[1][0],
+            cores=vcpus[1][1],
+            threads=vcpus[1][2],
         )
     )
     return "\n".join(output_list)
@@ -608,35 +619,44 @@ def vm_memory_get(config, vm):
     except Exception:
         return False, "ERROR: Failed to parse XML data."
 
-    data = dict()
-    data["name"] = vm
-    data["memory"] = int(parsed_xml.memory.text)
+    vm_memory = int(parsed_xml.memory.text)
 
-    return True, data
+    return True, vm_memory
 
 
-def format_vm_memory(config, data):
+def format_vm_memory(config, name, memory):
     """
     Format the output of a memory value in a nice table
     """
     output_list = []
 
+    name_length = 5
+    _name_length = len(name) + 1
+    if _name_length > name_length:
+        name_length = _name_length
+
     memory_length = 6
 
     output_list.append(
-        "{bold}{memory: <{memory_length}}{end_bold}".format(
+        "{bold}{name: <{name_length}}  \
+{memory: <{memory_length}}{end_bold}".format(
+            name_length=name_length,
             memory_length=memory_length,
             bold=ansiprint.bold(),
             end_bold=ansiprint.end(),
+            name="Name",
             memory="RAM (M)",
         )
     )
     output_list.append(
-        "{bold}{memory: <{memory_length}}{end_bold}".format(
+        "{bold}{name: <{name_length}}  \
+{memory: <{memory_length}}{end_bold}".format(
+            name_length=name_length,
             memory_length=memory_length,
             bold="",
             end_bold="",
-            memory=data["memory"],
+            name=name,
+            memory=memory,
         )
     )
     return "\n".join(output_list)
@@ -657,7 +677,7 @@ def vm_networks_add(
     from lxml.objectify import fromstring
     from lxml.etree import tostring
     from random import randint
-    import pvc.lib.network as pvc_network
+    import pvc.cli_lib.network as pvc_network
 
     network_exists, _ = pvc_network.net_info(config, network)
     if not network_exists:
@@ -926,9 +946,7 @@ def vm_networks_get(config, vm):
     except Exception:
         return False, "ERROR: Failed to parse XML data."
 
-    data = dict()
-    data["name"] = vm
-    data["networks"] = list()
+    network_data = list()
     for interface in parsed_xml.devices.find("interface"):
         mac_address = interface.mac.attrib.get("address")
         model = interface.model.attrib.get("type")
@@ -942,65 +960,76 @@ def vm_networks_get(config, vm):
         elif interface_type == "hostdev":
             network = "hostdev:{}".format(interface.source.attrib.get("dev"))
 
-        data["networks"].append(
-            {"network": network, "mac_address": mac_address, "model": model}
-        )
+        network_data.append((network, mac_address, model))
 
-    return True, data
+    return True, network_data
 
 
-def format_vm_networks(config, data):
+def format_vm_networks(config, name, networks):
     """
     Format the output of a network list in a nice table
     """
     output_list = []
 
-    network_length = 8
+    name_length = 5
+    vni_length = 8
     macaddr_length = 12
     model_length = 6
 
-    for network in data["networks"]:
-        _network_length = len(network["network"]) + 1
-        if _network_length > network_length:
-            network_length = _network_length
+    _name_length = len(name) + 1
+    if _name_length > name_length:
+        name_length = _name_length
 
-        _macaddr_length = len(network["mac_address"]) + 1
+    for network in networks:
+        _vni_length = len(network[0]) + 1
+        if _vni_length > vni_length:
+            vni_length = _vni_length
+
+        _macaddr_length = len(network[1]) + 1
         if _macaddr_length > macaddr_length:
             macaddr_length = _macaddr_length
 
-        _model_length = len(network["model"]) + 1
+        _model_length = len(network[2]) + 1
         if _model_length > model_length:
             model_length = _model_length
 
     output_list.append(
-        "{bold}{network: <{network_length}} \
+        "{bold}{name: <{name_length}}  \
+{vni: <{vni_length}} \
 {macaddr: <{macaddr_length}} \
 {model: <{model_length}}{end_bold}".format(
-            network_length=network_length,
+            name_length=name_length,
+            vni_length=vni_length,
             macaddr_length=macaddr_length,
             model_length=model_length,
             bold=ansiprint.bold(),
             end_bold=ansiprint.end(),
-            network="Network",
+            name="Name",
+            vni="Network",
             macaddr="MAC Address",
             model="Model",
         )
     )
     count = 0
-    for network in data["networks"]:
+    for network in networks:
+        if count > 0:
+            name = ""
         count += 1
         output_list.append(
-            "{bold}{network: <{network_length}} \
+            "{bold}{name: <{name_length}}  \
+{vni: <{vni_length}} \
 {macaddr: <{macaddr_length}} \
 {model: <{model_length}}{end_bold}".format(
-                network_length=network_length,
+                name_length=name_length,
+                vni_length=vni_length,
                 macaddr_length=macaddr_length,
                 model_length=model_length,
                 bold="",
                 end_bold="",
-                network=network["network"],
-                macaddr=network["mac_address"],
-                model=network["model"],
+                name=name,
+                vni=network[0],
+                macaddr=network[1],
+                model=network[2],
             )
         )
     return "\n".join(output_list)
@@ -1017,13 +1046,13 @@ def vm_volumes_add(config, vm, volume, disk_id, bus, disk_type, live, restart):
     from lxml.objectify import fromstring
     from lxml.etree import tostring
     from copy import deepcopy
-    import pvc.lib.storage as pvc_storage
+    import pvc.cli_lib.ceph as pvc_ceph
 
     if disk_type == "rbd":
         # Verify that the provided volume is valid
         vpool = volume.split("/")[0]
         vname = volume.split("/")[1]
-        retcode, retdata = pvc_storage.ceph_volume_info(config, vpool, vname)
+        retcode, retdata = pvc_ceph.ceph_volume_info(config, vpool, vname)
         if not retcode:
             return False, "Volume {} is not present in the cluster.".format(volume)
 
@@ -1241,9 +1270,7 @@ def vm_volumes_get(config, vm):
     except Exception:
         return False, "ERROR: Failed to parse XML data."
 
-    data = dict()
-    data["name"] = vm
-    data["volumes"] = list()
+    volume_data = list()
     for disk in parsed_xml.devices.find("disk"):
         protocol = disk.attrib.get("type")
         disk_id = disk.target.attrib.get("dev")
@@ -1258,52 +1285,58 @@ def vm_volumes_get(config, vm):
             protocol = "unknown"
             source = "unknown"
 
-        data["volumes"].append(
-            {"volume": source, "disk_id": disk_id, "protocol": protocol, "bus": bus}
-        )
+        volume_data.append((source, disk_id, protocol, bus))
 
-    return True, data
+    return True, volume_data
 
 
-def format_vm_volumes(config, data):
+def format_vm_volumes(config, name, volumes):
     """
     Format the output of a volume value in a nice table
     """
     output_list = []
 
+    name_length = 5
     volume_length = 7
     disk_id_length = 4
     protocol_length = 5
     bus_length = 4
 
-    for volume in data["volumes"]:
-        _volume_length = len(volume["volume"]) + 1
+    _name_length = len(name) + 1
+    if _name_length > name_length:
+        name_length = _name_length
+
+    for volume in volumes:
+        _volume_length = len(volume[0]) + 1
         if _volume_length > volume_length:
             volume_length = _volume_length
 
-        _disk_id_length = len(volume["disk_id"]) + 1
+        _disk_id_length = len(volume[1]) + 1
         if _disk_id_length > disk_id_length:
             disk_id_length = _disk_id_length
 
-        _protocol_length = len(volume["protocol"]) + 1
+        _protocol_length = len(volume[2]) + 1
         if _protocol_length > protocol_length:
             protocol_length = _protocol_length
 
-        _bus_length = len(volume["bus"]) + 1
+        _bus_length = len(volume[3]) + 1
         if _bus_length > bus_length:
             bus_length = _bus_length
 
     output_list.append(
-        "{bold}{volume: <{volume_length}} \
+        "{bold}{name: <{name_length}}  \
+{volume: <{volume_length}} \
 {disk_id: <{disk_id_length}} \
 {protocol: <{protocol_length}} \
 {bus: <{bus_length}}{end_bold}".format(
+            name_length=name_length,
             volume_length=volume_length,
             disk_id_length=disk_id_length,
             protocol_length=protocol_length,
             bus_length=bus_length,
             bold=ansiprint.bold(),
             end_bold=ansiprint.end(),
+            name="Name",
             volume="Volume",
             disk_id="Dev",
             protocol="Type",
@@ -1311,23 +1344,28 @@ def format_vm_volumes(config, data):
         )
     )
     count = 0
-    for volume in data["volumes"]:
+    for volume in volumes:
+        if count > 0:
+            name = ""
         count += 1
         output_list.append(
-            "{bold}{volume: <{volume_length}} \
+            "{bold}{name: <{name_length}}  \
+{volume: <{volume_length}} \
 {disk_id: <{disk_id_length}} \
 {protocol: <{protocol_length}} \
 {bus: <{bus_length}}{end_bold}".format(
+                name_length=name_length,
                 volume_length=volume_length,
                 disk_id_length=disk_id_length,
                 protocol_length=protocol_length,
                 bus_length=bus_length,
                 bold="",
                 end_bold="",
-                volume=volume["volume"],
-                disk_id=volume["disk_id"],
-                protocol=volume["protocol"],
-                bus=volume["bus"],
+                name=name,
+                volume=volume[0],
+                disk_id=volume[1],
+                protocol=volume[2],
+                bus=volume[3],
             )
         )
     return "\n".join(output_list)
@@ -1831,7 +1869,7 @@ def format_info(config, domain_information, long_output):
     return "\n".join(ainformation)
 
 
-def format_list(config, vm_list):
+def format_list(config, vm_list, raw):
     # Function to strip the "br" off of nets and return a nicer list
     def getNiceNetID(domain_information):
         # Network list
@@ -1850,6 +1888,13 @@ def format_list(config, vm_list):
             tag_list.append(tag["name"])
         return tag_list
 
+    # Handle raw mode since it just lists the names
+    if raw:
+        ainformation = list()
+        for vm in sorted(item["name"] for item in vm_list):
+            ainformation.append(vm)
+        return "\n".join(ainformation)
+
     vm_list_output = []
 
     # Determine optimal column widths

client-cli/setup.py

@@ -2,8 +2,8 @@ from setuptools import setup
 
 setup(
     name="pvc",
-    version="0.9.70",
-    packages=["pvc.cli", "pvc.lib"],
+    version="0.9.63",
+    packages=["pvc", "pvc.lib"],
     install_requires=[
         "Click",
         "PyYAML",
@@ -14,7 +14,7 @@ setup(
     ],
     entry_points={
         "console_scripts": [
-            "pvc = pvc.cli.cli:cli",
+            "pvc = pvc.pvc:cli",
         ],
     },
 )

debian/changelog

@@ -1,65 +1,3 @@
-pvc (0.9.70-0) unstable; urgency=high
-
-  * [Node Daemon] Fixes several compatibility issues for Debian 12 "Bookworm"
-
- -- Joshua M. Boniface <joshua@boniface.me>  Thu, 31 Aug 2023 14:15:54 -0400
-
-pvc (0.9.69-0) unstable; urgency=high
-
-  * [Node Daemon] Ensures that system load is always 2 decimal places on Bookworm
-  * [Node Daemon] Fixes bug blocking primary takeover at DNS Aggregator start if Patroni is down
-
- -- Joshua M. Boniface <joshua@boniface.me>  Tue, 29 Aug 2023 22:01:22 -0400
-
-pvc (0.9.68-0) unstable; urgency=high
-
-  * [CLI] Fixes another bug with network info view
-
- -- Joshua M. Boniface <joshua@boniface.me>  Sun, 27 Aug 2023 20:59:23 -0400
-
-pvc (0.9.67-0) unstable; urgency=high
-
-  * [CLI] Fixes several more bugs in the refactored CLI
-
- -- Joshua M. Boniface <joshua@boniface.me>  Sun, 27 Aug 2023 14:47:20 -0400
-
-pvc (0.9.66-0) unstable; urgency=high
-
-  * [CLI] Fixes a missing YAML import in CLI
-
- -- Joshua M. Boniface <joshua@boniface.me>  Sun, 27 Aug 2023 11:36:05 -0400
-
-pvc (0.9.65-0) unstable; urgency=high
-
-  * [CLI] Fixes a bug in the node list filtering command
-  * [CLI] Fixes a bug/default when no connection is specified
-
- -- Joshua M. Boniface <joshua@boniface.me>  Wed, 23 Aug 2023 01:56:57 -0400
-
-pvc (0.9.64-0) unstable; urgency=high
-
-  **Breaking Change [CLI]**: The CLI client root commands have been reorganized. The following commands have changed:
-
-   * `pvc cluster` -> `pvc connection` (all subcommands)
-   * `pvc task` -> `pvc cluster` (all subcommands)
-   * `pvc maintenance` -> `pvc cluster maintenance`
-   * `pvc status` -> `pvc cluster status`
-
-Ensure you have updated to the latest version of the PVC Ansible repository before deploying this version or using PVC Ansible oneshot playbooks for management.
-
-  **Breaking Change [CLI]**: The `--restart` option for VM configuration changes now has an explicit `--no-restart` to disable restarting, or a prompt if neither is specified; `--unsafe` no longer bypasses this prompt which was a bug. Applies to most `vm <cmd> set` commands like `vm vcpu set`, `vm memory set`, etc. All instances also feature restart confirmation afterwards, which, if `--restart` is provided, will prompt for confirmation unless `--yes` or `--unsafe` is specified.
-
-  **Breaking Change [CLI]**: The `--long` option previously on some `info` commands no longer exists; use `-f long`/`--format long` instead.
-
-  * [CLI] Significantly refactors the CLI client code for consistency and cleanliness
-  * [CLI] Implements `-f`/`--format` options for all `list` and `info` commands in a consistent way
-  * [CLI] Changes the behaviour of VM modification options with "--restart" to provide a "--no-restart"; defaults to a prompt if neither is specified and ignores the "--unsafe" global entirely
-  * [API] Fixes several bugs in the 3-debootstrap.py provisioner example script
-  * [Node] Fixes some bugs around VM shutdown on node flush
-  * [Documentation] Adds mentions of Ganeti and Harvester
-
- -- Joshua M. Boniface <joshua@boniface.me>  Fri, 18 Aug 2023 12:20:43 -0400
-
 pvc (0.9.63-0) unstable; urgency=high
 
   * Mentions Ganeti in the docs

debian/control

@@ -16,7 +16,7 @@ Description: Parallel Virtual Cluster node daemon (Python 3)
 
 Package: pvc-daemon-api
 Architecture: all
-Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml, python3-flask-migrate, fio
+Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml, python3-flask-migrate, python3-flask-script, fio
 Description: Parallel Virtual Cluster API daemon (Python 3)
  A KVM/Zookeeper/Ceph-based VM and private cloud manager
  .

node-daemon/pvcnoded/Daemon.py

@@ -49,7 +49,7 @@ import re
 import json
 
 # Daemon version
-version = "0.9.70"
+version = "0.9.63"
 
 
 ##########################################################

node-daemon/pvcnoded/dnsmasq-zookeeper-leases.py

@@ -77,7 +77,7 @@ def connect_zookeeper():
 
     with open(pvcnoded_config_file, "r") as cfgfile:
         try:
-            o_config = yaml.load(cfgfile, yaml.SafeLoader)
+            o_config = yaml.load(cfgfile)
         except Exception as e:
             print(
                 "ERROR: Failed to parse configuration file: {}".format(e),

node-daemon/pvcnoded/objects/NodeInstance.py

@@ -620,12 +620,9 @@ class NodeInstance(object):
         for network in self.d_network:
             self.d_network[network].startDHCPServer()
         # 9. Start DNS aggregator; just continue if we fail
-        try:
-            if not patroni_failed:
-                self.dns_aggregator.start_aggregator()
-            else:
-                raise
-        except Exception:
+        if not patroni_failed:
+            self.dns_aggregator.start_aggregator()
+        else:
             self.logger.out(
                 "Not starting DNS aggregator due to Patroni failures", state="e"
             )
@@ -793,19 +790,6 @@ class NodeInstance(object):
                 self.flush_stopper = False
                 return
 
-            # Wait for a VM in "restart" or "shutdown" state to complete transition
-            while self.zkhandler.read(("domain.state", dom_uuid)) in [
-                "restart",
-                "shutdown",
-            ]:
-                self.logger.out(
-                    'Waiting 2s for VM state change completion for VM "{}"'.format(
-                        dom_uuid
-                    ),
-                    state="i",
-                )
-                time.sleep(2)
-
             self.logger.out(
                 'Selecting target to migrate VM "{}"'.format(dom_uuid), state="i"
             )
@@ -822,19 +806,17 @@ class NodeInstance(object):
 
             if target_node is None:
                 self.logger.out(
-                    'Failed to find migration target for running VM "{}"; shutting down and setting autostart flag'.format(
+                    'Failed to find migration target for VM "{}"; shutting down and setting autostart flag'.format(
                         dom_uuid
                     ),
                     state="e",
                 )
-
-                if self.zkhandler.read(("domain.state", dom_uuid)) in ["start"]:
-                    self.zkhandler.write(
-                        [
-                            (("domain.state", dom_uuid), "shutdown"),
-                            (("domain.meta.autostart", dom_uuid), "True"),
-                        ]
-                    )
+                self.zkhandler.write(
+                    [
+                        (("domain.state", dom_uuid), "shutdown"),
+                        (("domain.meta.autostart", dom_uuid), "True"),
+                    ]
+                )
             else:
                 self.logger.out(
                     'Migrating VM "{}" to node "{}"'.format(dom_uuid, target_node),

node-daemon/pvcnoded/util/keepalive.py

@@ -338,21 +338,8 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
             line = re.sub(r"\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))", "", line)
             # Split it for parsing
             line = line.split()
-
-            # Ceph 14 format:
-            #  ['|', '0', '|', 'hv1.p.u.bonilan.net', '|', '318G', '|', '463G', '|', '213', '|', '1430k', '|', '22', '|', '124k', '|', 'exists,up', '|']
-            # Ceph 16 format:
-            #  ['0', 'hv1.t.u.bonilan.net', '2489M', '236G', '0', '0', '0', '0', 'exists,up']
-
-            # Bypass obviously invalid lines
-            if len(line) < 1:
-                continue
-            elif line[0] == "+":
-                continue
-
-            # If line begins with | and second entry is a digit (i.e. OSD ID)
-            if line[0] == "|" and line[1].isdigit():
-                # Parse the line in Ceph 14 format
+            if len(line) > 1 and line[1].isdigit():
+                # This is an OSD line so parse it
                 osd_id = line[1]
                 node = line[3].split(".")[0]
                 used = line[5]
@@ -362,39 +349,20 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
                 rd_ops = line[13]
                 rd_data = line[15]
                 state = line[17]
-            # If first entry is a digit (i.e. OSD ID)
-            elif line[0].isdigit():
-                # Parse the line in Ceph 16 format
-                osd_id = line[0]
-                node = line[1].split(".")[0]
-                used = line[2]
-                avail = line[3]
-                wr_ops = line[4]
-                wr_data = line[5]
-                rd_ops = line[6]
-                rd_data = line[7]
-                state = line[8]
-            # Otherwise, it's the header line and is ignored
-            else:
-                continue
-
-            # I don't know why 2018 me used this construct instead of a normal
-            # dictionary update, but it works so not changing it.
-            # ref: bfbe9188ce830381f3f2fa1da11f1973f08eca8c
-            osd_status.update(
-                {
-                    str(osd_id): {
-                        "node": node,
-                        "used": used,
-                        "avail": avail,
-                        "wr_ops": wr_ops,
-                        "wr_data": wr_data,
-                        "rd_ops": rd_ops,
-                        "rd_data": rd_data,
-                        "state": state,
+                osd_status.update(
+                    {
+                        str(osd_id): {
+                            "node": node,
+                            "used": used,
+                            "avail": avail,
+                            "wr_ops": wr_ops,
+                            "wr_data": wr_data,
+                            "rd_ops": rd_ops,
+                            "rd_data": rd_data,
+                            "state": state,
+                        }
                     }
-                }
-            )
+                )
 
         # Merge them together into a single meaningful dict
         if debug:
@@ -785,7 +753,7 @@ def node_keepalive(logger, config, zkhandler, this_node, monitoring_instance):
     this_node.memtotal = int(psutil.virtual_memory().total / 1024 / 1024)
     this_node.memused = int(psutil.virtual_memory().used / 1024 / 1024)
     this_node.memfree = int(psutil.virtual_memory().free / 1024 / 1024)
-    this_node.cpuload = round(os.getloadavg()[0], 2)
+    this_node.cpuload = os.getloadavg()[0]
 
     # Join against running threads
     if config["enable_hypervisor"]:

test-cluster.sh

@@ -1,54 +1,31 @@
 #!/usr/bin/env bash
 
+set -o errexit
+
 if [[ -z ${1} ]]; then
     echo "Please specify a cluster to run tests against."
     exit 1
 fi
 test_cluster="${1}"
-shift
-
-if [[ ${1} == "--test-dangerously" ]]; then
-    test_dangerously="y"
-else
-    test_dangerously=""
-fi
 
 _pvc() {
-    echo "> pvc --connection ${test_cluster} $@"
-    pvc --quiet --connection ${test_cluster} "$@"
+    echo "> pvc --cluster ${test_cluster} $@"
+    pvc --quiet --cluster ${test_cluster} "$@"
     sleep 1
 }
 
 time_start=$(date +%s)
 
-set -o errexit
-
-pushd $( git rev-parse --show-toplevel ) &>/dev/null
-
 # Cluster tests
-_pvc connection list
-_pvc connection detail
-
-_pvc cluster maintenance on
-_pvc cluster maintenance off
-_pvc cluster status
+_pvc maintenance on
+_pvc maintenance off
 backup_tmp=$(mktemp)
-_pvc cluster backup --file ${backup_tmp}
-if [[ -n ${test_dangerously} ]]; then
-    # This is dangerous, so don't test it unless option given
-    _pvc cluster restore --yes --file ${backup_tmp}
-fi
+_pvc task backup --file ${backup_tmp}
+_pvc task restore --yes --file ${backup_tmp}
 rm ${backup_tmp} || true
 
 # Provisioner tests
-_pvc provisioner profile list test || true
-_pvc provisioner template system add --vcpus 1 --vram 1024 --serial --vnc --vnc-bind 0.0.0.0 --node-limit hv1 --node-selector mem --node-autostart --migration-method live system-test || true
-_pvc provisioner template network add network-test || true
-_pvc provisioner template network vni add network-test 10000 || true
-_pvc provisioner template storage add storage-test || true
-_pvc provisioner template storage disk add --pool vms --size 8 --filesystem ext4 --mountpoint / storage-test sda || true
-_pvc provisioner script add script-test $( find . -name "3-debootstrap.py" ) || true
-_pvc provisioner profile add --profile-type provisioner --system-template system-test --network-template network-test --storage-template storage-test --userdata empty --script script-test --script-arg deb_release=bullseye test || true
+_pvc provisioner profile list test
 _pvc provisioner create --wait testx test
 sleep 30
 
@@ -59,7 +36,7 @@ _pvc vm shutdown --yes --wait testx
 _pvc vm start testx
 sleep 30
 _pvc vm stop --yes testx
-_pvc vm disable --yes testx
+_pvc vm disable testx
 _pvc vm undefine --yes testx
 _pvc vm define --target hv3 --tag pvc-test ${vm_tmp}
 _pvc vm start testx
@@ -72,21 +49,21 @@ _pvc vm unmigrate --wait testx
 sleep 5
 _pvc vm move --wait --target hv1 testx
 sleep 5
-_pvc vm meta testx --limit hv1 --node-selector vms --method live --profile test --no-autostart
+_pvc vm meta testx --limit hv1 --selector vms --method live --profile test --no-autostart
 _pvc vm tag add testx mytag
 _pvc vm tag get testx
 _pvc vm list --tag mytag
 _pvc vm tag remove testx mytag
 _pvc vm network get testx
-_pvc vm vcpu set --no-restart testx 4
+_pvc vm vcpu set testx 4
 _pvc vm vcpu get testx
-_pvc vm memory set --no-restart testx 4096
+_pvc vm memory set testx 4096
 _pvc vm memory get testx
-_pvc vm vcpu set --no-restart testx 2
+_pvc vm vcpu set testx 2
 _pvc vm memory set testx 2048 --restart --yes
-sleep 15
+sleep 5
 _pvc vm list testx
-_pvc vm info --format long testx
+_pvc vm info --long testx
 rm ${vm_tmp} || true
 
 # Node tests
@@ -100,7 +77,6 @@ _pvc node flush --wait hv1
 _pvc node ready --wait hv1
 _pvc node list hv1
 _pvc node info hv1
-sleep 15
 
 # Network tests
 _pvc network add 10001 --description testing --type managed --domain testing.local --ipnet 10.100.100.0/24 --gateway 10.100.100.1 --dhcp --dhcp-start 10.100.100.100 --dhcp-end 10.100.100.199
@@ -108,7 +84,7 @@ sleep 5
 _pvc vm network add --restart --yes testx 10001
 sleep 30
 _pvc vm network remove --restart --yes testx 10001
-sleep 15
+sleep 5
 
 _pvc network acl add 10001 --in --description test-acl --order 0 --rule "'ip daddr 10.0.0.0/8 counter'"
 _pvc network acl list 10001
@@ -119,34 +95,31 @@ _pvc network dhcp remove --yes 10001 12:34:56:78:90:ab
 
 _pvc network modify --domain test10001.local 10001
 _pvc network list
-_pvc network info --format long 10001
+_pvc network info --long 10001
 
 # Network-VM interaction tests
 _pvc vm network add testx 10001 --model virtio --restart --yes
 sleep 30
 _pvc vm network get testx
 _pvc vm network remove testx 10001 --restart --yes
-sleep 15
+sleep 5
 
 _pvc network remove --yes 10001
 
 # Storage tests
 _pvc storage status
 _pvc storage util
-if [[ -n ${test_dangerously} ]]; then
-    # This is dangerous, so don't test it unless option given
-    _pvc storage osd set noout
-    _pvc storage osd out 0
-    _pvc storage osd in 0
-    _pvc storage osd unset noout
-fi
+_pvc storage osd set noout
+_pvc storage osd out 0
+_pvc storage osd in 0
+_pvc storage osd unset noout
 _pvc storage osd list
 _pvc storage pool add testing 64 --replcfg "copies=3,mincopies=2"
 sleep 5
 _pvc storage pool list
 _pvc storage volume add testing testx 1G
-_pvc storage volume resize --yes testing testx 2G
-_pvc storage volume rename --yes testing testx testerX
+_pvc storage volume resize testing testx 2G
+_pvc storage volume rename testing testx testerX
 _pvc storage volume clone testing testerX testerY
 _pvc storage volume list --pool testing
 _pvc storage volume snapshot add testing testerX asnapshotX
@@ -159,7 +132,7 @@ _pvc vm volume add testx --type rbd --disk-id sdh --bus scsi testing/testerY --r
 sleep 30
 _pvc vm volume get testx
 _pvc vm volume remove testx testing/testerY --restart --yes
-sleep 15
+sleep 5
 
 _pvc storage volume remove --yes testing testerY
 _pvc storage volume remove --yes testing testerX
@@ -169,14 +142,6 @@ _pvc storage pool remove --yes testing
 _pvc vm stop --yes testx
 _pvc vm remove --yes testx
 
-_pvc provisioner profile remove --yes test
-_pvc provisioner script remove --yes script-test
-_pvc provisioner template system remove --yes system-test
-_pvc provisioner template network remove --yes network-test
-_pvc provisioner template storage remove --yes storage-test
-
-popd
-
 time_end=$(date +%s)
 
 echo