Bump version to 0.9.46

Prevents a daemon from waiting forever to terminate if it is primary,
and avoids this entirely if there is only a single node in the cluster.

.version

@@ -1 +1 @@
-0.9.43
+0.9.46

CHANGELOG.md

@@ -1,5 +1,31 @@
 ## PVC Changelog
 
+###### [v0.9.46](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.46)
+
+  * [API] Fixes bugs with legacy benchmark display
+  * [API] Fixes a bug around cloned image sizes
+  * [API] Removes extraneous message text in provisioner create command
+  * [API] Corrects bugs around fuzzy matching
+  * [CLI] Adds auditing for PVC CLI to local syslog
+  * [CLI] Adds --yes bypass for benchmark command
+  * [Node Daemon/API/CLI] Adds support for "detect" strings when specifying OSD or OSDDB devices
+  * [Node Daemon] Fixes a bug when removing OSDs
+  * [Node Daemon] Fixes a single-node cluster shutdown deadlock
+
+###### [v0.9.45](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.45)
+
+  * [Node Daemon] Fixes an ordering issue with pvcnoded.service
+  * [CLI Client] Fixes bad calls to echo() without argument
+
+###### [v0.9.44](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.44)
+
+  * [Node Daemon] Adds a Munin plugin for Ceph utilization
+  * [CLI] Fixes timeouts for long-running API commands
+
+###### [v0.9.44](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.44)
+
+  * [CLI] Fixes timeout issues with long-running API commands
+
 ###### [v0.9.43](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.43)
 
   * [Packaging] Fixes a bad test in postinst            

api-daemon/pvcapid/Daemon.py

@@ -25,7 +25,7 @@ import yaml
 from distutils.util import strtobool as dustrtobool
 
 # Daemon version
-version = "0.9.43"
+version = "0.9.46"
 
 # API version
 API_VERSION = 1.0

api-daemon/pvcapid/flaskapi.py

@@ -3849,7 +3849,7 @@ class API_Storage_Ceph_OSDDB_Root(Resource):
             {
                 "name": "device",
                 "required": True,
-                "helptext": "A valid device must be specified.",
+                "helptext": "A valid device or detect string must be specified.",
             },
         ]
     )
@@ -3871,7 +3871,7 @@ class API_Storage_Ceph_OSDDB_Root(Resource):
             name: device
             type: string
             required: true
-            description: The block device (e.g. "/dev/sdb", "/dev/disk/by-path/...", etc.) to create the OSD DB volume group on
+            description: The block device (e.g. "/dev/sdb", "/dev/disk/by-path/...", etc.) or detect string ("detect:NAME:SIZE:ID") to create the OSD DB volume group on
         responses:
           200:
             description: OK
@@ -4003,7 +4003,7 @@ class API_Storage_Ceph_OSD_Root(Resource):
             {
                 "name": "device",
                 "required": True,
-                "helptext": "A valid device must be specified.",
+                "helptext": "A valid device or detect string must be specified.",
             },
             {
                 "name": "weight",
@@ -4040,7 +4040,7 @@ class API_Storage_Ceph_OSD_Root(Resource):
             name: device
             type: string
             required: true
-            description: The block device (e.g. "/dev/sdb", "/dev/disk/by-path/...", etc.) to create the OSD on
+            description: The block device (e.g. "/dev/sdb", "/dev/disk/by-path/...", etc.) or detect string ("detect:NAME:SIZE:ID") to create the OSD on
           - in: query
             name: weight
             type: number

api-daemon/pvcapid/provisioner.py

@@ -1442,11 +1442,17 @@ def create_vm(
                 )
             if not volume["pool"] in pools:
                 pools[volume["pool"]] = int(
-                    volume_data["stats"]["size"].replace("G", "")
+                    pvc_ceph.format_bytes_fromhuman(volume_data["stats"]["size"])
+                    / 1024
+                    / 1024
+                    / 1024
                 )
             else:
                 pools[volume["pool"]] += int(
-                    volume_data["stats"]["size"].replace("G", "")
+                    pvc_ceph.format_bytes_fromhuman(volume_data["stats"]["size"])
+                    / 1024
+                    / 1024
+                    / 1024
                 )
         else:
             if not volume["pool"] in pools:
@@ -2080,7 +2086,7 @@ def create_vm(
     del zkhandler
 
     return {
-        "status": 'VM "{}" with profile "{}" has been provisioned and started successfully'.format(
+        "status": 'VM "{}" with profile "{}" has been provisioned successfully'.format(
             vm_name, vm_profile
         ),
         "current": 10,

build-unstable-deb.sh

@@ -15,7 +15,7 @@ cp -a debian/changelog client-cli/setup.py ${tmpdir}/
 cp -a node-daemon/pvcnoded/Daemon.py ${tmpdir}/node-Daemon.py
 cp -a api-daemon/pvcapid/Daemon.py ${tmpdir}/api-Daemon.py
 # Replace the "base" version with the git revision version
-sed -i "s/version = '${base_ver}'/version = '${new_ver}'/" node-daemon/pvcnoded/Daemon.py api-daemon/pvcapid/Daemon.py client-cli/setup.py
+sed -i "s/version = \"${base_ver}\"/version = \"${new_ver}\"/" node-daemon/pvcnoded/Daemon.py api-daemon/pvcapid/Daemon.py client-cli/setup.py
 sed -i "s/${base_ver}-0/${new_ver}/" debian/changelog 
 cat <<EOF > debian/changelog
 pvc (${new_ver}) unstable; urgency=medium

client-cli/pvc/cli_lib/ceph.py

@@ -21,7 +21,7 @@
 
 import math
 
-from json import dumps
+from json import dumps, loads
 from requests_toolbelt.multipart.encoder import (
     MultipartEncoder,
     MultipartEncoderMonitor,
@@ -1648,6 +1648,8 @@ def ceph_benchmark_list(config, job):
 
 
 def get_benchmark_list_results_legacy(benchmark_data):
+    if isinstance(benchmark_data, str):
+        benchmark_data = loads(benchmark_data)
     benchmark_bandwidth = dict()
     benchmark_iops = dict()
     for test in ["seq_read", "seq_write", "rand_read_4K", "rand_write_4K"]:
@@ -1732,7 +1734,7 @@ def format_list_benchmark(config, benchmark_information):
 
     for benchmark in benchmark_information:
         benchmark_job = benchmark["job"]
-        benchmark_format = benchmark["test_format"]  # noqa: F841
+        benchmark_format = benchmark.get("test_format", 0)  # noqa: F841
 
         _benchmark_job_length = len(benchmark_job)
         if _benchmark_job_length > benchmark_job_length:
@@ -1837,7 +1839,7 @@ def format_list_benchmark(config, benchmark_information):
 
     for benchmark in benchmark_information:
         benchmark_job = benchmark["job"]
-        benchmark_format = benchmark["test_format"]  # noqa: F841
+        benchmark_format = benchmark.get("test_format", 0)  # noqa: F841
 
         if benchmark["benchmark_result"] == "Running":
             seq_benchmark_bandwidth = "Running"

client-cli/pvc/cli_lib/common.py

@@ -123,8 +123,10 @@ def call_api(
     params=None,
     data=None,
     files=None,
-    timeout=3,
 ):
+    # Set the connect timeout to 3 seconds but extremely long (48 hour) data timeout
+    timeout = (3.05, 172800)
+
     # Craft the URI
     uri = "{}://{}{}{}".format(
         config["api_scheme"], config["api_host"], config["api_prefix"], request_uri

client-cli/pvc/cli_lib/vm.py

@@ -377,19 +377,12 @@ def vm_state(config, vm, target_state, force=False, wait=False):
     API arguments: state={state}, wait={wait}
     API schema: {"message":"{data}"}
     """
-    if wait or target_state == "disable":
-        timeout = 300
-    else:
-        timeout = 3
-
     params = {
         "state": target_state,
         "force": str(force).lower(),
         "wait": str(wait).lower(),
     }
-    response = call_api(
-        config, "post", "/vm/{vm}/state".format(vm=vm), params=params, timeout=timeout
-    )
+    response = call_api(config, "post", "/vm/{vm}/state".format(vm=vm), params=params)
 
     if response.status_code == 200:
         retstatus = True

client-cli/pvc/pvc.py

@@ -28,8 +28,11 @@ import time
 import colorama
 import yaml
 import json
+import syslog
 import lxml.etree as etree
 
+from sys import argv
+
 from distutils.util import strtobool
 
 from functools import wraps
@@ -51,6 +54,22 @@ default_store_data = {"cfgfile": "/etc/pvc/pvcapid.yaml"}
 config = dict()
 
 
+#
+# Audit function
+#
+def audit():
+    args = argv
+    args[0] = "pvc"
+    syslog.openlog(facility=syslog.LOG_AUTH)
+    syslog.syslog(
+        'client audit: command "{}" by user "{}"'.format(
+            " ".join(args),
+            os.environ.get("USER", None),
+        )
+    )
+    syslog.closelog()
+
+
 #
 # Version function
 #
@@ -364,7 +383,7 @@ def cluster_list(raw):
     if not raw:
         # Display the data nicely
         echo("Available clusters:")
-        echo()
+        echo("")
         echo(
             "{bold}{name: <{name_length}} {description: <{description_length}} {address: <{address_length}} {port: <{port_length}} {scheme: <{scheme_length}} {api_key: <{api_key_length}}{end_bold}".format(
                 bold=ansiprint.bold(),
@@ -481,7 +500,7 @@ def node_secondary(node, wait):
             "      These jobs will continue executing, but status will not be visible until the current"
         )
         echo("      node returns to primary state.")
-        echo()
+        echo("")
 
     retcode, retmsg = pvc_node.node_coordinator_state(config, node, "secondary")
     if not retcode:
@@ -534,7 +553,7 @@ def node_primary(node, wait):
             "      These jobs will continue executing, but status will not be visible until the current"
         )
         echo("      node returns to primary state.")
-        echo()
+        echo("")
 
     retcode, retmsg = pvc_node.node_coordinator_state(config, node, "primary")
     if not retcode:
@@ -3159,20 +3178,29 @@ def ceph_benchmark():
 # pvc storage benchmark run
 ###############################################################################
 @click.command(name="run", short_help="Run a storage benchmark.")
+@click.option(
+    "-y",
+    "--yes",
+    "confirm_flag",
+    is_flag=True,
+    default=False,
+    help="Confirm the run",
+)
 @click.argument("pool")
 @cluster_req
-def ceph_benchmark_run(pool):
+def ceph_benchmark_run(confirm_flag, pool):
     """
     Run a storage benchmark on POOL in the background.
     """
-    try:
-        click.confirm(
-            "NOTE: Storage benchmarks take approximately 10 minutes to run and generate significant load on the cluster; they should be run sparingly. Continue",
-            prompt_suffix="? ",
-            abort=True,
-        )
-    except Exception:
-        exit(0)
+    if not confirm_flag and not config["unsafe"]:
+        try:
+            click.confirm(
+                "NOTE: Storage benchmarks take approximately 10 minutes to run and generate significant load on the cluster; they should be run sparingly. Continue",
+                prompt_suffix="? ",
+                abort=True,
+            )
+        except Exception:
+            exit(0)
 
     retcode, retmsg = pvc_ceph.ceph_benchmark_run(config, pool)
     cleanup(retcode, retmsg)
@@ -3253,7 +3281,9 @@ def ceph_osd():
 @cluster_req
 def ceph_osd_create_db_vg(node, device, confirm_flag):
     """
-    Create a new Ceph OSD database volume group on node NODE with block device DEVICE. DEVICE must be a valid raw block device, one of e.g. '/dev/sda', '/dev/nvme0n1', '/dev/disk/by-path/...', '/dev/disk/by-id/...', etc. Using partitions is not supported.
+    Create a new Ceph OSD database volume group on node NODE with block device DEVICE. DEVICE must be a valid raw block device (e.g. '/dev/nvme0n1', '/dev/disk/by-path/...') or a "detect" string. Using partitions is not supported.
+
+    A "detect" string is a string in the form "detect:<NAME>:<HUMAN-SIZE>:<ID>". Detect strings allow for automatic determination of Linux block device paths from known basic information about disks by leveraging "lsscsi" on the target host. The "NAME" should be some descriptive identifier, for instance the manufacturer (e.g. "INTEL"), the "HUMAN-SIZE" should be the labeled human-readable size of the device (e.g. "480GB", "1.92TB"), and "ID" specifies the Nth 0-indexed device which matches the "NAME" and "HUMAN-SIZE" values (e.g. "2" would match the third device with the corresponding "NAME" and "HUMAN-SIZE"). When matching against sizes, there is +/- 3% flexibility to account for base-1000 vs. base-1024 differences and rounding errors. The "NAME" may contain whitespace but if so the entire detect string should be quoted, and is case-insensitive. More information about detect strings can be found in the pvcbootstrapd manual.
 
     This volume group will be used for Ceph OSD database and WAL functionality if the '--ext-db' flag is passed to newly-created OSDs during 'pvc storage osd add'. DEVICE should be an extremely fast SSD device (NVMe, Intel Optane, etc.) which is significantly faster than the normal OSD disks and with very high write endurance. Only one OSD database volume group on a single physical device is supported per node, so it must be fast and large enough to act as an effective OSD database device for all OSDs on the node. Attempting to add additional database volume groups after the first will fail.
     """
@@ -3315,7 +3345,9 @@ def ceph_osd_create_db_vg(node, device, confirm_flag):
 @cluster_req
 def ceph_osd_add(node, device, weight, ext_db_flag, ext_db_ratio, confirm_flag):
     """
-    Add a new Ceph OSD on node NODE with block device DEVICE. DEVICE must be a valid raw block device, one of e.g. '/dev/sda', '/dev/nvme0n1', '/dev/disk/by-path/...', '/dev/disk/by-id/...', etc. Using partitions is not supported.
+    Add a new Ceph OSD on node NODE with block device DEVICE. DEVICE must be a valid raw block device (e.g. '/dev/sda', '/dev/nvme0n1', '/dev/disk/by-path/...', '/dev/disk/by-id/...') or a "detect" string. Using partitions is not supported.
+
+    A "detect" string is a string in the form "detect:<NAME>:<HUMAN-SIZE>:<ID>". Detect strings allow for automatic determination of Linux block device paths from known basic information about disks by leveraging "lsscsi" on the target host. The "NAME" should be some descriptive identifier, for instance the manufacturer (e.g. "INTEL"), the "HUMAN-SIZE" should be the labeled human-readable size of the device (e.g. "480GB", "1.92TB"), and "ID" specifies the Nth 0-indexed device which matches the "NAME" and "HUMAN-SIZE" values (e.g. "2" would match the third device with the corresponding "NAME" and "HUMAN-SIZE"). When matching against sizes, there is +/- 3% flexibility to account for base-1000 vs. base-1024 differences and rounding errors. The "NAME" may contain whitespace but if so the entire detect string should be quoted, and is case-insensitive. More information about detect strings can be found in the pvcbootstrapd manual.
 
     The weight of an OSD should reflect the ratio of the OSD to other OSDs in the storage cluster. For example, if all OSDs are the same size as recommended for PVC, 1 (the default) is a valid weight so that all are treated identically. If a new OSD is added later which is 4x the size of the existing OSDs, the new OSD's weight should then be 4 to tell the cluster that 4x the data can be stored on the OSD. Weights can also be tweaked for performance reasons, since OSDs with more data will incur more I/O load. For more information about CRUSH weights, please see the Ceph documentation.
 
@@ -5329,7 +5361,7 @@ def provisioner_create(name, profile, wait_flag, define_flag, start_flag, script
         task_id = retdata
 
         echo("Task ID: {}".format(task_id))
-        echo()
+        echo("")
 
         # Wait for the task to start
         echo("Waiting for task to start...", nl=False)
@@ -5340,7 +5372,7 @@ def provisioner_create(name, profile, wait_flag, define_flag, start_flag, script
                 break
             echo(".", nl=False)
         echo(" done.")
-        echo()
+        echo("")
 
         # Start following the task state, updating progress as we go
         total_task = task_status.get("total")
@@ -5371,7 +5403,7 @@ def provisioner_create(name, profile, wait_flag, define_flag, start_flag, script
             if task_status.get("state") == "SUCCESS":
                 bar.update(total_task - last_task)
 
-        echo()
+        echo("")
         retdata = task_status.get("state") + ": " + task_status.get("status")
 
     cleanup(retcode, retdata)
@@ -5702,6 +5734,8 @@ def cli(_cluster, _debug, _quiet, _unsafe, _colour):
             )
             echo("", err=True)
 
+    audit()
+
 
 #
 # Click command tree

client-cli/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="pvc",
-    version="0.9.43",
+    version="0.9.46",
     packages=["pvc", "pvc.cli_lib"],
     install_requires=[
         "Click",

daemon-common/ceph.py

@@ -375,7 +375,7 @@ def get_list_osd(zkhandler, limit, is_fuzzy=True):
     for osd in full_osd_list:
         if limit:
             try:
-                if re.match(limit, osd):
+                if re.fullmatch(limit, osd):
                     osd_list.append(getOSDInformation(zkhandler, osd))
             except Exception as e:
                 return False, "Regex Error: {}".format(e)
@@ -496,16 +496,19 @@ def remove_pool(zkhandler, name):
 def get_list_pool(zkhandler, limit, is_fuzzy=True):
     full_pool_list = zkhandler.children("base.pool")
 
-    if limit:
-        if not is_fuzzy:
-            limit = "^" + limit + "$"
+    if is_fuzzy and limit:
+        # Implicitly assume fuzzy limits
+        if not re.match(r"\^.*", limit):
+            limit = ".*" + limit
+        if not re.match(r".*\$", limit):
+            limit = limit + ".*"
 
     get_pool_info = dict()
     for pool in full_pool_list:
         is_limit_match = False
         if limit:
             try:
-                if re.match(limit, pool):
+                if re.fullmatch(limit, pool):
                     is_limit_match = True
             except Exception as e:
                 return False, "Regex Error: {}".format(e)
@@ -848,15 +851,12 @@ def get_list_volume(zkhandler, pool, limit, is_fuzzy=True):
 
     full_volume_list = getCephVolumes(zkhandler, pool)
 
-    if limit:
-        if not is_fuzzy:
-            limit = "^" + limit + "$"
-        else:
-            # Implicitly assume fuzzy limits
-            if not re.match(r"\^.*", limit):
-                limit = ".*" + limit
-            if not re.match(r".*\$", limit):
-                limit = limit + ".*"
+    if is_fuzzy and limit:
+        # Implicitly assume fuzzy limits
+        if not re.match(r"\^.*", limit):
+            limit = ".*" + limit
+        if not re.match(r".*\$", limit):
+            limit = limit + ".*"
 
     get_volume_info = dict()
     for volume in full_volume_list:
@@ -867,7 +867,7 @@ def get_list_volume(zkhandler, pool, limit, is_fuzzy=True):
         if limit:
             # Try to match the limit against the volume name
             try:
-                if re.match(limit, volume_name):
+                if re.fullmatch(limit, volume_name):
                     is_limit_match = True
             except Exception as e:
                 return False, "Regex Error: {}".format(e)
@@ -1073,7 +1073,7 @@ def get_list_snapshot(zkhandler, pool, volume, limit, is_fuzzy=True):
         pool_name, volume_name = volume.split("/")
         if limit:
             try:
-                if re.match(limit, snapshot_name):
+                if re.fullmatch(limit, snapshot_name):
                     snapshot_list.append(
                         {
                             "pool": pool_name,

daemon-common/network.py

@@ -665,16 +665,20 @@ def get_list(zkhandler, limit, is_fuzzy=True):
     net_list = []
     full_net_list = zkhandler.children("base.network")
 
+    if is_fuzzy and limit:
+        # Implicitly assume fuzzy limits
+        if not re.match(r"\^.*", limit):
+            limit = ".*" + limit
+        if not re.match(r".*\$", limit):
+            limit = limit + ".*"
+
     for net in full_net_list:
         description = zkhandler.read(("network", net))
         if limit:
             try:
-                if not is_fuzzy:
-                    limit = "^" + limit + "$"
-
-                if re.match(limit, net):
+                if re.fullmatch(limit, net):
                     net_list.append(getNetworkInformation(zkhandler, net))
-                if re.match(limit, description):
+                if re.fullmatch(limit, description):
                     net_list.append(getNetworkInformation(zkhandler, net))
             except Exception as e:
                 return False, "Regex Error: {}".format(e)
@@ -700,25 +704,19 @@ def get_list_dhcp(zkhandler, network, limit, only_static=False, is_fuzzy=True):
         full_dhcp_list = getNetworkDHCPReservations(zkhandler, net_vni)
         full_dhcp_list += getNetworkDHCPLeases(zkhandler, net_vni)
 
-    if limit:
-        try:
-            if not is_fuzzy:
-                limit = "^" + limit + "$"
-
-            # Implcitly assume fuzzy limits
-            if not re.match(r"\^.*", limit):
-                limit = ".*" + limit
-            if not re.match(r".*\$", limit):
-                limit = limit + ".*"
-        except Exception as e:
-            return False, "Regex Error: {}".format(e)
+    if is_fuzzy and limit:
+        # Implicitly assume fuzzy limits
+        if not re.match(r"\^.*", limit):
+            limit = ".*" + limit
+        if not re.match(r".*\$", limit):
+            limit = limit + ".*"
 
     for lease in full_dhcp_list:
         valid_lease = False
         if limit:
-            if re.match(limit, lease):
+            if re.fullmatch(limit, lease):
                 valid_lease = True
-            if re.match(limit, lease):
+            if re.fullmatch(limit, lease):
                 valid_lease = True
         else:
             valid_lease = True
@@ -748,23 +746,17 @@ def get_list_acl(zkhandler, network, limit, direction, is_fuzzy=True):
     acl_list = []
     full_acl_list = getNetworkACLs(zkhandler, net_vni, direction)
 
-    if limit:
-        try:
-            if not is_fuzzy:
-                limit = "^" + limit + "$"
-
-            # Implcitly assume fuzzy limits
-            if not re.match(r"\^.*", limit):
-                limit = ".*" + limit
-            if not re.match(r".*\$", limit):
-                limit = limit + ".*"
-        except Exception as e:
-            return False, "Regex Error: {}".format(e)
+    if is_fuzzy and limit:
+        # Implicitly assume fuzzy limits
+        if not re.match(r"\^.*", limit):
+            limit = ".*" + limit
+        if not re.match(r".*\$", limit):
+            limit = limit + ".*"
 
     for acl in full_acl_list:
         valid_acl = False
         if limit:
-            if re.match(limit, acl["description"]):
+            if re.fullmatch(limit, acl["description"]):
                 valid_acl = True
         else:
             valid_acl = True

daemon-common/node.py

@@ -237,13 +237,17 @@ def get_list(
     node_list = []
     full_node_list = zkhandler.children("base.node")
 
+    if is_fuzzy and limit:
+        # Implicitly assume fuzzy limits
+        if not re.match(r"\^.*", limit):
+            limit = ".*" + limit
+        if not re.match(r".*\$", limit):
+            limit = limit + ".*"
+
     for node in full_node_list:
         if limit:
             try:
-                if not is_fuzzy:
-                    limit = "^" + limit + "$"
-
-                if re.match(limit, node):
+                if re.fullmatch(limit, node):
                     node_list.append(getNodeInformation(zkhandler, node))
             except Exception as e:
                 return False, "Regex Error: {}".format(e)

daemon-common/vm.py

@@ -1227,9 +1227,9 @@ def get_list(zkhandler, node, state, tag, limit, is_fuzzy=True, negate=False):
         if limit:
             # Try to match the limit against the UUID (if applicable) and name
             try:
-                if is_limit_uuid and re.match(limit, vm):
+                if is_limit_uuid and re.fullmatch(limit, vm):
                     is_limit_match = True
-                if re.match(limit, name):
+                if re.fullmatch(limit, name):
                     is_limit_match = True
             except Exception as e:
                 return False, "Regex Error: {}".format(e)

debian/changelog

@@ -1,3 +1,37 @@
+pvc (0.9.46-0) unstable; urgency=high
+
+  * [API] Fixes bugs with legacy benchmark display
+  * [API] Fixes a bug around cloned image sizes
+  * [API] Removes extraneous message text in provisioner create command
+  * [API] Corrects bugs around fuzzy matching
+  * [CLI] Adds auditing for PVC CLI to local syslog
+  * [CLI] Adds --yes bypass for benchmark command
+  * [Node Daemon/API/CLI] Adds support for "detect" strings when specifying OSD or OSDDB devices
+  * [Node Daemon] Fixes a bug when removing OSDs
+  * [Node Daemon] Fixes a single-node cluster shutdown deadlock
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Tue, 28 Dec 2021 15:02:14 -0500
+
+pvc (0.9.45-0) unstable; urgency=high
+
+  * [Node Daemon] Fixes an ordering issue with pvcnoded.service
+  * [CLI Client] Fixes bad calls to echo() without argument
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Thu, 25 Nov 2021 09:34:20 -0500
+
+pvc (0.9.44-0) unstable; urgency=high
+
+  * [Node Daemon] Adds a Munin plugin for Ceph utilization
+  * [CLI] Fixes timeouts for long-running API commands
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Thu, 11 Nov 2021 16:20:38 -0500
+
+pvc (0.9.44-0) unstable; urgency=high
+
+  * [CLI] Fixes timeout issues with long-running API commands
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Thu, 11 Nov 2021 16:19:32 -0500
+
 pvc (0.9.43-0) unstable; urgency=high
 
   * [Packaging] Fixes a bad test in postinst            

docs/manuals/swagger.json

@@ -5034,7 +5034,7 @@
                         "type": "string"
                     },
                     {
-                        "description": "The block device (e.g. \"/dev/sdb\", \"/dev/disk/by-path/...\", etc.) to create the OSD on",
+                        "description": "The block device (e.g. \"/dev/sdb\", \"/dev/disk/by-path/...\", etc.) or detect string (\"detect:NAME:SIZE:ID\") to create the OSD on",
                         "in": "query",
                         "name": "device",
                         "required": true,
@@ -5194,7 +5194,7 @@
                         "type": "string"
                     },
                     {
-                        "description": "The block device (e.g. \"/dev/sdb\", \"/dev/disk/by-path/...\", etc.) to create the OSD DB volume group on",
+                        "description": "The block device (e.g. \"/dev/sdb\", \"/dev/disk/by-path/...\", etc.) or detect string (\"detect:NAME:SIZE:ID\") to create the OSD DB volume group on",
                         "in": "query",
                         "name": "device",
                         "required": true,

node-daemon/monitoring/munin/ceph_utilization

@@ -0,0 +1,325 @@
+#!/bin/bash
+# -*- sh -*-
+
+: << =cut
+
+=head1 NAME
+
+ceph_utilization - Plugin to monitor a Ceph cluster's utilization
+
+=head1 CONFIGURATION
+
+Defaults (no config required) for the total utilization thresholds:
+
+[ceph_utilization]
+env.warning 80
+env.critical 90
+
+=head1 AUTHOR
+
+Joshua Boniface <joshua@boniface.me>
+
+=head1 LICENSE
+
+GPLv3
+
+=head1 BUGS
+
+=back
+
+=head1 MAGIC MARKERS
+
+ #%# family=auto
+ #%# capabilities=autoconf
+
+=cut
+
+. "$MUNIN_LIBDIR/plugins/plugin.sh"
+
+is_multigraph
+
+warning=80
+critical=90
+
+RADOSDF_CMD="/usr/bin/sudo /usr/bin/rados df --format json"
+OSDDF_CMD="/usr/bin/sudo /usr/bin/ceph osd df --format json"
+JQ_CMD="/usr/bin/jq"
+
+output_usage() {
+    echo "This plugin outputs information about a Ceph cluster"
+    exit 0
+}
+
+output_autoconf() {
+    $RADOSDF_CMD &>/dev/null
+    radosdf_ret=$?
+    $OSDDF_CMD &>/dev/null
+    osddf_ret=$?
+    $JQ_CMD --version &>/dev/null
+    jq_ret=$?
+
+    if [[ ${radosdf_ret} -eq 0 && ${osddf_ret} -eq 0 && ${jq_ret} -eq 0 ]]; then
+        echo "yes"
+    elif [[ ${radosdf_ret} -ne 0 || ${osddf_ret} -ne 0 ]]; then
+        echo "no (no 'rados' or 'ceph' command found)"
+    elif [[ ${jq_ret} -ne 0 ]]; then
+        echo "no (no 'jq' command found)"
+    else
+        echo "no (general failure)"
+    fi
+}
+
+output_config() {
+    # Graph set 1 - Ceph cluster utilization
+    echo 'multigraph cluster_utilization'
+    echo 'graph_title Cluster Utilization'
+    echo 'graph_args --base 1000'
+    echo 'graph_vlabel % Utilization'
+    echo 'graph_category ceph'
+    echo 'graph_info This graph shows the cluster utilization.'
+
+    echo 'cluster_utilization.label Cluster Utilization'
+    echo 'cluster_utilization.type GAUGE'
+    echo 'cluster_utilization.max 100'
+    echo 'cluster_utilization.info Percentage utilization of the cluster.'
+    print_warning cluster_utilization
+    print_critical cluster_utilization
+
+    # Graph set 2 - Ceph cluster objects
+    echo 'multigraph cluster_objects'
+    echo 'graph_title Cluster Objects'
+    echo 'graph_args --base 1000'
+    echo 'graph_vlabel Objects'
+    echo 'graph_category ceph'
+    echo 'graph_info This graph shows the cluster object count.'
+
+    echo 'cluster_objects.label Cluster Objects'
+    echo 'cluster_objects.type GAUGE'
+    echo 'cluster_objects.min 0'
+    echo 'cluster_objects.info Total objects in the cluster.'
+
+    POOL_LIST="$( $RADOSDF_CMD | jq -r '.pools[].name' )"
+
+    # Graph set 3 - Cluster I/O Bytes Lifetime
+    echo 'multigraph pool_rdbytes'
+    echo "graph_title IO Bytes (Lifetime)"
+    echo "graph_args --base 1000"
+    echo "graph_vlabel bytes read (-) / write (+)"
+    echo "graph_category ceph"
+    echo "graph_info This graph shows the lifetime cluster bytes."
+    for pool in ${POOL_LIST}; do
+        # Graph set 3 - Cluster I/O Bytes Lifetime
+        echo "pool_rdbytes_${pool}.label Pool ${pool} IO (Bytes)"
+        echo "pool_rdbytes_${pool}.type GAUGE"
+        echo "pool_rdbytes_${pool}.min 0"
+        echo "pool_rdbytes_${pool}.draw LINE1"
+        echo "pool_rdbytes_${pool}.graph no"
+        echo "pool_wrbytes_${pool}.label Pool ${pool} IO (Bytes)"
+        echo "pool_wrbytes_${pool}.type GAUGE"
+        echo "pool_wrbytes_${pool}.min 0"
+        echo "pool_wrbytes_${pool}.draw LINE1"
+        echo "pool_wrbytes_${pool}.negative pool_rdbytes_${pool}"
+    done
+
+    # Graph set 4 - Cluster I/O Operations Lifetime
+    echo 'multigraph pool_rdops'
+    echo "graph_title IO Operations (Lifetime)"
+    echo "graph_args --base 1000"
+    echo "graph_vlabel IOs read (-) / write (+)"
+    echo "graph_category ceph"
+    echo "graph_info This graph shows the lifetime cluster IOs."
+    for pool in ${POOL_LIST}; do
+        # Graph set 4 - Cluster I/O Operations Lifetime
+        echo "pool_rdops_${pool}.label Pool ${pool} IO (Ops)"
+        echo "pool_rdops_${pool}.type GAUGE"
+        echo "pool_rdops_${pool}.min 0"
+        echo "pool_rdops_${pool}.draw LINE1"
+        echo "pool_rdops_${pool}.graph no"
+        echo "pool_wrops_${pool}.label Pool ${pool} IO (Ops)"
+        echo "pool_wrops_${pool}.type GAUGE"
+        echo "pool_wrops_${pool}.min 0"
+        echo "pool_wrops_${pool}.draw LINE1"
+        echo "pool_wrops_${pool}.negative pool_rdops_${pool}"
+    done
+
+    # Graph set 5 - Ceph pool objects
+    echo 'multigraph pool_objects_total'
+    echo "graph_title Objects"
+    echo "graph_args --base 1000"
+    echo "graph_vlabel Objects"
+    echo "graph_category ceph"
+    echo "graph_info This graph shows the cluster object count."
+    for pool in ${POOL_LIST}; do
+        # Graph set 5 - Ceph pool objects
+        echo "pool_objects_total_${pool}.label Pool ${pool} Objects"
+        echo "pool_objects_total_${pool}.type GAUGE"
+        echo "pool_objects_total_${pool}.min 0"
+        echo "pool_objects_total_${pool}.info Total objects in the pool."
+    done
+
+    # Graph set 6 - Ceph pool objects copies
+    echo 'multigraph pool_objects_copies'
+    echo "graph_title Objects Copies"
+    echo "graph_args --base 1000"
+    echo "graph_vlabel Objects"
+    echo "graph_category ceph"
+    echo "graph_info This graph shows the cluster object copy count."
+    for pool in ${POOL_LIST}; do
+        # Graph set 6 - Ceph pool objects copies
+        echo "pool_objects_copies_${pool}.label Pool ${pool} Objects Copies"
+        echo "pool_objects_copies_${pool}.type GAUGE"
+        echo "pool_objects_copies_${pool}.min 0"
+        echo "pool_objects_copies_${pool}.info Total object copies in the pool."
+    done
+
+    # Graph set 7 - Ceph pool objects degraded
+    echo 'multigraph pool_objects_degraded'
+    echo "graph_title Objects Degraded"
+    echo "graph_args --base 1000"
+    echo "graph_vlabel Objects"
+    echo "graph_category ceph"
+    echo "graph_info This graph shows the cluster object degraded count."
+    for pool in ${POOL_LIST}; do
+        # Graph set 7 - Ceph pool objects degraded
+        echo "pool_objects_degraded_${pool}.label Pool ${pool} Objects Degraded"
+        echo "pool_objects_degraded_${pool}.type GAUGE"
+        echo "pool_objects_degraded_${pool}.min 0"
+        echo "pool_objects_degraded_${pool}.info Total degraded objects in the pool."
+    done
+
+    OSD_LIST="$( $OSDDF_CMD | jq -r '.nodes[].id' | sort -n )"
+
+    # Graph set 8 - Ceph OSD status
+    echo 'multigraph osd_status'
+    echo "graph_title OSD Status"
+    echo "graph_args --base 1000"
+    echo "graph_vlabel Status Up (1) / Down (0)"
+    echo "graph_category ceph"
+    echo "graph_info This graph shows the OSD status."
+    for osd in ${OSD_LIST}; do
+        # Graph set 8 - Ceph OSD status
+        echo "osd_status_${osd}.label osd.${osd} Status"
+        echo "osd_status_${osd}.type GAUGE"
+        echo "osd_status_${osd}.min 0"
+        echo "osd_status_${osd}.max 1"
+        echo "osd_status_${osd}.info Status of the OSD."
+    done
+
+    # Graph set 9 - Ceph OSD utilization
+    echo 'multigraph osd_utilization'
+    echo "graph_title OSD Utilization"
+    echo "graph_args --base 1000"
+    echo "graph_vlabel % Utilization"
+    echo "graph_category ceph"
+    echo "graph_info This graph shows the OSD utilization."
+    for osd in ${OSD_LIST}; do
+        # Graph set 9 - Ceph OSD utilization
+        echo "osd_utilization_${osd}.label osd.${osd} Utilization"
+        echo "osd_utilization_${osd}.type GAUGE"
+        echo "osd_utilization_${osd}.max 100"
+        echo "osd_utilization_${osd}.info Utilization of the OSD."
+    done
+
+    exit 0
+}
+
+output_values() {
+    RADOS_JSON_OUTPUT="$( $RADOSDF_CMD )"
+    OSD_JSON_OUTPUT="$( $OSDDF_CMD )"
+
+    cluster_utilization="$( $JQ_CMD -r '.total_used' <<<"${RADOS_JSON_OUTPUT}" )"
+    cluster_size="$( $JQ_CMD -r '.total_space' <<<"${RADOS_JSON_OUTPUT}" )"
+    pct_utilization="$( echo "scale=4; ${cluster_utilization} / ${cluster_size} * 100" | bc -l )"
+    cluster_objects="$( $JQ_CMD -r '.total_objects' <<<"${RADOS_JSON_OUTPUT}" )"
+
+    echo "multigraph cluster_utilization"
+    echo "cluster_utilization.value ${pct_utilization}"
+    echo "multigraph cluster_objects"
+    echo "cluster_objects.value ${cluster_objects}"
+
+    cluster_pool_count="$( $JQ_CMD -r '.pools[].name' <<<"${RADOS_JSON_OUTPUT}" | wc -l )"
+    echo "multigraph pool_rdbytes"
+    for id in $( seq 0 $(( ${cluster_pool_count} - 1 )) ); do
+        pool="$( $JQ_CMD -r ".pools[$id].name" <<<"${RADOS_JSON_OUTPUT}" )"
+        pool_rdbytes="$( $JQ_CMD -r ".pools[$id].read_bytes" <<<"${RADOS_JSON_OUTPUT}" )"
+        pool_wrbytes="$( $JQ_CMD -r ".pools[$id].write_bytes" <<<"${RADOS_JSON_OUTPUT}" )"
+        echo "pool_rdbytes_${pool}.value ${pool_rdbytes}"
+        echo "pool_wrbytes_${pool}.value ${pool_wrbytes}"
+    done
+
+    echo "multigraph pool_rdops"
+    for id in $( seq 0 $(( ${cluster_pool_count} - 1 )) ); do
+        pool="$( $JQ_CMD -r ".pools[$id].name" <<<"${RADOS_JSON_OUTPUT}" )"
+        pool_rdops="$( $JQ_CMD -r ".pools[$id].read_ops" <<<"${RADOS_JSON_OUTPUT}" )"
+        pool_wrops="$( $JQ_CMD -r ".pools[$id].write_ops" <<<"${RADOS_JSON_OUTPUT}" )"
+        echo "pool_rdops_${pool}.value ${pool_rdops}"
+        echo "pool_wrops_${pool}.value ${pool_wrops}"
+    done
+
+    echo "multigraph pool_objects_total"
+    for id in $( seq 0 $(( ${cluster_pool_count} - 1 )) ); do
+        pool="$( $JQ_CMD -r ".pools[$id].name" <<<"${RADOS_JSON_OUTPUT}" )"
+        pool_objects="$( $JQ_CMD -r ".pools[$id].num_objects" <<<"${RADOS_JSON_OUTPUT}" )"
+        echo "pool_objects_total_${pool}.value ${pool_objects}"
+    done
+
+    echo "multigraph pool_objects_copies"
+    for id in $( seq 0 $(( ${cluster_pool_count} - 1 )) ); do
+        pool="$( $JQ_CMD -r ".pools[$id].name" <<<"${RADOS_JSON_OUTPUT}" )"
+        pool_copies="$( $JQ_CMD -r ".pools[$id].num_object_copies" <<<"${RADOS_JSON_OUTPUT}" )"
+        echo "pool_objects_copies_${pool}.value ${pool_copies}"
+    done
+
+    echo "multigraph pool_objects_degraded"
+    for id in $( seq 0 $(( ${cluster_pool_count} - 1 )) ); do
+        pool="$( $JQ_CMD -r ".pools[$id].name" <<<"${RADOS_JSON_OUTPUT}" )"
+        pool_degraded="$( $JQ_CMD -r ".pools[$id].num_objects_degraded" <<<"${RADOS_JSON_OUTPUT}" )"
+        echo "pool_objects_degraded_${pool}.value ${pool_degraded}"
+    done
+
+    cluster_osd_count="$( $JQ_CMD -r '.nodes[].id' <<<"${OSD_JSON_OUTPUT}" | wc -l)"
+    echo "multigraph osd_status"
+    for id in $( seq 0 $(( ${cluster_osd_count} - 1 )) ); do
+        osd="$( $JQ_CMD -r ".nodes[$id].id" <<<"${OSD_JSON_OUTPUT}" )"
+        osd_status="$( $JQ_CMD -r ".nodes[$id].status" <<<"${OSD_JSON_OUTPUT}" )"
+        case ${osd_status} in
+            up)
+                osd_status="1"
+                ;;
+            *)
+                osd_status="0"
+                ;;
+        esac
+        echo "osd_status_${osd}.value ${osd_status}"
+    done
+
+    echo "multigraph osd_utilization"
+    for id in $( seq 0 $(( ${cluster_osd_count} - 1 )) ); do
+        osd="$( $JQ_CMD -r ".nodes[$id].id" <<<"${OSD_JSON_OUTPUT}" )"
+        osd_utilization="$( $JQ_CMD -r ".nodes[$id].utilization" <<<"${OSD_JSON_OUTPUT}" )"
+        echo "osd_utilization_${osd}.value ${osd_utilization}"
+    done
+}
+
+case $# in
+    0)
+        output_values
+        ;;
+    1)
+        case $1 in
+            autoconf)
+                output_autoconf
+                ;;
+            config)
+                output_config
+                ;;
+            *)
+                output_usage
+                exit 1
+                ;;
+        esac
+        ;;
+    *)
+        output_usage
+        exit 1
+esac

node-daemon/pvcnoded.service

@@ -2,7 +2,8 @@
 
 [Unit]
 Description = Parallel Virtual Cluster node daemon
-After = network-online.target
+After = network.target
+Wants = network-online.target
 PartOf = pvc.target
 
 [Service]

node-daemon/pvcnoded/Daemon.py

@@ -48,7 +48,7 @@ import re
 import json
 
 # Daemon version
-version = "0.9.43"
+version = "0.9.46"
 
 
 ##########################################################
@@ -233,11 +233,14 @@ def entrypoint():
 
         # Force into secondary coordinator state if needed
         try:
-            if this_node.router_state == "primary":
+            if this_node.router_state == "primary" and len(d_node) > 1:
                 zkhandler.write([("base.config.primary_node", "none")])
                 logger.out("Waiting for primary migration", state="s")
-                while this_node.router_state != "secondary":
+                timeout = 240
+                count = 0
+                while this_node.router_state != "secondary" and count < timeout:
                     sleep(0.5)
+                    count += 1
         except Exception:
             pass
 

node-daemon/pvcnoded/objects/CephInstance.py

@@ -26,7 +26,76 @@ import psutil
 import daemon_lib.common as common
 
 from distutils.util import strtobool
-from re import search
+from re import search, match, sub
+
+
+def get_detect_device(detect_string):
+    """
+    Parses a "detect:" string into a normalized block device path using lsscsi.
+
+    A detect string is formatted "detect:<NAME>:<SIZE>:<ID>", where
+    NAME is some unique identifier in lsscsi, SIZE is a human-readable
+    size value to within +/- 3% of the real size of the device, and
+    ID is the Nth (0-indexed) matching entry of that NAME and SIZE.
+    """
+    _, name, size, idd = detect_string.split(":")
+    if _ != "detect":
+        return None
+
+    retcode, stdout, stderr = common.run_os_command("lsscsi -s")
+    if retcode:
+        print(f"Failed to run lsscsi: {stderr}")
+        return None
+
+    # Get valid lines
+    lsscsi_lines_raw = stdout.split("\n")
+    lsscsi_lines = list()
+    for line in lsscsi_lines_raw:
+        if not line:
+            continue
+        split_line = line.split()
+        if split_line[1] != "disk":
+            continue
+        lsscsi_lines.append(line)
+
+    # Handle size determination (+/- 3%)
+    lsscsi_sizes = set()
+    for line in lsscsi_lines:
+        lsscsi_sizes.add(split_line[-1])
+    for l_size in lsscsi_sizes:
+        b_size = float(sub(r"\D.", "", size))
+        t_size = float(sub(r"\D.", "", l_size))
+
+        plusthreepct = t_size * 1.03
+        minusthreepct = t_size * 0.97
+
+        if b_size > minusthreepct and b_size < plusthreepct:
+            size = l_size
+            break
+
+    blockdev = None
+    matches = list()
+    for idx, line in enumerate(lsscsi_lines):
+        # Skip non-disk entries
+        if line.split()[1] != "disk":
+            continue
+        # Skip if name is not contained in the line (case-insensitive)
+        if name.lower() not in line.lower():
+            continue
+        # Skip if the size does not match
+        if size != line.split()[-1]:
+            continue
+        # Get our blockdev and append to the list
+        matches.append(line.split()[-2])
+
+    blockdev = None
+    # Find the blockdev at index {idd}
+    for idx, _blockdev in enumerate(matches):
+        if int(idx) == int(idd):
+            blockdev = _blockdev
+            break
+
+    return blockdev
 
 
 class CephOSDInstance(object):
@@ -76,6 +145,22 @@ class CephOSDInstance(object):
     def add_osd(
         zkhandler, logger, node, device, weight, ext_db_flag=False, ext_db_ratio=0.05
     ):
+        # Handle a detect device if that is passed
+        if match(r"detect:", device):
+            ddevice = get_detect_device(device)
+            if ddevice is None:
+                logger.out(
+                    f"Failed to determine block device from detect string {device}",
+                    state="e",
+                )
+                return False
+            else:
+                logger.out(
+                    f"Determined block device {ddevice} from detect string {device}",
+                    state="i",
+                )
+                device = ddevice
+
         # We are ready to create a new OSD on this node
         logger.out("Creating new OSD disk on block device {}".format(device), state="i")
         try:
@@ -228,7 +313,7 @@ class CephOSDInstance(object):
     def remove_osd(zkhandler, logger, osd_id, osd_obj):
         logger.out("Removing OSD disk {}".format(osd_id), state="i")
         try:
-            # 1. Verify the OSD is present
+            # Verify the OSD is present
             retcode, stdout, stderr = common.run_os_command("ceph osd ls")
             osd_list = stdout.split("\n")
             if osd_id not in osd_list:
@@ -237,7 +322,17 @@ class CephOSDInstance(object):
                 )
                 return True
 
-            # 1. Set the OSD out so it will flush
+            # 1. Set the OSD down and out so it will flush
+            logger.out("Setting down OSD disk with ID {}".format(osd_id), state="i")
+            retcode, stdout, stderr = common.run_os_command(
+                "ceph osd down {}".format(osd_id)
+            )
+            if retcode:
+                print("ceph osd down")
+                print(stdout)
+                print(stderr)
+                raise Exception
+
             logger.out("Setting out OSD disk with ID {}".format(osd_id), state="i")
             retcode, stdout, stderr = common.run_os_command(
                 "ceph osd out {}".format(osd_id)
@@ -354,17 +449,33 @@ class CephOSDInstance(object):
 
     @staticmethod
     def add_db_vg(zkhandler, logger, device):
+        # Check if an existsing volume group exists
+        retcode, stdout, stderr = common.run_os_command("vgdisplay osd-db")
+        if retcode != 5:
+            logger.out('Ceph OSD database VG "osd-db" already exists', state="e")
+            return False
+
+        # Handle a detect device if that is passed
+        if match(r"detect:", device):
+            ddevice = get_detect_device(device)
+            if ddevice is None:
+                logger.out(
+                    f"Failed to determine block device from detect string {device}",
+                    state="e",
+                )
+                return False
+            else:
+                logger.out(
+                    f"Determined block device {ddevice} from detect string {device}",
+                    state="i",
+                )
+                device = ddevice
+
         logger.out(
             "Creating new OSD database volume group on block device {}".format(device),
             state="i",
         )
         try:
-            # 0. Check if an existsing volume group exists
-            retcode, stdout, stderr = common.run_os_command("vgdisplay osd-db")
-            if retcode != 5:
-                logger.out('Ceph OSD database VG "osd-db" already exists', state="e")
-                return False
-
             # 1. Create an empty partition table
             logger.out(
                 "Creating partitions on block device {}".format(device), state="i"