Bump version to 0.9.95

Supports niche usecases whereby a network template should contain the
same VNI(s) more than once.

.version

@@ -1 +1 @@
-0.9.93
+0.9.95

CHANGELOG.md

@@ -1,5 +1,18 @@
 ## PVC Changelog
 
+###### [v0.9.95](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.95)
+
+  * [API Daemon/CLI Client] Adds a flag to allow duplicate VNIs in network templates
+  * [API Daemon] Ensures that storage template disks are returned in disk ID order
+  * [Client CLI] Fixes a display bug showing all OSDs as split
+
+###### [v0.9.94](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.94)
+
+  * [CLI Client] Fixes an incorrect ordering issue with autobackup summary emails
+  * [API Daemon/CLI Client] Adds an additional safety check for 80% cluster fullness when doing volume adds or resizes
+  * [API Daemon/CLI Client] Adds safety checks to volume clones as well
+  * [API Daemon] Fixes a few remaining memory bugs for stopped/disabled VMs
+
 ###### [v0.9.93](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.93)
 
   * [API Daemon] Fixes a bug where stuck zkhandler threads were not cleaned up on error

api-daemon/pvcapid/Daemon.py

@@ -27,7 +27,7 @@ from distutils.util import strtobool as dustrtobool
 import daemon_lib.config as cfg
 
 # Daemon version
-version = "0.9.93"
+version = "0.9.95"
 
 # API version
 API_VERSION = 1.0

api-daemon/pvcapid/flaskapi.py

@@ -5744,6 +5744,10 @@ class API_Storage_Ceph_Volume_Root(Resource):
                 "required": True,
                 "helptext": "A volume size in bytes (B implied or with SI suffix k/M/G/T) must be specified.",
             },
+            {
+                "name": "force",
+                "required": False,
+            },
         ]
     )
     @Authenticator
@@ -5769,6 +5773,12 @@ class API_Storage_Ceph_Volume_Root(Resource):
             type: string
             required: true
             description: The volume size, in bytes (B implied) or with a single-character SI suffix (k/M/G/T)
+          - in: query
+            name: force
+            type: boolean
+            required: false
+            default: flase
+            description: Force action if volume creation would violate 80% full soft cap on the pool
         responses:
           200:
             description: OK
@@ -5785,6 +5795,7 @@ class API_Storage_Ceph_Volume_Root(Resource):
             reqargs.get("pool", None),
             reqargs.get("volume", None),
             reqargs.get("size", None),
+            reqargs.get("force", False),
         )
 
 
@@ -5819,7 +5830,11 @@ class API_Storage_Ceph_Volume_Element(Resource):
                 "name": "size",
                 "required": True,
                 "helptext": "A volume size in bytes (or with k/M/G/T suffix) must be specified.",
-            }
+            },
+            {
+                "name": "force",
+                "required": False,
+            },
         ]
     )
     @Authenticator
@@ -5835,6 +5850,12 @@ class API_Storage_Ceph_Volume_Element(Resource):
             type: string
             required: true
             description: The volume size in bytes (or with a metric suffix, i.e. k/M/G/T)
+          - in: query
+            name: force
+            type: boolean
+            required: false
+            default: flase
+            description: Force action if volume creation would violate 80% full soft cap on the pool
         responses:
           200:
             description: OK
@@ -5852,9 +5873,17 @@ class API_Storage_Ceph_Volume_Element(Resource):
               type: object
               id: Message
         """
-        return api_helper.ceph_volume_add(pool, volume, reqargs.get("size", None))
+        return api_helper.ceph_volume_add(
+            pool, volume, reqargs.get("size", None), reqargs.get("force", False)
+        )
 
-    @RequestParser([{"name": "new_size"}, {"name": "new_name"}])
+    @RequestParser(
+        [
+            {"name": "new_size"},
+            {"name": "new_name"},
+            {"name": "force", "required": False},
+        ]
+    )
     @Authenticator
     def put(self, pool, volume, reqargs):
         """
@@ -5873,6 +5902,12 @@ class API_Storage_Ceph_Volume_Element(Resource):
             type: string
             required: false
             description: The new volume name
+          - in: query
+            name: force
+            type: boolean
+            required: false
+            default: flase
+            description: Force action if new volume size would violate 80% full soft cap on the pool
         responses:
           200:
             description: OK
@@ -5894,7 +5929,9 @@ class API_Storage_Ceph_Volume_Element(Resource):
             return {"message": "Can only perform one modification at once"}, 400
 
         if reqargs.get("new_size", None):
-            return api_helper.ceph_volume_resize(pool, volume, reqargs.get("new_size"))
+            return api_helper.ceph_volume_resize(
+                pool, volume, reqargs.get("new_size"), reqargs.get("force", False)
+            )
         if reqargs.get("new_name", None):
             return api_helper.ceph_volume_rename(pool, volume, reqargs.get("new_name"))
         return {"message": "At least one modification must be specified"}, 400
@@ -5935,7 +5972,11 @@ class API_Storage_Ceph_Volume_Element_Clone(Resource):
                 "name": "new_volume",
                 "required": True,
                 "helptext": "A new volume name must be specified.",
-            }
+            },
+            {
+                "name": "force",
+                "required": False,
+            },
         ]
     )
     @Authenticator
@@ -5951,6 +5992,12 @@ class API_Storage_Ceph_Volume_Element_Clone(Resource):
             type: string
             required: true
             description: The name of the new cloned volume
+          - in: query
+            name: force
+            type: boolean
+            required: false
+            default: flase
+            description: Force action if clone volume size would violate 80% full soft cap on the pool
         responses:
           200:
             description: OK
@@ -5969,7 +6016,7 @@ class API_Storage_Ceph_Volume_Element_Clone(Resource):
               id: Message
         """
         return api_helper.ceph_volume_clone(
-            pool, reqargs.get("new_volume", None), volume
+            pool, reqargs.get("new_volume", None), volume, reqargs.get("force", None)
         )
 
 
@@ -7092,7 +7139,11 @@ class API_Provisioner_Template_Network_Net_Root(Resource):
                 "name": "vni",
                 "required": True,
                 "helptext": "A valid VNI must be specified.",
-            }
+            },
+            {
+                "name": "permit_duplicate",
+                "required": False,
+            },
         ]
     )
     @Authenticator
@@ -7108,6 +7159,11 @@ class API_Provisioner_Template_Network_Net_Root(Resource):
             type: integer
             required: false
             description: PVC network VNI
+          - in: query
+            name: permit_duplicate
+            type: boolean
+            required: false
+            description: Bypass checks to permit duplicate VNIs for niche usecases
         responses:
           200:
             description: OK
@@ -7121,7 +7177,7 @@ class API_Provisioner_Template_Network_Net_Root(Resource):
               id: Message
         """
         return api_provisioner.create_template_network_element(
-            template, reqargs.get("vni", None)
+            template, reqargs.get("vni", None), reqargs.get("permit_duplicate", False)
         )
 
 
@@ -7159,13 +7215,27 @@ class API_Provisioner_Template_Network_Net_Element(Resource):
                 return _vni, 200
         abort(404)
 
+    @RequestParser(
+        [
+            {
+                "name": "permit_duplicate",
+                "required": False,
+            }
+        ]
+    )
     @Authenticator
-    def post(self, template, vni):
+    def post(self, template, vni, reqargs):
         """
         Create a new network {vni} in network template {template}
         ---
         tags:
           - provisioner / template
+        parameters:
+          - in: query
+            name: permit_duplicate
+            type: boolean
+            required: false
+            description: Bypass checks to permit duplicate VNIs for niche usecases
         responses:
           200:
             description: OK
@@ -7178,7 +7248,9 @@ class API_Provisioner_Template_Network_Net_Element(Resource):
               type: object
               id: Message
         """
-        return api_provisioner.create_template_network_element(template, vni)
+        return api_provisioner.create_template_network_element(
+            template, vni, reqargs.get("permit_duplicate", False)
+        )
 
     @Authenticator
     def delete(self, template, vni):

api-daemon/pvcapid/helper.py

@@ -1869,11 +1869,13 @@ def ceph_volume_list(zkhandler, pool=None, limit=None, is_fuzzy=True):
 
 
 @ZKConnection(config)
-def ceph_volume_add(zkhandler, pool, name, size):
+def ceph_volume_add(zkhandler, pool, name, size, force_flag):
     """
     Add a Ceph RBD volume to the PVC Ceph storage cluster.
     """
-    retflag, retdata = pvc_ceph.add_volume(zkhandler, pool, name, size)
+    retflag, retdata = pvc_ceph.add_volume(
+        zkhandler, pool, name, size, force_flag=force_flag
+    )
 
     if retflag:
         retcode = 200
@@ -1885,11 +1887,13 @@ def ceph_volume_add(zkhandler, pool, name, size):
 
 
 @ZKConnection(config)
-def ceph_volume_clone(zkhandler, pool, name, source_volume):
+def ceph_volume_clone(zkhandler, pool, name, source_volume, force_flag):
     """
     Clone a Ceph RBD volume to a new volume on the PVC Ceph storage cluster.
     """
-    retflag, retdata = pvc_ceph.clone_volume(zkhandler, pool, source_volume, name)
+    retflag, retdata = pvc_ceph.clone_volume(
+        zkhandler, pool, source_volume, name, force_flag=force_flag
+    )
 
     if retflag:
         retcode = 200
@@ -1901,11 +1905,13 @@ def ceph_volume_clone(zkhandler, pool, name, source_volume):
 
 
 @ZKConnection(config)
-def ceph_volume_resize(zkhandler, pool, name, size):
+def ceph_volume_resize(zkhandler, pool, name, size, force_flag):
     """
     Resize an existing Ceph RBD volume in the PVC Ceph storage cluster.
     """
-    retflag, retdata = pvc_ceph.resize_volume(zkhandler, pool, name, size)
+    retflag, retdata = pvc_ceph.resize_volume(
+        zkhandler, pool, name, size, force_flag=force_flag
+    )
 
     if retflag:
         retcode = 200

api-daemon/pvcapid/provisioner.py

@@ -125,7 +125,7 @@ def list_template(limit, table, is_fuzzy=True):
             args = (template_data["id"],)
             cur.execute(query, args)
             disks = cur.fetchall()
-            data[template_id]["disks"] = disks
+            data[template_id]["disks"] = sorted(disks, key=lambda x: x["disk_id"])
 
     close_database(conn, cur)
 
@@ -284,27 +284,28 @@ def create_template_network(name, mac_template=None):
     return retmsg, retcode
 
 
-def create_template_network_element(name, vni):
+def create_template_network_element(name, vni, permit_duplicate=False):
     if list_template_network(name, is_fuzzy=False)[-1] != 200:
         retmsg = {"message": 'The network template "{}" does not exist.'.format(name)}
         retcode = 400
         return retmsg, retcode
 
-    networks, code = list_template_network_vnis(name)
-    if code != 200:
-        networks = []
-    found_vni = False
-    for network in networks:
-        if network["vni"] == vni:
-            found_vni = True
-    if found_vni:
-        retmsg = {
-            "message": 'The VNI "{}" in network template "{}" already exists.'.format(
-                vni, name
-            )
-        }
-        retcode = 400
-        return retmsg, retcode
+    if not permit_duplicate:
+        networks, code = list_template_network_vnis(name)
+        if code != 200:
+            networks = []
+        found_vni = False
+        for network in networks:
+            if network["vni"] == vni:
+                found_vni = True
+        if found_vni:
+            retmsg = {
+                "message": 'The VNI "{}" in network template "{}" already exists.'.format(
+                    vni, name
+                )
+            }
+            retcode = 400
+            return retmsg, retcode
 
     conn, cur = open_database(config)
     try:

client-cli/pvc/cli/cli.py

@@ -4100,12 +4100,26 @@ def cli_storage_volume():
 @click.argument("pool")
 @click.argument("name")
 @click.argument("size")
-def cli_storage_volume_add(pool, name, size):
+@click.option(
+    "-f",
+    "--force",
+    "force_flag",
+    is_flag=True,
+    default=False,
+    help="Force creation even if volume would violate 80% full safe free space.",
+)
+def cli_storage_volume_add(pool, name, size, force_flag):
     """
     Add a new Ceph RBD volume in pool POOL with name NAME and size SIZE (in human units, e.g. 1024M, 20G, etc.).
+
+    PVC will prevent the creation of a volume who's size is greater than the available free space on the pool. This cannot be overridden.
+
+    PVC will prevent the creation of a volume who's size is greater than the 80% full safe free space on the pool. This can be overridden with the "-f"/"--force" option but this may be dangerous!
     """
 
-    retcode, retmsg = pvc.lib.storage.ceph_volume_add(CLI_CONFIG, pool, name, size)
+    retcode, retmsg = pvc.lib.storage.ceph_volume_add(
+        CLI_CONFIG, pool, name, size, force_flag=force_flag
+    )
     finish(retcode, retmsg)
 
 
@@ -4171,14 +4185,26 @@ def cli_storage_volume_remove(pool, name):
 @click.argument("pool")
 @click.argument("name")
 @click.argument("size")
+@click.option(
+    "-f",
+    "--force",
+    "force_flag",
+    is_flag=True,
+    default=False,
+    help="Force resize even if volume would violate 80% full safe free space.",
+)
 @confirm_opt("Resize volume {name} in pool {pool} to size {size}")
-def cli_storage_volume_resize(pool, name, size):
+def cli_storage_volume_resize(pool, name, size, force_flag):
     """
     Resize an existing Ceph RBD volume with name NAME in pool POOL to size SIZE (in human units, e.g. 1024M, 20G, etc.).
+
+    PVC will prevent the resize of a volume who's new size is greater than the available free space on the pool. This cannot be overridden.
+
+    PVC will prevent the resize of a volume who's new size is greater than the 80% full safe free space on the pool. This can be overridden with the "-f"/"--force" option but this may be dangerous!
     """
 
     retcode, retmsg = pvc.lib.storage.ceph_volume_modify(
-        CLI_CONFIG, pool, name, new_size=size
+        CLI_CONFIG, pool, name, new_size=size, force_flag=force_flag
     )
     finish(retcode, retmsg)
 
@@ -4211,13 +4237,25 @@ def cli_storage_volume_rename(pool, name, new_name):
 @click.argument("pool")
 @click.argument("name")
 @click.argument("new_name")
-def cli_storage_volume_clone(pool, name, new_name):
+@click.option(
+    "-f",
+    "--force",
+    "force_flag",
+    is_flag=True,
+    default=False,
+    help="Force clone even if volume would violate 80% full safe free space.",
+)
+def cli_storage_volume_clone(pool, name, new_name, force_flag):
     """
     Clone a Ceph RBD volume with name NAME in pool POOL to name NEW_NAME in pool POOL.
+
+    PVC will prevent the clone of a volume who's new size is greater than the available free space on the pool. This cannot be overridden.
+
+    PVC will prevent the clone of a volume who's new size is greater than the 80% full safe free space on the pool. This can be overridden with the "-f"/"--force" option but this may be dangerous!
     """
 
     retcode, retmsg = pvc.lib.storage.ceph_volume_clone(
-        CLI_CONFIG, pool, name, new_name
+        CLI_CONFIG, pool, name, new_name, force_flag=force_flag
     )
     finish(retcode, retmsg)
 
@@ -4811,13 +4849,27 @@ def cli_provisioner_template_network_vni():
 @connection_req
 @click.argument("name")
 @click.argument("vni")
-def cli_provisioner_template_network_vni_add(name, vni):
+@click.option(
+    "-d",
+    "--permit-duplicate",
+    "permit_duplicate_flag",
+    is_flag=True,
+    default=False,
+    help="Permit a duplicate VNI if one already exists",
+)
+def cli_provisioner_template_network_vni_add(name, vni, permit_duplicate_flag):
     """
     Add a new network VNI to network template NAME.
 
     Networks will be added to VMs in the order they are added and displayed within the template.
+
+    NOTE: Normally, the API prevents duplicate VNIs from being added to the same network template
+    by returning an error, as this requirement is very niche. If you do not desire this behaviour,
+    use the "-d"/"--permit-duplicate" option to bypass the check.
     """
     params = dict()
+    if permit_duplicate_flag:
+        params["permit_duplicate"] = True
 
     retcode, retdata = pvc.lib.provisioner.template_element_add(
         CLI_CONFIG, name, vni, params, element_type="net", template_type="network"

client-cli/pvc/cli/helpers.py

@@ -246,6 +246,8 @@ def vm_autobackup(
     Perform automatic backups of VMs based on an external config file.
     """
 
+    backup_summary = dict()
+
     if email_report is not None:
         from email.utils import formatdate
         from socket import gethostname
@@ -553,6 +555,8 @@ def vm_autobackup(
         with open(autobackup_state_file, "w") as fh:
             jdump(state_data, fh)
 
+        backup_summary[vm] = tracked_backups
+
     if autobackup_config["auto_mount_enabled"]:
         # Execute each unmount_cmds command in sequence
         for cmd in autobackup_config["unmount_cmds"]:
@@ -588,20 +592,6 @@ def vm_autobackup(
     if email_report is not None:
         echo(CLI_CONFIG, "")
         echo(CLI_CONFIG, f"Sending email summary report to {email_report}")
-        backup_summary = dict()
-        for vm in backup_vms:
-            backup_path = f"{backup_suffixed_path}/{vm}"
-            autobackup_state_file = f"{backup_path}/.autobackup.json"
-            if not path.exists(backup_path) or not path.exists(autobackup_state_file):
-                # There are no new backups so the list is empty
-                state_data = dict()
-                tracked_backups = list()
-            else:
-                with open(autobackup_state_file) as fh:
-                    state_data = jload(fh)
-                tracked_backups = state_data["tracked_backups"]
-
-            backup_summary[vm] = tracked_backups
 
         current_datetime = datetime.now()
         email_datetime = formatdate(float(current_datetime.strftime("%s")))

client-cli/pvc/lib/storage.py

@@ -430,7 +430,9 @@ def format_list_osd(config, osd_list):
             )
             continue
 
-        if osd_information.get("is_split") is not None:
+        if osd_information.get("is_split") is not None and osd_information.get(
+            "is_split"
+        ):
             osd_information["device"] = f"{osd_information['device']} [s]"
 
         # Deal with the size to human readable
@@ -1172,15 +1174,15 @@ def ceph_volume_list(config, limit, pool):
         return False, response.json().get("message", "")
 
 
-def ceph_volume_add(config, pool, volume, size):
+def ceph_volume_add(config, pool, volume, size, force_flag=False):
     """
     Add new Ceph volume
 
     API endpoint: POST /api/v1/storage/ceph/volume
-    API arguments: volume={volume}, pool={pool}, size={size}
+    API arguments: volume={volume}, pool={pool}, size={size}, force={force_flag}
     API schema: {"message":"{data}"}
     """
-    params = {"volume": volume, "pool": pool, "size": size}
+    params = {"volume": volume, "pool": pool, "size": size, "force": force_flag}
     response = call_api(config, "post", "/storage/ceph/volume", params=params)
 
     if response.status_code == 200:
@@ -1261,12 +1263,14 @@ def ceph_volume_remove(config, pool, volume):
     return retstatus, response.json().get("message", "")
 
 
-def ceph_volume_modify(config, pool, volume, new_name=None, new_size=None):
+def ceph_volume_modify(
+    config, pool, volume, new_name=None, new_size=None, force_flag=False
+):
     """
     Modify Ceph volume
 
     API endpoint: PUT /api/v1/storage/ceph/volume/{pool}/{volume}
-    API arguments:
+    API arguments: [new_name={new_name}], [new_size={new_size}], force_flag={force_flag}
     API schema: {"message":"{data}"}
     """
 
@@ -1275,6 +1279,7 @@ def ceph_volume_modify(config, pool, volume, new_name=None, new_size=None):
         params["new_name"] = new_name
     if new_size:
         params["new_size"] = new_size
+        params["force"] = force_flag
 
     response = call_api(
         config,
@@ -1291,15 +1296,15 @@ def ceph_volume_modify(config, pool, volume, new_name=None, new_size=None):
     return retstatus, response.json().get("message", "")
 
 
-def ceph_volume_clone(config, pool, volume, new_volume):
+def ceph_volume_clone(config, pool, volume, new_volume, force_flag=False):
     """
     Clone Ceph volume
 
     API endpoint: POST /api/v1/storage/ceph/volume/{pool}/{volume}
-    API arguments: new_volume={new_volume
+    API arguments: new_volume={new_volume, force_flag={force_flag}
     API schema: {"message":"{data}"}
     """
-    params = {"new_volume": new_volume}
+    params = {"new_volume": new_volume, "force_flag": force_flag}
     response = call_api(
         config,
         "post",

client-cli/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="pvc",
-    version="0.9.93",
+    version="0.9.95",
     packages=["pvc.cli", "pvc.lib"],
     install_requires=[
         "Click",

daemon-common/ceph.py

@@ -553,7 +553,7 @@ def getVolumeInformation(zkhandler, pool, volume):
     return volume_information
 
 
-def add_volume(zkhandler, pool, name, size):
+def add_volume(zkhandler, pool, name, size, force_flag=False):
     # 1. Verify the size of the volume
     pool_information = getPoolInformation(zkhandler, pool)
     size_bytes = format_bytes_fromhuman(size)
@@ -563,12 +563,27 @@ def add_volume(zkhandler, pool, name, size):
             f"ERROR: Requested volume size '{size}' does not have a valid SI unit",
         )
 
-    if size_bytes >= int(pool_information["stats"]["free_bytes"]):
+    pool_total_free_bytes = int(pool_information["stats"]["free_bytes"])
+    if size_bytes >= pool_total_free_bytes:
         return (
             False,
             f"ERROR: Requested volume size '{format_bytes_tohuman(size_bytes)}' is greater than the available free space in the pool ('{format_bytes_tohuman(pool_information['stats']['free_bytes'])}')",
         )
 
+    # Check if we're greater than 80% utilization after the create; error if so unless we have the force flag
+    pool_total_bytes = (
+        int(pool_information["stats"]["used_bytes"]) + pool_total_free_bytes
+    )
+    pool_safe_total_bytes = int(pool_total_bytes * 0.80)
+    pool_safe_free_bytes = pool_safe_total_bytes - int(
+        pool_information["stats"]["used_bytes"]
+    )
+    if size_bytes >= pool_safe_free_bytes and not force_flag:
+        return (
+            False,
+            f"ERROR: Requested volume size '{format_bytes_tohuman(size_bytes)}' is greater than the safe free space in the pool ('{format_bytes_tohuman(pool_safe_free_bytes)}' for 80% full); retry with force to ignore this error",
+        )
+
     # 2. Create the volume
     retcode, stdout, stderr = common.run_os_command(
         "rbd create --size {}B {}/{}".format(size_bytes, pool, name)
@@ -596,13 +611,39 @@ def add_volume(zkhandler, pool, name, size):
     )
 
 
-def clone_volume(zkhandler, pool, name_src, name_new):
+def clone_volume(zkhandler, pool, name_src, name_new, force_flag=False):
+    # 1. Verify the volume
     if not verifyVolume(zkhandler, pool, name_src):
         return False, 'ERROR: No volume with name "{}" is present in pool "{}".'.format(
             name_src, pool
         )
 
-    # 1. Clone the volume
+    volume_stats_raw = zkhandler.read(("volume.stats", f"{pool}/{name_src}"))
+    volume_stats = dict(json.loads(volume_stats_raw))
+    size_bytes = volume_stats["size"]
+    pool_information = getPoolInformation(zkhandler, pool)
+    pool_total_free_bytes = int(pool_information["stats"]["free_bytes"])
+    if size_bytes >= pool_total_free_bytes:
+        return (
+            False,
+            f"ERROR: Clone volume size '{format_bytes_tohuman(size_bytes)}' is greater than the available free space in the pool ('{format_bytes_tohuman(pool_information['stats']['free_bytes'])}')",
+        )
+
+    # Check if we're greater than 80% utilization after the create; error if so unless we have the force flag
+    pool_total_bytes = (
+        int(pool_information["stats"]["used_bytes"]) + pool_total_free_bytes
+    )
+    pool_safe_total_bytes = int(pool_total_bytes * 0.80)
+    pool_safe_free_bytes = pool_safe_total_bytes - int(
+        pool_information["stats"]["used_bytes"]
+    )
+    if size_bytes >= pool_safe_free_bytes and not force_flag:
+        return (
+            False,
+            f"ERROR: Clone volume size '{format_bytes_tohuman(size_bytes)}' is greater than the safe free space in the pool ('{format_bytes_tohuman(pool_safe_free_bytes)}' for 80% full); retry with force to ignore this error",
+        )
+
+    # 2. Clone the volume
     retcode, stdout, stderr = common.run_os_command(
         "rbd copy {}/{} {}/{}".format(pool, name_src, pool, name_new)
     )
@@ -614,13 +655,13 @@ def clone_volume(zkhandler, pool, name_src, name_new):
             ),
         )
 
-    # 2. Get volume stats
+    # 3. Get volume stats
     retcode, stdout, stderr = common.run_os_command(
         "rbd info --format json {}/{}".format(pool, name_new)
     )
     volstats = stdout
 
-    # 3. Add the new volume to Zookeeper
+    # 4. Add the new volume to Zookeeper
     zkhandler.write(
         [
             (("volume", f"{pool}/{name_new}"), ""),
@@ -634,7 +675,7 @@ def clone_volume(zkhandler, pool, name_src, name_new):
     )
 
 
-def resize_volume(zkhandler, pool, name, size):
+def resize_volume(zkhandler, pool, name, size, force_flag=False):
     if not verifyVolume(zkhandler, pool, name):
         return False, 'ERROR: No volume with name "{}" is present in pool "{}".'.format(
             name, pool
@@ -649,12 +690,27 @@ def resize_volume(zkhandler, pool, name, size):
             f"ERROR: Requested volume size '{size}' does not have a valid SI unit",
         )
 
-    if size_bytes >= int(pool_information["stats"]["free_bytes"]):
+    pool_total_free_bytes = int(pool_information["stats"]["free_bytes"])
+    if size_bytes >= pool_total_free_bytes:
         return (
             False,
             f"ERROR: Requested volume size '{format_bytes_tohuman(size_bytes)}' is greater than the available free space in the pool ('{format_bytes_tohuman(pool_information['stats']['free_bytes'])}')",
         )
 
+    # Check if we're greater than 80% utilization after the create; error if so unless we have the force flag
+    pool_total_bytes = (
+        int(pool_information["stats"]["used_bytes"]) + pool_total_free_bytes
+    )
+    pool_safe_total_bytes = int(pool_total_bytes * 0.80)
+    pool_safe_free_bytes = pool_safe_total_bytes - int(
+        pool_information["stats"]["used_bytes"]
+    )
+    if size_bytes >= pool_safe_free_bytes and not force_flag:
+        return (
+            False,
+            f"ERROR: Requested volume size '{format_bytes_tohuman(size_bytes)}' is greater than the safe free space in the pool ('{format_bytes_tohuman(pool_safe_free_bytes)}' for 80% full); retry with force to ignore this error",
+        )
+
     # 2. Resize the volume
     retcode, stdout, stderr = common.run_os_command(
         "rbd resize --size {} {}/{}".format(

daemon-common/cluster.py

@@ -1230,7 +1230,7 @@ def get_resource_metrics(zkhandler):
     )
     output_lines.append("# TYPE pvc_vm_memory_stats_actual gauge")
     for vm in vm_data:
-        actual_memory = vm["memory_stats"]["actual"]
+        actual_memory = vm["memory_stats"].get("actual", 0)
         output_lines.append(
             f"pvc_vm_memory_stats_actual{{vm=\"{vm['name']}\"}} {actual_memory}"
         )
@@ -1238,7 +1238,7 @@ def get_resource_metrics(zkhandler):
     output_lines.append("# HELP pvc_vm_memory_stats_rss PVC VM RSS memory KB")
     output_lines.append("# TYPE pvc_vm_memory_stats_rss gauge")
     for vm in vm_data:
-        rss_memory = vm["memory_stats"]["rss"]
+        rss_memory = vm["memory_stats"].get("rss", 0)
         output_lines.append(
             f"pvc_vm_memory_stats_rss{{vm=\"{vm['name']}\"}} {rss_memory}"
         )

debian/changelog

@@ -1,3 +1,20 @@
+pvc (0.9.95-0) unstable; urgency=high
+
+  * [API Daemon/CLI Client] Adds a flag to allow duplicate VNIs in network templates
+  * [API Daemon] Ensures that storage template disks are returned in disk ID order
+  * [Client CLI] Fixes a display bug showing all OSDs as split
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Fri, 09 Feb 2024 12:42:00 -0500
+
+pvc (0.9.94-0) unstable; urgency=high
+
+  * [CLI Client] Fixes an incorrect ordering issue with autobackup summary emails
+  * [API Daemon/CLI Client] Adds an additional safety check for 80% cluster fullness when doing volume adds or resizes
+  * [API Daemon/CLI Client] Adds safety checks to volume clones as well
+  * [API Daemon] Fixes a few remaining memory bugs for stopped/disabled VMs
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Mon, 05 Feb 2024 09:58:07 -0500
+
 pvc (0.9.93-0) unstable; urgency=high
 
   * [API Daemon] Fixes a bug where stuck zkhandler threads were not cleaned up on error

health-daemon/pvchealthd/Daemon.py

@@ -33,7 +33,7 @@ import os
 import signal
 
 # Daemon version
-version = "0.9.93"
+version = "0.9.95"
 
 
 ##########################################################

node-daemon/pvcnoded/Daemon.py

@@ -49,7 +49,7 @@ import re
 import json
 
 # Daemon version
-version = "0.9.93"
+version = "0.9.95"
 
 
 ##########################################################

worker-daemon/pvcworkerd/Daemon.py

@@ -44,7 +44,7 @@ from daemon_lib.vmbuilder import (
 )
 
 # Daemon version
-version = "0.9.93"
+version = "0.9.95"
 
 
 config = cfg.get_configuration()