Fix bug cleaning up successful benchmark results

Supports niche usecases whereby a network template should contain the
same VNI(s) more than once.

.version

@@ -1 +1 @@
-0.9.94
+0.9.95

CHANGELOG.md

@@ -1,5 +1,11 @@
 ## PVC Changelog
 
+###### [v0.9.95](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.95)
+
+  * [API Daemon/CLI Client] Adds a flag to allow duplicate VNIs in network templates
+  * [API Daemon] Ensures that storage template disks are returned in disk ID order
+  * [Client CLI] Fixes a display bug showing all OSDs as split
+
 ###### [v0.9.94](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.94)
 
   * [CLI Client] Fixes an incorrect ordering issue with autobackup summary emails

api-daemon/pvcapid/Daemon.py

@@ -27,7 +27,7 @@ from distutils.util import strtobool as dustrtobool
 import daemon_lib.config as cfg
 
 # Daemon version
-version = "0.9.94"
+version = "0.9.95"
 
 # API version
 API_VERSION = 1.0

api-daemon/pvcapid/flaskapi.py

@@ -7139,7 +7139,11 @@ class API_Provisioner_Template_Network_Net_Root(Resource):
                 "name": "vni",
                 "required": True,
                 "helptext": "A valid VNI must be specified.",
-            }
+            },
+            {
+                "name": "permit_duplicate",
+                "required": False,
+            },
         ]
     )
     @Authenticator
@@ -7155,6 +7159,11 @@ class API_Provisioner_Template_Network_Net_Root(Resource):
             type: integer
             required: false
             description: PVC network VNI
+          - in: query
+            name: permit_duplicate
+            type: boolean
+            required: false
+            description: Bypass checks to permit duplicate VNIs for niche usecases
         responses:
           200:
             description: OK
@@ -7168,7 +7177,7 @@ class API_Provisioner_Template_Network_Net_Root(Resource):
               id: Message
         """
         return api_provisioner.create_template_network_element(
-            template, reqargs.get("vni", None)
+            template, reqargs.get("vni", None), reqargs.get("permit_duplicate", False)
         )
 
 
@@ -7206,13 +7215,27 @@ class API_Provisioner_Template_Network_Net_Element(Resource):
                 return _vni, 200
         abort(404)
 
+    @RequestParser(
+        [
+            {
+                "name": "permit_duplicate",
+                "required": False,
+            }
+        ]
+    )
     @Authenticator
-    def post(self, template, vni):
+    def post(self, template, vni, reqargs):
         """
         Create a new network {vni} in network template {template}
         ---
         tags:
           - provisioner / template
+        parameters:
+          - in: query
+            name: permit_duplicate
+            type: boolean
+            required: false
+            description: Bypass checks to permit duplicate VNIs for niche usecases
         responses:
           200:
             description: OK
@@ -7225,7 +7248,9 @@ class API_Provisioner_Template_Network_Net_Element(Resource):
               type: object
               id: Message
         """
-        return api_provisioner.create_template_network_element(template, vni)
+        return api_provisioner.create_template_network_element(
+            template, vni, reqargs.get("permit_duplicate", False)
+        )
 
     @Authenticator
     def delete(self, template, vni):

api-daemon/pvcapid/provisioner.py

@@ -125,7 +125,7 @@ def list_template(limit, table, is_fuzzy=True):
             args = (template_data["id"],)
             cur.execute(query, args)
             disks = cur.fetchall()
-            data[template_id]["disks"] = disks
+            data[template_id]["disks"] = sorted(disks, key=lambda x: x["disk_id"])
 
     close_database(conn, cur)
 
@@ -284,27 +284,28 @@ def create_template_network(name, mac_template=None):
     return retmsg, retcode
 
 
-def create_template_network_element(name, vni):
+def create_template_network_element(name, vni, permit_duplicate=False):
     if list_template_network(name, is_fuzzy=False)[-1] != 200:
         retmsg = {"message": 'The network template "{}" does not exist.'.format(name)}
         retcode = 400
         return retmsg, retcode
 
-    networks, code = list_template_network_vnis(name)
-    if code != 200:
-        networks = []
-    found_vni = False
-    for network in networks:
-        if network["vni"] == vni:
-            found_vni = True
-    if found_vni:
-        retmsg = {
-            "message": 'The VNI "{}" in network template "{}" already exists.'.format(
-                vni, name
-            )
-        }
-        retcode = 400
-        return retmsg, retcode
+    if not permit_duplicate:
+        networks, code = list_template_network_vnis(name)
+        if code != 200:
+            networks = []
+        found_vni = False
+        for network in networks:
+            if network["vni"] == vni:
+                found_vni = True
+        if found_vni:
+            retmsg = {
+                "message": 'The VNI "{}" in network template "{}" already exists.'.format(
+                    vni, name
+                )
+            }
+            retcode = 400
+            return retmsg, retcode
 
     conn, cur = open_database(config)
     try:

client-cli/pvc/cli/cli.py

@@ -4849,13 +4849,27 @@ def cli_provisioner_template_network_vni():
 @connection_req
 @click.argument("name")
 @click.argument("vni")
-def cli_provisioner_template_network_vni_add(name, vni):
+@click.option(
+    "-d",
+    "--permit-duplicate",
+    "permit_duplicate_flag",
+    is_flag=True,
+    default=False,
+    help="Permit a duplicate VNI if one already exists",
+)
+def cli_provisioner_template_network_vni_add(name, vni, permit_duplicate_flag):
     """
     Add a new network VNI to network template NAME.
 
     Networks will be added to VMs in the order they are added and displayed within the template.
+
+    NOTE: Normally, the API prevents duplicate VNIs from being added to the same network template
+    by returning an error, as this requirement is very niche. If you do not desire this behaviour,
+    use the "-d"/"--permit-duplicate" option to bypass the check.
     """
     params = dict()
+    if permit_duplicate_flag:
+        params["permit_duplicate"] = True
 
     retcode, retdata = pvc.lib.provisioner.template_element_add(
         CLI_CONFIG, name, vni, params, element_type="net", template_type="network"

client-cli/pvc/lib/storage.py

@@ -430,7 +430,9 @@ def format_list_osd(config, osd_list):
             )
             continue
 
-        if osd_information.get("is_split") is not None:
+        if osd_information.get("is_split") is not None and osd_information.get(
+            "is_split"
+        ):
             osd_information["device"] = f"{osd_information['device']} [s]"
 
         # Deal with the size to human readable

client-cli/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="pvc",
-    version="0.9.94",
+    version="0.9.95",
     packages=["pvc.cli", "pvc.lib"],
     install_requires=[
         "Click",

daemon-common/benchmark.py

@@ -115,12 +115,13 @@ class BenchmarkError(Exception):
 #
 
 
-def cleanup(job_name, db_conn=None, db_cur=None, zkhandler=None):
+def cleanup(job_name, db_conn=None, db_cur=None, zkhandler=None, final=False):
     if db_conn is not None and db_cur is not None:
-        # Clean up our dangling result
-        query = "DELETE FROM storage_benchmarks WHERE job = %s;"
-        args = (job_name,)
-        db_cur.execute(query, args)
+        if not final:
+            # Clean up our dangling result (non-final runs only)
+            query = "DELETE FROM storage_benchmarks WHERE job = %s;"
+            args = (job_name,)
+            db_cur.execute(query, args)
         db_conn.commit()
         # Close the database connections cleanly
         close_database(db_conn, db_cur)
@@ -410,6 +411,7 @@ def worker_run_benchmark(zkhandler, celery, config, pool):
         db_conn=db_conn,
         db_cur=db_cur,
         zkhandler=zkhandler,
+        final=True,
     )
 
     current_stage += 1

daemon-common/node.py

@@ -69,6 +69,8 @@ def getNodeHealthDetails(zkhandler, node_name, node_health_plugins):
             plugin_message,
             plugin_data,
         ) = tuple(all_plugin_data[pos_start:pos_end])
+        if plugin_data is None:
+            continue
         plugin_output = {
             "name": plugin,
             "last_run": int(plugin_last_run) if plugin_last_run is not None else None,
@@ -156,9 +158,9 @@ def getNodeInformation(zkhandler, node_name):
         zkhandler, node_name, node_health_plugins
     )
 
-    if _node_network_stats is not None:
+    try:
         node_network_stats = json.loads(_node_network_stats)
-    else:
+    except Exception:
         node_network_stats = dict()
 
     # Construct a data structure to represent the data

debian/changelog

@@ -1,3 +1,11 @@
+pvc (0.9.95-0) unstable; urgency=high
+
+  * [API Daemon/CLI Client] Adds a flag to allow duplicate VNIs in network templates
+  * [API Daemon] Ensures that storage template disks are returned in disk ID order
+  * [Client CLI] Fixes a display bug showing all OSDs as split
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Fri, 09 Feb 2024 12:42:00 -0500
+
 pvc (0.9.94-0) unstable; urgency=high
 
   * [CLI Client] Fixes an incorrect ordering issue with autobackup summary emails

health-daemon/pvchealthd/Daemon.py

@@ -33,7 +33,7 @@ import os
 import signal
 
 # Daemon version
-version = "0.9.94"
+version = "0.9.95"
 
 
 ##########################################################

node-daemon/pvcnoded/Daemon.py

@@ -49,7 +49,7 @@ import re
 import json
 
 # Daemon version
-version = "0.9.94"
+version = "0.9.95"
 
 
 ##########################################################

worker-daemon/pvcworkerd/Daemon.py

@@ -44,7 +44,7 @@ from daemon_lib.vmbuilder import (
 )
 
 # Daemon version
-version = "0.9.94"
+version = "0.9.95"
 
 
 config = cfg.get_configuration()