Bump version to 0.9.89

.version

@@ -1 +1 @@
-0.9.87
+0.9.89

CHANGELOG.md

@@ -1,5 +1,18 @@
 ## PVC Changelog
 
+###### [v0.9.89](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.89)
+
+  * [API/Worker Daemons] Fixes a bug with the Celery result backends not being properly initialized on Debian 10/11.
+  * [API Daemon] Fixes a bug if VM CPU stats are missing on Debian 10.
+
+###### [v0.9.88](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.88)
+
+  * [API Daemon] Adds an additional Prometheus metrics proxy for Zookeeper stats.
+  * [API Daemon] Adds a new configuration to enable or disable metric endpoints if desired, defaulting to enabled.
+  * [API Daemon] Alters and adjusts the metrics output for VMs to complement new dashboard.
+  * [CLI Client] Adds a "json-prometheus" output format to "pvc connection list" to auto-generate file SD configs.
+  * [Monitoring] Adds a new VM dashboard, updates the Cluster dashboard, and adds a README.
+
 ###### [v0.9.87](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.87)
 
   * [API Daemon] Adds cluster Prometheus resource utilization metrics and an updated Grafana dashboard.

api-daemon/pvcapid/Daemon.py

@@ -27,7 +27,7 @@ from distutils.util import strtobool as dustrtobool
 import daemon_lib.config as cfg
 
 # Daemon version
-version = "0.9.87"
+version = "0.9.89"
 
 # API version
 API_VERSION = 1.0

api-daemon/pvcapid/flaskapi.py

@@ -118,7 +118,7 @@ celery_task_uri = "redis://{}:{}{}".format(
 celery = Celery(
     app.name,
     broker=celery_task_uri,
-    result_backend=celery_task_uri,
+    backend=celery_task_uri,
     result_extended=True,
 )
 app.config["broker_url"] = celery_task_uri

build-and-deploy.sh

@@ -68,8 +68,8 @@ for HOST in ${HOSTS[@]}; do
     ssh $HOST $SUDO systemctl restart pvcworkerd &>/dev/null
     sleep 2
     ssh $HOST $SUDO systemctl restart pvchealthd &>/dev/null
-    sleep 2
-    ssh $HOST $SUDO systemctl restart pvcnoded &>/dev/null
+#    sleep 2
+#    ssh $HOST $SUDO systemctl restart pvcnoded &>/dev/null
     echo " done."
     echo -n "Waiting for node daemon to be running..."
     while [[ $( ssh $HOST "pvc -q node list -f json ${HOST%%.*} | jq -r '.[].daemon_state'" 2>/dev/null ) != "run" ]]; do

client-cli/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="pvc",
-    version="0.9.87",
+    version="0.9.89",
     packages=["pvc.cli", "pvc.lib"],
     install_requires=[
         "Click",

daemon-common/cluster.py

@@ -1016,6 +1016,103 @@ def get_resource_metrics(zkhandler):
     #
     # VM stats
     #
+    output_lines.append("# HELP pvc_vm_uuid PVC VM UUID")
+    output_lines.append("# TYPE pvc_vm_uuid gauge")
+    for vm in vm_data:
+        uuid = vm["uuid"]
+        output_lines.append(f"pvc_vm_uuid{{vm=\"{vm['name']}\", uuid=\"{uuid}\"}} 1")
+
+    output_lines.append("# HELP pvc_vm_description PVC VM description")
+    output_lines.append("# TYPE pvc_vm_description gauge")
+    for vm in vm_data:
+        description = vm["description"]
+        output_lines.append(
+            f"pvc_vm_description{{vm=\"{vm['name']}\", description=\"{description}\"}} 1"
+        )
+
+    output_lines.append("# HELP pvc_vm_profile PVC VM profile")
+    output_lines.append("# TYPE pvc_vm_profile gauge")
+    for vm in vm_data:
+        profile = vm["profile"]
+        output_lines.append(
+            f"pvc_vm_profile{{vm=\"{vm['name']}\", profile=\"{profile}\"}} 1"
+        )
+
+    output_lines.append("# HELP pvc_vm_state PVC VM state")
+    output_lines.append("# TYPE pvc_vm_state gauge")
+    for vm in vm_data:
+        state_colour_map = {
+            "start": 0,
+            "migrate": 1,
+            "unmigrate": 2,
+            "provision": 3,
+            "disable": 4,
+            "shutdown": 5,
+            "restart": 6,
+            "stop": 7,
+            "fail": 8,
+        }
+        state = vm["state"]
+        output_lines.append(
+            f"pvc_vm_state{{vm=\"{vm['name']}\", state=\"{state}\"}} {state_colour_map[vm['state']]}"
+        )
+
+    output_lines.append("# HELP pvc_vm_failed_reason PVC VM failed_reason")
+    output_lines.append("# TYPE pvc_vm_failed_reason gauge")
+    for vm in vm_data:
+        failed_reason = vm["failed_reason"] if vm["failed_reason"] else "N/A"
+        output_lines.append(
+            f"pvc_vm_failed_reason{{vm=\"{vm['name']}\", failed_reason=\"{failed_reason}\"}} 1"
+        )
+
+    output_lines.append("# HELP pvc_vm_node_limit PVC VM node_limit")
+    output_lines.append("# TYPE pvc_vm_node_limit gauge")
+    for vm in vm_data:
+        node_limit = vm["node_limit"]
+        output_lines.append(
+            f"pvc_vm_node_limit{{vm=\"{vm['name']}\", node_limit=\"{node_limit}\"}} 1"
+        )
+
+    output_lines.append("# HELP pvc_vm_node_selector PVC VM node_selector")
+    output_lines.append("# TYPE pvc_vm_node_selector gauge")
+    for vm in vm_data:
+        node_selector = (
+            "Default"
+            if vm["node_selector"] is None or vm["node_selector"] == "None"
+            else vm["node_selector"]
+        )
+        output_lines.append(
+            f"pvc_vm_node_selector{{vm=\"{vm['name']}\", node_selector=\"{node_selector}\"}} 1"
+        )
+
+    output_lines.append("# HELP pvc_vm_node_autostart PVC VM node_autostart")
+    output_lines.append("# TYPE pvc_vm_node_autostart gauge")
+    for vm in vm_data:
+        autostart = vm["node_autostart"]
+        autostart_val = 1 if vm["node_autostart"] else 0
+        output_lines.append(
+            f"pvc_vm_autostart{{vm=\"{vm['name']}\", autostart=\"{autostart}\"}} {autostart_val}"
+        )
+
+    output_lines.append("# HELP pvc_vm_migration_method PVC VM migration_method")
+    output_lines.append("# TYPE pvc_vm_migration_method gauge")
+    for vm in vm_data:
+        migration_method = (
+            "Default"
+            if vm["migration_method"] is None or vm["migration_method"] == "None"
+            else vm["migration_method"]
+        )
+        output_lines.append(
+            f"pvc_vm_migration_method{{vm=\"{vm['name']}\", migration_method=\"{migration_method}\"}} 1"
+        )
+
+    output_lines.append("# HELP pvc_vm_tags PVC VM tags")
+    output_lines.append("# TYPE pvc_vm_tags gauge")
+    for vm in vm_data:
+        tags = [f"tag=\"{t['name']}\"" for t in vm["tags"]]
+        for tag in tags:
+            output_lines.append(f"pvc_vm_tags{{vm=\"{vm['name']}\", {tag}}} 1")
+
     output_lines.append("# HELP pvc_vm_active_node PVC VM active node")
     output_lines.append("# TYPE pvc_vm_active_node gauge")
     for vm in vm_data:
@@ -1024,12 +1121,21 @@ def get_resource_metrics(zkhandler):
             f"pvc_vm_active_node{{vm=\"{vm['name']}\", node=\"{active_node}\"}} 1"
         )
 
+    output_lines.append("# HELP pvc_vm_migrated PVC VM migrated state")
+    output_lines.append("# TYPE pvc_vm_migrated gauge")
+    for vm in vm_data:
+        migrated = 0 if vm["migrated"] == "no" else 1
+        last_node = vm["last_node"] if vm["last_node"] else "No"
+        output_lines.append(
+            f"pvc_vm_migrated{{vm=\"{vm['name']}\", last_node=\"{last_node}\"}} {migrated}"
+        )
+
     output_lines.append("# HELP pvc_vm_machine_type PVC VM machine type")
     output_lines.append("# TYPE pvc_vm_machine_type gauge")
     for vm in vm_data:
         machine_type = vm["machine"]
         output_lines.append(
-            f"pvc_vm_machine_type{{vm=\"{vm['name']}\", node=\"{machine_type}\"}} 1"
+            f"pvc_vm_machine_type{{vm=\"{vm['name']}\", machine_type=\"{machine_type}\"}} 1"
         )
 
     output_lines.append("# HELP pvc_vm_serial_console PVC VM serial console")
@@ -1068,24 +1174,47 @@ def get_resource_metrics(zkhandler):
         vcpus = vm["vcpu"]
         output_lines.append(f"pvc_vm_vcpus{{vm=\"{vm['name']}\"}} {vcpus}")
 
-    output_lines.append("# HELP pvc_vm_vcpus_cpu_time PVC VM vCPU CPU time")
+    output_lines.append("# HELP pvc_vm_vcpu_topology PVC VM vCPU topology")
+    output_lines.append("# TYPE pvc_vm_vcpu_topology gauge")
+    for vm in vm_data:
+        vcpu_topology = vm["vcpu_topology"]
+        output_lines.append(
+            f"pvc_vm_vcpu_topology{{vm=\"{vm['name']}\", topology=\"{vcpu_topology}\"}} 1"
+        )
+
+    output_lines.append(
+        "# HELP pvc_vm_vcpus_cpu_time PVC VM vCPU CPU time milliseconds"
+    )
     output_lines.append("# TYPE pvc_vm_vcpus_cpu_time gauge")
     for vm in vm_data:
-        cpu_time = vm["vcpu_stats"]["cpu_time"]
+        try:
+            cpu_time = vm["vcpu_stats"]["cpu_time"] / 1000000
+        except Exception:
+            cpu_time = 0
         output_lines.append(f"pvc_vm_vcpus_cpu_time{{vm=\"{vm['name']}\"}} {cpu_time}")
 
-    output_lines.append("# HELP pvc_vm_vcpus_user_time PVC VM vCPU User time")
+    output_lines.append(
+        "# HELP pvc_vm_vcpus_user_time PVC VM vCPU User time milliseconds"
+    )
     output_lines.append("# TYPE pvc_vm_vcpus_user_time gauge")
     for vm in vm_data:
-        user_time = vm["vcpu_stats"]["user_time"]
+        try:
+            user_time = vm["vcpu_stats"]["user_time"] / 1000000
+        except Exception:
+            cpu_time = 0
         output_lines.append(
             f"pvc_vm_vcpus_user_time{{vm=\"{vm['name']}\"}} {user_time}"
         )
 
-    output_lines.append("# HELP pvc_vm_vcpus_system_time PVC VM vCPU System time")
+    output_lines.append(
+        "# HELP pvc_vm_vcpus_system_time PVC VM vCPU System time milliseconds"
+    )
     output_lines.append("# TYPE pvc_vm_vcpus_system_time gauge")
     for vm in vm_data:
-        system_time = vm["vcpu_stats"]["system_time"]
+        try:
+            system_time = vm["vcpu_stats"]["system_time"] / 1000000
+        except Exception:
+            system_time = 0
         output_lines.append(
             f"pvc_vm_vcpus_system_time{{vm=\"{vm['name']}\"}} {system_time}"
         )
@@ -1097,7 +1226,7 @@ def get_resource_metrics(zkhandler):
         output_lines.append(f"pvc_vm_memory{{vm=\"{vm['name']}\"}} {memory}")
 
     output_lines.append(
-        "# HELP pvc_vm_memory_stats_actual PVC VM actual memory allocation"
+        "# HELP pvc_vm_memory_stats_actual PVC VM actual memory allocation KB"
     )
     output_lines.append("# TYPE pvc_vm_memory_stats_actual gauge")
     for vm in vm_data:
@@ -1106,7 +1235,7 @@ def get_resource_metrics(zkhandler):
             f"pvc_vm_memory_stats_actual{{vm=\"{vm['name']}\"}} {actual_memory}"
         )
 
-    output_lines.append("# HELP pvc_vm_memory_stats_rss PVC VM RSS memory")
+    output_lines.append("# HELP pvc_vm_memory_stats_rss PVC VM RSS memory KB")
     output_lines.append("# TYPE pvc_vm_memory_stats_rss gauge")
     for vm in vm_data:
         rss_memory = vm["memory_stats"]["rss"]
@@ -1114,7 +1243,7 @@ def get_resource_metrics(zkhandler):
             f"pvc_vm_memory_stats_rss{{vm=\"{vm['name']}\"}} {rss_memory}"
         )
 
-    output_lines.append("# HELP pvc_vm_memory_stats_unused PVC VM unused memory")
+    output_lines.append("# HELP pvc_vm_memory_stats_unused PVC VM unused memory KB")
     output_lines.append("# TYPE pvc_vm_memory_stats_unused gauge")
     for vm in vm_data:
         unused_memory = vm["memory_stats"].get("unused", 0)
@@ -1122,7 +1251,9 @@ def get_resource_metrics(zkhandler):
             f"pvc_vm_memory_stats_unused{{vm=\"{vm['name']}\"}} {unused_memory}"
         )
 
-    output_lines.append("# HELP pvc_vm_memory_stats_available PVC VM available memory")
+    output_lines.append(
+        "# HELP pvc_vm_memory_stats_available PVC VM available memory KB"
+    )
     output_lines.append("# TYPE pvc_vm_memory_stats_available gauge")
     for vm in vm_data:
         available_memory = vm["memory_stats"].get("available", 0)
@@ -1130,7 +1261,7 @@ def get_resource_metrics(zkhandler):
             f"pvc_vm_memory_stats_available{{vm=\"{vm['name']}\"}} {available_memory}"
         )
 
-    output_lines.append("# HELP pvc_vm_memory_stats_usable PVC VM usable memory")
+    output_lines.append("# HELP pvc_vm_memory_stats_usable PVC VM usable memory KB")
     output_lines.append("# TYPE pvc_vm_memory_stats_usable gauge")
     for vm in vm_data:
         usable_memory = vm["memory_stats"].get("usable", 0)
@@ -1139,7 +1270,7 @@ def get_resource_metrics(zkhandler):
         )
 
     output_lines.append(
-        "# HELP pvc_vm_memory_stats_disk_caches PVC VM disk cache memory"
+        "# HELP pvc_vm_memory_stats_disk_caches PVC VM disk cache memory KB"
     )
     output_lines.append("# TYPE pvc_vm_memory_stats_disk_caches gauge")
     for vm in vm_data:
@@ -1233,14 +1364,14 @@ def get_resource_metrics(zkhandler):
                 f"pvc_vm_network_rd_packets{{vm=\"{vm['name']}\",vni=\"{vni}\"}} {rd_packets}"
             )
 
-    output_lines.append("# HELP pvc_vm_network_rd_bytes PVC VM network bytes read")
-    output_lines.append("# TYPE pvc_vm_network_rd_bytes gauge")
+    output_lines.append("# HELP pvc_vm_network_rd_bits PVC VM network bits read")
+    output_lines.append("# TYPE pvc_vm_network_rd_bits gauge")
     for vm in vm_data:
         for network in vm["networks"]:
             vni = network["vni"]
-            rd_bytes = network["rd_bytes"]
+            rd_bits = network["rd_bytes"] * 8
             output_lines.append(
-                f"pvc_vm_network_rd_bytes{{vm=\"{vm['name']}\",vni=\"{vni}\"}} {rd_bytes}"
+                f"pvc_vm_network_rd_bits{{vm=\"{vm['name']}\",vni=\"{vni}\"}} {rd_bits}"
             )
 
     output_lines.append("# HELP pvc_vm_network_rd_errors PVC VM network read errors")
@@ -1273,14 +1404,14 @@ def get_resource_metrics(zkhandler):
                 f"pvc_vm_network_wr_packets{{vm=\"{vm['name']}\",vni=\"{vni}\"}} {wr_packets}"
             )
 
-    output_lines.append("# HELP pvc_vm_network_wr_bytes PVC VM network bytes write")
-    output_lines.append("# TYPE pvc_vm_network_wr_bytes gauge")
+    output_lines.append("# HELP pvc_vm_network_wr_bits PVC VM network bits write")
+    output_lines.append("# TYPE pvc_vm_network_wr_bits gauge")
     for vm in vm_data:
         for network in vm["networks"]:
             vni = network["vni"]
-            wr_bytes = network["wr_bytes"]
+            wr_bits = network["wr_bytes"] * 8
             output_lines.append(
-                f"pvc_vm_network_wr_bytes{{vm=\"{vm['name']}\",vni=\"{vni}\"}} {wr_bytes}"
+                f"pvc_vm_network_wr_bits{{vm=\"{vm['name']}\",vni=\"{vni}\"}} {wr_bits}"
             )
 
     output_lines.append("# HELP pvc_vm_network_wr_errors PVC VM network write errors")

debian/changelog

@@ -1,3 +1,20 @@
+pvc (0.9.89-0) unstable; urgency=high
+
+  * [API/Worker Daemons] Fixes a bug with the Celery result backends not being properly initialized on Debian 10/11.
+  * [API Daemon] Fixes a bug if VM CPU stats are missing on Debian 10.
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Tue, 09 Jan 2024 12:15:53 -0500
+
+pvc (0.9.88-0) unstable; urgency=high
+
+  * [API Daemon] Adds an additional Prometheus metrics proxy for Zookeeper stats.
+  * [API Daemon] Adds a new configuration to enable or disable metric endpoints if desired, defaulting to enabled.
+  * [API Daemon] Alters and adjusts the metrics output for VMs to complement new dashboard.
+  * [CLI Client] Adds a "json-prometheus" output format to "pvc connection list" to auto-generate file SD configs.
+  * [Monitoring] Adds a new VM dashboard, updates the Cluster dashboard, and adds a README.
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Fri, 29 Dec 2023 14:50:40 -0500
+
 pvc (0.9.87-0) unstable; urgency=high
 
   * [API Daemon] Adds cluster Prometheus resource utilization metrics and an updated Grafana dashboard.

health-daemon/pvchealthd/Daemon.py

@@ -33,7 +33,7 @@ import os
 import signal
 
 # Daemon version
-version = "0.9.87"
+version = "0.9.89"
 
 
 ##########################################################

monitoring/prometheus/README.md

@@ -37,4 +37,8 @@ This file can be autogenerated from the list of configured PVC clusters in a PVC
 
 ## `grafana-pvc-cluster-dashboard.json`
 
-This JSON-based Grafana dashboard allows for a nice presentation of the metrics collected by the above Prometheus pollers. The cluster can be selected (based on the `pvc_cluster_name` value) and useful information about the cluster is then displayed.
+This JSON-based Grafana dashboard allows for a nice presentation of the Cluster and Node metrics collected by the above Prometheus pollers. The cluster can be selected (based on the `pvc_cluster_name` value) and useful information about the cluster is then displayed.
+
+## `grafana-pvc-vms-dashboard.json`
+
+This JSON-based Grafana dashboard allows for a nice presentation of the VM metrics collected by the above Prometheus pollers. The cluster can be selected (based on the `pvc_cluster_name` value) and useful information about a given VM is then displayed.

monitoring/prometheus/grafana-pvc-cluster-dashboard.json

@@ -1590,8 +1590,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -1893,8 +1892,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -2154,8 +2152,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   }
                 ]
               },
@@ -2544,8 +2541,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -3976,7 +3972,7 @@
             "h": 13,
             "w": 4,
             "x": 0,
-            "y": 16
+            "y": 31
           },
           "id": 9,
           "options": {
@@ -4087,7 +4083,7 @@
             "h": 13,
             "w": 9,
             "x": 4,
-            "y": 16
+            "y": 31
           },
           "id": 44,
           "options": {
@@ -4211,7 +4207,7 @@
             "h": 13,
             "w": 11,
             "x": 13,
-            "y": 16
+            "y": 31
           },
           "id": 50,
           "options": {
@@ -4309,7 +4305,7 @@
             "h": 13,
             "w": 4,
             "x": 0,
-            "y": 29
+            "y": 44
           },
           "id": 45,
           "options": {
@@ -4420,7 +4416,7 @@
             "h": 13,
             "w": 9,
             "x": 4,
-            "y": 29
+            "y": 44
           },
           "id": 46,
           "options": {
@@ -4559,7 +4555,7 @@
             "h": 13,
             "w": 11,
             "x": 13,
-            "y": 29
+            "y": 44
           },
           "id": 51,
           "options": {
@@ -4656,7 +4652,7 @@
             "h": 13,
             "w": 4,
             "x": 0,
-            "y": 42
+            "y": 57
           },
           "id": 48,
           "options": {
@@ -4767,7 +4763,7 @@
             "h": 13,
             "w": 9,
             "x": 4,
-            "y": 42
+            "y": 57
           },
           "id": 49,
           "options": {
@@ -4891,7 +4887,7 @@
             "h": 13,
             "w": 11,
             "x": 13,
-            "y": 42
+            "y": 57
           },
           "id": 52,
           "options": {
@@ -4989,7 +4985,7 @@
             "h": 13,
             "w": 4,
             "x": 0,
-            "y": 55
+            "y": 70
           },
           "id": 89,
           "options": {
@@ -5100,7 +5096,7 @@
             "h": 13,
             "w": 9,
             "x": 4,
-            "y": 55
+            "y": 70
           },
           "id": 90,
           "options": {
@@ -5224,7 +5220,7 @@
             "h": 13,
             "w": 11,
             "x": 13,
-            "y": 55
+            "y": 70
           },
           "id": 91,
           "options": {
@@ -5322,7 +5318,7 @@
             "h": 13,
             "w": 4,
             "x": 0,
-            "y": 68
+            "y": 83
           },
           "id": 92,
           "options": {
@@ -5433,7 +5429,7 @@
             "h": 13,
             "w": 9,
             "x": 4,
-            "y": 68
+            "y": 83
           },
           "id": 93,
           "options": {
@@ -5557,7 +5553,7 @@
             "h": 13,
             "w": 11,
             "x": 13,
-            "y": 68
+            "y": 83
           },
           "id": 94,
           "options": {
@@ -5670,7 +5666,7 @@
             "h": 14,
             "w": 8,
             "x": 0,
-            "y": 17
+            "y": 32
           },
           "id": 60,
           "options": {
@@ -5783,7 +5779,7 @@
             "h": 7,
             "w": 16,
             "x": 8,
-            "y": 17
+            "y": 32
           },
           "id": 59,
           "options": {
@@ -5890,7 +5886,7 @@
             "h": 7,
             "w": 16,
             "x": 8,
-            "y": 24
+            "y": 39
           },
           "id": 98,
           "options": {
@@ -6085,7 +6081,7 @@
             "h": 10,
             "w": 24,
             "x": 0,
-            "y": 31
+            "y": 46
           },
           "id": 62,
           "options": {
@@ -6268,7 +6264,7 @@
             "h": 16,
             "w": 9,
             "x": 0,
-            "y": 18
+            "y": 98
           },
           "id": 58,
           "options": {
@@ -6379,7 +6375,7 @@
             "h": 8,
             "w": 15,
             "x": 9,
-            "y": 18
+            "y": 98
           },
           "id": 61,
           "options": {
@@ -6485,7 +6481,7 @@
             "h": 8,
             "w": 15,
             "x": 9,
-            "y": 26
+            "y": 106
           },
           "id": 99,
           "options": {
@@ -6595,7 +6591,7 @@
                   }
                 ]
               },
-              "unit": "none"
+              "unit": "iops"
             },
             "overrides": [
               {
@@ -6614,9 +6610,9 @@
           },
           "gridPos": {
             "h": 12,
-            "w": 12,
+            "w": 24,
             "x": 0,
-            "y": 35
+            "y": 99
           },
           "id": 103,
           "options": {
@@ -6786,9 +6782,9 @@
           },
           "gridPos": {
             "h": 12,
-            "w": 12,
-            "x": 12,
-            "y": 35
+            "w": 24,
+            "x": 0,
+            "y": 111
           },
           "id": 57,
           "options": {
@@ -6945,10 +6941,10 @@
             "overrides": []
           },
           "gridPos": {
-            "h": 12,
-            "w": 12,
+            "h": 9,
+            "w": 24,
             "x": 0,
-            "y": 47
+            "y": 123
           },
           "id": 100,
           "links": [],
@@ -7077,10 +7073,10 @@
             "overrides": []
           },
           "gridPos": {
-            "h": 12,
-            "w": 12,
-            "x": 12,
-            "y": 47
+            "h": 9,
+            "w": 24,
+            "x": 0,
+            "y": 132
           },
           "id": 102,
           "links": [],
@@ -7231,7 +7227,7 @@
             "h": 4,
             "w": 4,
             "x": 0,
-            "y": 20
+            "y": 100
           },
           "id": 108,
           "options": {
@@ -7325,7 +7321,7 @@
             "h": 4,
             "w": 5,
             "x": 4,
-            "y": 20
+            "y": 100
           },
           "id": 113,
           "options": {
@@ -7400,7 +7396,7 @@
             "h": 4,
             "w": 3,
             "x": 9,
-            "y": 20
+            "y": 100
           },
           "id": 114,
           "options": {
@@ -7505,7 +7501,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 20
+            "y": 100
           },
           "id": 111,
           "options": {
@@ -7596,7 +7592,7 @@
             "h": 4,
             "w": 6,
             "x": 0,
-            "y": 24
+            "y": 104
           },
           "id": 115,
           "options": {
@@ -7672,7 +7668,7 @@
             "h": 4,
             "w": 6,
             "x": 6,
-            "y": 24
+            "y": 104
           },
           "id": 116,
           "options": {
@@ -7777,7 +7773,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 28
+            "y": 108
           },
           "id": 105,
           "options": {
@@ -7880,7 +7876,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 28
+            "y": 108
           },
           "id": 112,
           "options": {
@@ -7983,7 +7979,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 36
+            "y": 116
           },
           "id": 106,
           "options": {
@@ -8200,7 +8196,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 36
+            "y": 116
           },
           "id": 117,
           "options": {

node-daemon/pvcnoded/Daemon.py

@@ -49,7 +49,7 @@ import re
 import json
 
 # Daemon version
-version = "0.9.87"
+version = "0.9.89"
 
 
 ##########################################################

worker-daemon/pvcworkerd/Daemon.py

@@ -44,7 +44,7 @@ from daemon_lib.vmbuilder import (
 )
 
 # Daemon version
-version = "0.9.87"
+version = "0.9.89"
 
 
 config = cfg.get_configuration()
@@ -58,7 +58,7 @@ celery_task_uri = "redis://{}:{}{}".format(
 celery = Celery(
     "pvcworkerd",
     broker=celery_task_uri,
-    result_backend=celery_task_uri,
+    backend=celery_task_uri,
     result_extended=True,
 )