Bump version to 0.9.103

Since storage_hosts now includes the storage domain as FQDNs, don't
re-append it within vmbuilder.

.version

@@ -1 +1 @@
-0.9.101
+0.9.103

CHANGELOG.md

@@ -1,5 +1,15 @@
 ## PVC Changelog
 
+###### [v0.9.103](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.103)
+
+  * [Provisioner] Fixes a bug with the change in `storage_hosts` to FQDNs affecting the VM Builder
+  * [Monitoring] Fixes the Munin plugin to work properly with sudo
+
+###### [v0.9.102](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.102)
+
+  * [API Daemon] Ensures that received config snapshots update storage hosts in addition to secret UUIDs
+  * [CLI Client] Fixes several bugs around local connection handling and connection listings
+
 ###### [v0.9.101](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.101)
 
   **New Feature**: Adds VM snapshot sending (`vm snapshot send`), VM mirroring (`vm mirror create`), and (offline) mirror promotion (`vm mirror promote`). Permits transferring VM snapshots to remote clusters, individually or repeatedly, and promoting them to active status, for disaster recovery and migration between clusters.

README.md

@@ -1,10 +1,11 @@
 <p align="center">
-<img alt="Logo banner" src="images/pvc_logo_black.png"/>
+<img alt="Logo banner" src="https://docs.parallelvirtualcluster.org/en/latest/images/pvc_logo_black.png"/>
 <br/><br/>
+<a href="https://www.parallelvirtualcluster.org"><img alt="Website" src="https://img.shields.io/badge/visit-website-blue"/></a>
+<a href="https://github.com/parallelvirtualcluster/pvc/releases"><img alt="Latest Release" src="https://img.shields.io/github/release-pre/parallelvirtualcluster/pvc"/></a>
+<a href="https://docs.parallelvirtualcluster.org/en/latest/?badge=latest"><img alt="Documentation Status" src="https://readthedocs.org/projects/parallelvirtualcluster/badge/?version=latest"/></a>
 <a href="https://github.com/parallelvirtualcluster/pvc"><img alt="License" src="https://img.shields.io/github/license/parallelvirtualcluster/pvc"/></a>
 <a href="https://github.com/psf/black"><img alt="Code style: Black" src="https://img.shields.io/badge/code%20style-black-000000.svg"/></a>
-<a href="https://github.com/parallelvirtualcluster/pvc/releases"><img alt="Release" src="https://img.shields.io/github/release-pre/parallelvirtualcluster/pvc"/></a>
-<a href="https://docs.parallelvirtualcluster.org/en/latest/?badge=latest"><img alt="Documentation Status" src="https://readthedocs.org/projects/parallelvirtualcluster/badge/?version=latest"/></a>
 </p>
 
 ## What is PVC?
@@ -23,62 +24,64 @@ Installation of PVC is accomplished by two main components: a [Node installer IS
 
 Just give it physical servers, and it will run your VMs without you having to think about it, all in just an hour or two of setup time.
 
+More information about PVC, its motivations, the hardware requirements, and setting up and managing a cluster [can be found over at our docs page](https://docs.parallelvirtualcluster.org).
+
 ## Getting Started
 
 To get started with PVC, please see the [About](https://docs.parallelvirtualcluster.org/en/latest/about-pvc/) page for general information about the project, and the [Getting Started](https://docs.parallelvirtualcluster.org/en/latest/deployment/getting-started/) page for details on configuring your first cluster.
 
 ## Changelog
 
-View the changelog in [CHANGELOG.md](CHANGELOG.md). **Please note that any breaking changes are announced here; ensure you read the changelog before upgrading!**
+View the changelog in [CHANGELOG.md](https://github.com/parallelvirtualcluster/pvc/blob/master/CHANGELOG.md). **Please note that any breaking changes are announced here; ensure you read the changelog before upgrading!**
 
 ## Screenshots
 
 These screenshots show some of the available functionality of the PVC system and CLI as of PVC v0.9.85.
 
-<p><img alt="0. Integrated help" src="images/0-integrated-help.png"/><br/>
+<p><img alt="0. Integrated help" src="https://raw.githubusercontent.com/parallelvirtualcluster/pvc/refs/heads/master/images/0-integrated-help.png"/><br/>
 <i>The CLI features an integrated, fully-featured help system to show details about every possible command.</i>
 </p>
 
-<p><img alt="1. Connection management" src="images/1-connection-management.png"/><br/>
+<p><img alt="1. Connection management" src="https://raw.githubusercontent.com/parallelvirtualcluster/pvc/refs/heads/master/images/1-connection-management.png"/><br/>
 <i>A single CLI instance can manage multiple clusters, including a quick detail view, and will default to a "local" connection if an "/etc/pvc/pvc.conf" file is found; sensitive API keys are hidden by default.</i>
 </p>
 
-<p><img alt="2. Cluster details and output formats" src="images/2-cluster-details-and-output-formats.png"/><br/>
+<p><img alt="2. Cluster details and output formats" src="https://raw.githubusercontent.com/parallelvirtualcluster/pvc/refs/heads/master/images/2-cluster-details-and-output-formats.png"/><br/>
 <i>PVC can show the key details of your cluster at a glance, including health, persistent fault events, and key resources; the CLI can output both in pretty human format and JSON for easier machine parsing in scripts.</i>
 </p>
 
-<p><img alt="3. Node information" src="images/3-node-information.png"/><br/>
+<p><img alt="3. Node information" src="https://raw.githubusercontent.com/parallelvirtualcluster/pvc/refs/heads/master/images/3-node-information.png"/><br/>
 <i>PVC can show details about the nodes in the cluster, including their live health and resource utilization.</i>
 </p>
 
-<p><img alt="4. VM information" src="images/4-vm-information.png"/><br/>
+<p><img alt="4. VM information" src="https://raw.githubusercontent.com/parallelvirtualcluster/pvc/refs/heads/master/images/4-vm-information.png"/><br/>
 <i>PVC can show details about the VMs in the cluster, including their state, resource allocations, current hosting node, and metadata.</i>
 </p>
 
-<p><img alt="5. VM details" src="images/5-vm-details.png"/><br/>
+<p><img alt="5. VM details" src="https://raw.githubusercontent.com/parallelvirtualcluster/pvc/refs/heads/master/images/5-vm-details.png"/><br/>
 <i>In addition to the above basic details, PVC can also show extensive information about a running VM's devices and other resource utilization.</i>
 </p>
 
-<p><img alt="6. Network information" src="images/6-network-information.png"/><br/>
+<p><img alt="6. Network information" src="https://raw.githubusercontent.com/parallelvirtualcluster/pvc/refs/heads/master/images/6-network-information.png"/><br/>
 <i>PVC has two major client network types, and ensures a consistent configuration of client networks across the entire cluster; managed networks can feature DHCP, DNS, firewall, and other functionality including DHCP reservations.</i>
 </p>
 
-<p><img alt="7. Storage information" src="images/7-storage-information.png"/><br/>
+<p><img alt="7. Storage information" src="https://raw.githubusercontent.com/parallelvirtualcluster/pvc/refs/heads/master/images/7-storage-information.png"/><br/>
 <i>PVC provides a convenient abstracted view of the underlying Ceph system and can manage all core aspects of it.</i>
 </p>
 
-<p><img alt="8. VM and node logs" src="images/8-vm-and-node-logs.png"/><br/>
+<p><img alt="8. VM and node logs" src="https://raw.githubusercontent.com/parallelvirtualcluster/pvc/refs/heads/master/images/8-vm-and-node-logs.png"/><br/>
 <i>PVC can display logs from VM serial consoles (if properly configured) and nodes in-client to facilitate quick troubleshooting.</i>
 </p>
 
-<p><img alt="9. VM and worker tasks" src="images/9-vm-and-worker-tasks.png"/><br/>
+<p><img alt="9. VM and worker tasks" src="https://raw.githubusercontent.com/parallelvirtualcluster/pvc/refs/heads/master/images/9-vm-and-worker-tasks.png"/><br/>
 <i>PVC provides full VM lifecycle management, as well as long-running worker-based commands (in this example, clearing a VM's storage locks).</i>
 </p>
 
-<p><img alt="10. Provisioner" src="images/10-provisioner.png"/><br/>
+<p><img alt="10. Provisioner" src="https://raw.githubusercontent.com/parallelvirtualcluster/pvc/refs/heads/master/images/10-provisioner.png"/><br/>
 <i>PVC features an extensively customizable and configurable VM provisioner system, including EC2-compatible CloudInit support, allowing you to define flexible VM profiles and provision new VMs with a single command.</i>
 </p>
 
-<p><img alt="11. Prometheus and Grafana dashboard" src="images/11-prometheus-grafana.png"/><br/>
+<p><img alt="11. Prometheus and Grafana dashboard" src="https://raw.githubusercontent.com/parallelvirtualcluster/pvc/refs/heads/master/images/11-prometheus-grafana.png"/><br/>
 <i>PVC features several monitoring integration examples under "node-daemon/monitoring", including CheckMK, Munin, and, most recently, Prometheus, including an example Grafana dashboard for cluster monitoring and alerting.</i>
 </p>

api-daemon/pvcapid/helper.py

@@ -1438,15 +1438,7 @@ def vm_snapshot_receive_block_createsnap(zkhandler, pool, volume, snapshot):
 @ZKConnection(config)
 def vm_snapshot_receive_config(zkhandler, snapshot, vm_config, source_snapshot=None):
     """
-    Receive a VM configuration from a remote system
+    Receive a VM configuration snapshot from a remote system, and modify it to work on our system
-
-    This function requires some explanation.
-
-    We get a full JSON dump of the VM configuration as provided by `pvc vm info`. This contains all the information we
-    reasonably need to replicate the VM at the given snapshot, including metainformation.
-
-    First, we need to determine if this is an incremental or full send. If it's full, and the VM already exists,
-    this is an issue and we have to error. But this should have already happened with the RBD volumes.
     """
 
     def parse_unified_diff(diff_text, original_text):
@@ -1503,13 +1495,28 @@ def vm_snapshot_receive_config(zkhandler, snapshot, vm_config, source_snapshot=N
     vm_xml = vm_config["xml"]
     vm_xml_diff = "\n".join(current_snapshot["xml_diff_lines"])
     snapshot_vm_xml = parse_unified_diff(vm_xml_diff, vm_xml)
+    xml_data = etree.fromstring(snapshot_vm_xml)
 
     # Replace the Ceph storage secret UUID with this cluster's
     our_ceph_secret_uuid = config["ceph_secret_uuid"]
-    xml_data = etree.fromstring(snapshot_vm_xml)
     ceph_secrets = xml_data.xpath("//secret[@type='ceph']")
     for ceph_secret in ceph_secrets:
         ceph_secret.set("uuid", our_ceph_secret_uuid)
+
+    # Replace the Ceph source hosts with this cluster's
+    our_ceph_storage_hosts = config["storage_hosts"]
+    our_ceph_storage_port = str(config["ceph_monitor_port"])
+    ceph_sources = xml_data.xpath("//source[@protocol='rbd']")
+    for ceph_source in ceph_sources:
+        for host in ceph_source.xpath("host"):
+            ceph_source.remove(host)
+        for ceph_storage_host in our_ceph_storage_hosts:
+            new_host = etree.Element("host")
+            new_host.set("name", ceph_storage_host)
+            new_host.set("port", our_ceph_storage_port)
+            ceph_source.append(new_host)
+
+    # Regenerate the VM XML
     snapshot_vm_xml = etree.tostring(xml_data, pretty_print=True).decode("utf8")
 
     if (

client-cli/pvc/cli/formatters.py

@@ -905,7 +905,7 @@ def cli_connection_list_format_pretty(CLI_CONFIG, data):
     # Parse each connection and adjust field lengths
     for connection in data:
         for field, length in [(f, fields[f]["length"]) for f in fields]:
-            _length = len(str(connection[field]))
+            _length = len(str(connection[field])) + 1
             if _length > length:
                 length = len(str(connection[field])) + 1
 
@@ -1005,7 +1005,7 @@ def cli_connection_detail_format_pretty(CLI_CONFIG, data):
     # Parse each connection and adjust field lengths
     for connection in data:
         for field, length in [(f, fields[f]["length"]) for f in fields]:
-            _length = len(str(connection[field]))
+            _length = len(str(connection[field])) + 1
             if _length > length:
                 length = len(str(connection[field])) + 1
 

client-cli/pvc/cli/helpers.py

@@ -167,9 +167,17 @@ def get_store(store_path):
     with open(store_file) as fh:
         try:
             store_data = jload(fh)
-            return store_data
         except Exception:
-            return dict()
+            store_data = dict()
+
+    if path.exists(DEFAULT_STORE_DATA["cfgfile"]):
+        if store_data.get("local", None) != DEFAULT_STORE_DATA:
+            del store_data["local"]
+        if "local" not in store_data.keys():
+            store_data["local"] = DEFAULT_STORE_DATA
+            update_store(store_path, store_data)
+
+    return store_data
 
 
 def update_store(store_path, store_data):

client-cli/pvc/cli/parsers.py

@@ -68,7 +68,8 @@ def cli_connection_list_parser(connections_config, show_keys_flag):
                 }
             )
 
-    return connections_data
+    # Return, ensuring local is always first
+    return sorted(connections_data, key=lambda x: (x.get("name") != "local"))
 
 
 def cli_connection_detail_parser(connections_config):
@@ -121,4 +122,5 @@ def cli_connection_detail_parser(connections_config):
                 }
             )
 
-    return connections_data
+    # Return, ensuring local is always first
+    return sorted(connections_data, key=lambda x: (x.get("name") != "local"))

client-cli/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="pvc",
-    version="0.9.101",
+    version="0.9.103",
     packages=["pvc.cli", "pvc.lib"],
     install_requires=[
         "Click",

daemon-common/config.py

@@ -375,8 +375,11 @@ def get_parsed_configuration(config_file):
         config = {**config, **config_api_ssl}
 
         # Use coordinators as storage hosts if not explicitly specified
+        # These are added as FQDNs in the storage domain
         if not config["storage_hosts"] or len(config["storage_hosts"]) < 1:
-            config["storage_hosts"] = config["coordinators"]
+            config["storage_hosts"] = []
+            for host in config["coordinators"]:
+                config["storage_hosts"].append(f"{host}.{config['storage_domain']}")
 
         # Set up our token list if specified
         if config["api_auth_source"] == "token":

daemon-common/vmbuilder.py

@@ -336,11 +336,7 @@ def worker_create_vm(
     retcode, stdout, stderr = pvc_common.run_os_command("uname -m")
     vm_data["system_architecture"] = stdout.strip()
 
-    monitor_list = list()
+    vm_data["ceph_monitor_list"] = config["storage_hosts"]
-    monitor_names = config["storage_hosts"]
-    for monitor in monitor_names:
-        monitor_list.append("{}.{}".format(monitor, config["storage_domain"]))
-    vm_data["ceph_monitor_list"] = monitor_list
     vm_data["ceph_monitor_port"] = config["ceph_monitor_port"]
     vm_data["ceph_monitor_secret"] = config["ceph_secret_uuid"]
 

debian/changelog

@@ -1,3 +1,17 @@
+pvc (0.9.103-0) unstable; urgency=high
+
+  * [Provisioner] Fixes a bug with the change in `storage_hosts` to FQDNs affecting the VM Builder
+  * [Monitoring] Fixes the Munin plugin to work properly with sudo
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Fri, 01 Nov 2024 17:19:44 -0400
+
+pvc (0.9.102-0) unstable; urgency=high
+
+  * [API Daemon] Ensures that received config snapshots update storage hosts in addition to secret UUIDs
+  * [CLI Client] Fixes several bugs around local connection handling and connection listings
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Thu, 17 Oct 2024 10:48:31 -0400
+
 pvc (0.9.101-0) unstable; urgency=high
 
   **New Feature**: Adds VM snapshot sending (`vm snapshot send`), VM mirroring (`vm mirror create`), and (offline) mirror promotion (`vm mirror promote`). Permits transferring VM snapshots to remote clusters, individually or repeatedly, and promoting them to active status, for disaster recovery and migration between clusters.

health-daemon/pvchealthd/Daemon.py

@@ -33,7 +33,7 @@ import os
 import signal
 
 # Daemon version
-version = "0.9.101"
+version = "0.9.103"
 
 
 ##########################################################

monitoring/munin/pvc

@@ -34,7 +34,7 @@ warning=0.99
 critical=1.99
 
 export PVC_CLIENT_DIR="/run/shm/munin-pvc"
-PVC_CMD="/usr/bin/pvc --quiet --cluster local status --format json-pretty"
+PVC_CMD="/usr/bin/sudo -E /usr/bin/pvc --quiet cluster status --format json-pretty"
 JQ_CMD="/usr/bin/jq"
 
 output_usage() {
@@ -126,7 +126,7 @@ output_values() {
     is_maintenance="$( $JQ_CMD ".maintenance" <<<"${PVC_OUTPUT}" | tr -d '"' )"
 
     cluster_health="$( $JQ_CMD ".cluster_health.health" <<<"${PVC_OUTPUT}" | tr -d '"' )"
-    cluster_health_messages="$( $JQ_CMD -r ".cluster_health.messages | @csv" <<<"${PVC_OUTPUT}" | tr -d '"' | sed 's/,/, /g' )"
+    cluster_health_messages="$( $JQ_CMD -r ".cluster_health.messages | map(.text) | join(\", \")" <<<"${PVC_OUTPUT}" )"
     echo 'multigraph pvc_cluster_health'
     echo "pvc_cluster_health.value ${cluster_health}"
     echo "pvc_cluster_health.extinfo ${cluster_health_messages}"
@@ -142,7 +142,7 @@ output_values() {
     echo "pvc_cluster_alert.value ${cluster_health_alert}"
 
     node_health="$( $JQ_CMD ".node_health.${HOST}.health" <<<"${PVC_OUTPUT}" | tr -d '"' )"
-    node_health_messages="$( $JQ_CMD -r ".node_health.${HOST}.messages | @csv" <<<"${PVC_OUTPUT}" | tr -d '"' | sed 's/,/, /g' )"
+    node_health_messages="$( $JQ_CMD -r ".node_health.${HOST}.messages | join(\", \")" <<<"${PVC_OUTPUT}" )"
     echo 'multigraph pvc_node_health'
     echo "pvc_node_health.value ${node_health}"
     echo "pvc_node_health.extinfo ${node_health_messages}"

node-daemon/pvcnoded/Daemon.py

@@ -49,7 +49,7 @@ import re
 import json
 
 # Daemon version
-version = "0.9.101"
+version = "0.9.103"
 
 
 ##########################################################

worker-daemon/pvcworkerd/Daemon.py

@@ -55,7 +55,7 @@ from daemon_lib.autobackup import (
 )
 
 # Daemon version
-version = "0.9.101"
+version = "0.9.103"
 
 
 config = cfg.get_configuration()