Optimize CLI client dependency loading

The CLI client was quite heavy in loading in a lot of libraries and
modules during runtime, which slowed it down quite a bit, especially on
slower systems.

This commit makes several major changes to help improve the situation.

1. Don't use pkg_resources to get our version, just hardcode it.

2. Reimplement our entire API call to use a custom http.client-based
system that prevents importing any unnecessary libraries (with a
custom User-Agent too).

3. Implement a lazy-loading method for some of the heavier modules, so
that they are only loaded if absolutely necessary.

.gitignore

@@ -8,3 +8,4 @@ debian/pvc-*/
 debian/*.log
 debian/*.substvars
 debian/files
+client-cli/build/

.version

@@ -1 +1 @@
-0.9.106
+0.9.107

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## PVC Changelog
 
+###### [v0.9.107](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.107)
+
+  * [Worker Daemon] Fixes a bug where snapshot removal fails during autobackups
+
 ###### [v0.9.106](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.106)
 
   * [API Daemon] Fixes a calculation bug when checking storage free space

api-daemon/pvcapid/Daemon.py

@@ -81,6 +81,7 @@ def create_app():
     print("|--------------------------------------------------------------|")
     print("| Parallel Virtual Cluster API daemon v{0: <23} |".format(version))
     print("| Debug: {0: <53} |".format(str(config["debug"])))
+    print("| Cluster: {0: <51} |".format(config["cluster_name"]))
     print("| API version: v{0: <46} |".format(API_VERSION))
     print(
         "| Listen: {0: <52} |".format(

bump-version

@@ -22,7 +22,8 @@ sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," node-d
 sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," health-daemon/pvchealthd/Daemon.py
 sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," worker-daemon/pvcworkerd/Daemon.py
 sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," api-daemon/pvcapid/Daemon.py
-sed -i "s,version=\"${current_version}\",version=\"${new_version}\"," client-cli/setup.py
+sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," client-cli/pyproject.toml
+sed -i "s,VERSION = \"${current_version}\",VERSION = \"${new_version}\"," client-cli/pvc/cli/cli.py
 echo ${new_version} > .version
 
 changelog_tmpdir=$( mktemp -d )

client-cli/pvc/cli/cli.py

@@ -19,13 +19,23 @@
 #
 ###############################################################################
 
+VERSION = "0.9.107"
+
+# Import only the essential parts of click for CLI definition
+from click import group, command, option, argument, pass_context
+
+# Import minimal required modules
+from pvc.lib.common import call_api
+
+# Use lazy imports for modules not needed during initial CLI parsing
+from pvc.lib.lazy_imports import yaml, click_advanced
+
 from colorama import Fore
 from difflib import unified_diff
 from functools import wraps
 from json import dump as jdump
 from json import dumps as jdumps
 from json import loads as jloads
-from lxml.etree import fromstring, tostring
 from os import environ, makedirs, path
 from re import sub, match
 from yaml import load as yload
@@ -98,10 +108,7 @@ def version(ctx, param, value):
     if not value or ctx.resilient_parsing:
         return
 
-    from pkg_resources import get_distribution
-
-    version = get_distribution("pvc").version
-    echo(CLI_CONFIG, f"Parallel Virtual Cluster CLI client version {version}")
+    echo(CLI_CONFIG, f"Parallel Virtual Cluster CLI client version {VERSION}")
     ctx.exit()
 
 
@@ -1192,6 +1199,8 @@ def cli_vm_define(
 
     # Verify our XML is sensible
     try:
+        from lxml.etree import fromstring, tostring
+
         xml_data = fromstring(vmconfig_data)
         new_cfg = tostring(xml_data, pretty_print=True).decode("utf8")
     except Exception:
@@ -1377,6 +1386,9 @@ def cli_vm_modify(
 
     # Grab the current config
     current_vm_cfg_raw = vm_information.get("xml")
+
+    from lxml.etree import fromstring, tostring
+
     xml_data = fromstring(current_vm_cfg_raw)
     current_vm_cfgfile = tostring(xml_data, pretty_print=True).decode("utf8").strip()
 
@@ -1435,6 +1447,8 @@ def cli_vm_modify(
 
     # Verify our XML is sensible
     try:
+        from lxml.etree import fromstring, tostring
+
         xml_data = fromstring(new_vm_cfgfile)
         new_cfg = tostring(xml_data, pretty_print=True).decode("utf8")
     except Exception as e:
@@ -3265,6 +3279,9 @@ def cli_vm_dump(filename, domain):
         finish(False, 'ERROR: Could not find VM "{}"!'.format(domain))
 
     current_vm_cfg_raw = retdata.get("xml")
+
+    from lxml.etree import fromstring, tostring
+
     xml_data = fromstring(current_vm_cfg_raw)
     current_vm_cfgfile = tostring(xml_data, pretty_print=True).decode("utf8")
     xml = current_vm_cfgfile.strip()

client-cli/pvc/cli/formatters.py

@@ -221,7 +221,7 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data):
             continue
         if state in ["start"]:
             state_colour = ansii["green"]
-        elif state in ["migrate", "disable", "provision", "mirror"]:
+        elif state in ["migrate", "disable", "provision"]:
             state_colour = ansii["blue"]
         elif state in ["mirror"]:
             state_colour = ansii["purple"]

client-cli/pvc/cli/helpers.py

@@ -20,7 +20,6 @@
 ###############################################################################
 
 from click import echo as click_echo
-from distutils.util import strtobool
 from json import load as jload
 from json import dump as jdump
 from os import chmod, environ, getpid, path, get_terminal_size
@@ -150,9 +149,7 @@ def get_config(store_data, connection=None):
     if connection == "local":
         config["verify_ssl"] = False
     else:
-        config["verify_ssl"] = bool(
-            strtobool(environ.get("PVC_CLIENT_VERIFY_SSL", "True"))
-        )
+        config["verify_ssl"] = environ.get("PVC_CLIENT_VERIFY_SSL", "True") == "True"
 
     return config
 

client-cli/pvc/lib/common.py

@@ -19,13 +19,289 @@
 #
 ###############################################################################
 
-import os
-import math
-import time
-import requests
-import click
 from ast import literal_eval
-from urllib3 import disable_warnings
+from click import echo, progressbar
+from math import ceil
+from os.path import getsize
+from time import time
+import json
+import socket
+import ssl
+
+# Define a Response class to mimic requests.Response
+class Response:
+    def __init__(self, status_code, headers, content):
+        self.status_code = status_code
+        self.headers = headers
+        self.content = content
+        self._json = None
+        self.text = content.decode('utf-8', errors='replace') if content else ''
+        self.ok = 200 <= status_code < 300
+        self.url = None  # Will be set by call_api
+        self.reason = None  # HTTP reason phrase
+        self.encoding = 'utf-8'
+        self.elapsed = None  # Time elapsed
+        self.request = None  # Original request
+
+    def json(self):
+        if self._json is None:
+            # Don't catch JSONDecodeError - let it propagate like requests does
+            self._json = json.loads(self.content.decode('utf-8'))
+        return self._json
+
+    def raise_for_status(self):
+        """Raises HTTPError if the status code is 4XX/5XX"""
+        if 400 <= self.status_code < 600:
+            raise Exception(f"HTTP Error {self.status_code}")
+
+# Define ConnectionError to mimic requests.exceptions.ConnectionError
+class ConnectionError(Exception):
+    pass
+
+class ErrorResponse(Response):
+    def __init__(self, json_data, status_code, headers):
+        self.status_code = status_code
+        self.headers = headers
+        self._json = json_data
+        self.content = json.dumps(json_data).encode('utf-8') if json_data else b''
+
+    def json(self):
+        return self._json
+
+def _encode_params(params):
+    """Simple URL parameter encoder"""
+    if not params:
+        return ''
+
+    parts = []
+    for key, value in params.items():
+        if isinstance(value, bool):
+            value = str(value).lower()
+        elif value is None:
+            value = ''
+        elif isinstance(value, (list, tuple)):
+            # Handle lists and tuples by creating multiple parameters with the same name
+            for item in value:
+                parts.append(f"{key}={item}")
+            continue  # Skip the normal append since we've already added the parts
+        else:
+            value = str(value)
+        parts.append(f"{key}={value}")
+
+    return '?' + '&'.join(parts)
+
+def call_api(
+    config,
+    operation,
+    request_uri,
+    headers={},
+    params=None,
+    data=None,
+    files=None,
+):
+    """
+    Make an API call to the PVC API using native Python libraries.
+    """
+    # Set timeouts - fast connection timeout, longer read timeout
+    connect_timeout = 2.05  # Connection timeout in seconds
+    read_timeout = 172800.0 # Read timeout in seconds (much longer to allow for slow operations)
+
+    # Import VERSION from cli.py - this is a lightweight import since cli.py is already loaded
+    from pvc.cli.cli import VERSION
+
+    # Set User-Agent header if not already set
+    if "User-Agent" not in headers:
+        headers["User-Agent"] = f"pvc-client-cli/{VERSION}"
+
+    # Craft the URI
+    uri = "{}://{}{}{}".format(
+        config["api_scheme"], config["api_host"], config["api_prefix"], request_uri
+    )
+
+    # Parse the URI without using urllib
+    if '://' in uri:
+        scheme, rest = uri.split('://', 1)
+    else:
+        scheme = 'http'
+        rest = uri
+
+    if '/' in rest:
+        netloc, path = rest.split('/', 1)
+        path = '/' + path
+    else:
+        netloc = rest
+        path = '/'
+
+    # Extract host and port
+    if ':' in netloc:
+        host, port_str = netloc.split(':', 1)
+        port = int(port_str)
+    else:
+        host = netloc
+        port = 443 if scheme == 'https' else 80
+
+    # Craft the authentication header if required
+    if config["api_key"]:
+        headers["X-Api-Key"] = config["api_key"]
+
+    # Add content type if not present
+    if "Content-Type" not in headers and data is not None and files is None:
+        headers["Content-Type"] = "application/json"
+
+    # Prepare query string
+    query_string = _encode_params(params)
+
+    # Prepare path with query string
+    full_path = path + query_string
+
+    # Prepare body
+    body = None
+    if data is not None and files is None:
+        if isinstance(data, dict):
+            body = json.dumps(data).encode('utf-8')
+        else:
+            body = data.encode('utf-8') if isinstance(data, str) else data
+
+    # Handle file uploads (multipart/form-data)
+    if files:
+        boundary = '----WebKitFormBoundary' + str(int(time()))
+        headers['Content-Type'] = f'multipart/form-data; boundary={boundary}'
+
+        body = b''
+        # Add form fields
+        if data:
+            for key, value in data.items():
+                body += f'--{boundary}\r\n'.encode()
+                body += f'Content-Disposition: form-data; name="{key}"\r\n\r\n'.encode()
+                body += f'{value}\r\n'.encode()
+
+        # Add files
+        for key, file_tuple in files.items():
+            filename, fileobj, content_type = file_tuple
+            body += f'--{boundary}\r\n'.encode()
+            body += f'Content-Disposition: form-data; name="{key}"; filename="{filename}"\r\n'.encode()
+            body += f'Content-Type: {content_type}\r\n\r\n'.encode()
+            body += fileobj.read()
+            body += b'\r\n'
+
+        body += f'--{boundary}--\r\n'.encode()
+
+    # Use http.client instead of raw sockets for better HTTP protocol handling
+    try:
+        # Special handling for GET with retries
+        if operation == "get":
+            retry_on_code = [429, 500, 502, 503, 504]
+            for i in range(3):
+                failed = False
+                try:
+                    # Create the appropriate connection with separate timeouts
+                    if scheme == 'https':
+                        import ssl
+                        context = None
+                        if not config["verify_ssl"]:
+                            context = ssl._create_unverified_context()
+
+                        import http.client
+                        conn = http.client.HTTPSConnection(
+                            host, 
+                            port=port,
+                            timeout=connect_timeout,  # This is the connection timeout
+                            context=context
+                        )
+                    else:
+                        import http.client
+                        conn = http.client.HTTPConnection(
+                            host,
+                            port=port,
+                            timeout=connect_timeout  # This is the connection timeout
+                        )
+
+                    # Make the request
+                    conn.request(operation.upper(), full_path, body=body, headers=headers)
+
+                    # Set a longer timeout for reading the response
+                    conn.sock.settimeout(read_timeout)
+
+                    http_response = conn.getresponse()
+
+                    # Read response data
+                    status_code = http_response.status
+                    response_headers = dict(http_response.getheaders())
+                    response_data = http_response.read()
+
+                    conn.close()
+
+                    # Create a Response object
+                    response = Response(status_code, response_headers, response_data)
+
+                    if response.status_code in retry_on_code:
+                        failed = True
+                        continue
+                    break
+                except Exception as e:
+                    failed = True
+                    if 'conn' in locals():
+                        conn.close()
+                    continue
+
+            if failed:
+                error = f"Code {response.status_code}" if 'response' in locals() else "Timeout"
+                raise ConnectionError(f"Failed to connect after 3 tries ({error})")
+        else:
+            # Handle other HTTP methods
+            if scheme == 'https':
+                import ssl
+                context = None
+                if not config["verify_ssl"]:
+                    context = ssl._create_unverified_context()
+
+                import http.client
+                conn = http.client.HTTPSConnection(
+                    host, 
+                    port=port,
+                    timeout=connect_timeout,  # This is the connection timeout
+                    context=context
+                )
+            else:
+                import http.client
+                conn = http.client.HTTPConnection(
+                    host,
+                    port=port,
+                    timeout=connect_timeout  # This is the connection timeout
+                )
+
+            # Make the request
+            conn.request(operation.upper(), full_path, body=body, headers=headers)
+
+            # Set a longer timeout for reading the response
+            conn.sock.settimeout(read_timeout)
+
+            http_response = conn.getresponse()
+
+            # Read response data
+            status_code = http_response.status
+            response_headers = dict(http_response.getheaders())
+            response_data = http_response.read()
+
+            conn.close()
+
+            # Create a Response object
+            response = Response(status_code, response_headers, response_data)
+
+    except Exception as e:
+        message = f"Failed to connect to the API: {e}"
+        code = getattr(response, 'status_code', 504) if 'response' in locals() else 504
+        response = ErrorResponse({"message": message}, code, None)
+
+    # Display debug output
+    if config["debug"]:
+        echo("API endpoint: {}".format(uri), err=True)
+        echo("Response code: {}".format(response.status_code), err=True)
+        echo("Response headers: {}".format(response.headers), err=True)
+        echo(err=True)
+
+    # Always return the response object - no special handling
+    return response
 
 
 def format_bytes(size_bytes):
@@ -39,7 +315,7 @@ def format_bytes(size_bytes):
     }
     human_bytes = "0B"
     for unit in sorted(byte_unit_matrix, key=byte_unit_matrix.get):
-        formatted_bytes = int(math.ceil(size_bytes / byte_unit_matrix[unit]))
+        formatted_bytes = int(ceil(size_bytes / byte_unit_matrix[unit]))
         if formatted_bytes < 10000:
             human_bytes = "{}{}".format(formatted_bytes, unit)
             break
@@ -57,7 +333,7 @@ def format_metric(integer):
     }
     human_integer = "0"
     for unit in sorted(integer_unit_matrix, key=integer_unit_matrix.get):
-        formatted_integer = int(math.ceil(integer / integer_unit_matrix[unit]))
+        formatted_integer = int(ceil(integer / integer_unit_matrix[unit]))
         if formatted_integer < 10000:
             human_integer = "{}{}".format(formatted_integer, unit)
             break
@@ -97,12 +373,12 @@ def format_age(age_secs):
 
 class UploadProgressBar(object):
     def __init__(self, filename, end_message="", end_nl=True):
-        file_size = os.path.getsize(filename)
+        file_size = getsize(filename)
         file_size_human = format_bytes(file_size)
-        click.echo("Uploading file (total size {})...".format(file_size_human))
+        echo("Uploading file (total size {})...".format(file_size_human))
 
         self.length = file_size
-        self.time_last = int(round(time.time() * 1000)) - 1000
+        self.time_last = int(round(time() * 1000)) - 1000
         self.bytes_last = 0
         self.bytes_diff = 0
         self.is_end = False
@@ -114,7 +390,7 @@ class UploadProgressBar(object):
         else:
             self.end_suffix = ""
 
-        self.bar = click.progressbar(length=self.length, width=20, show_eta=True)
+        self.bar = progressbar(length=self.length, width=20, show_eta=True)
 
     def update(self, monitor):
         bytes_cur = monitor.bytes_read
@@ -123,7 +399,7 @@ class UploadProgressBar(object):
             self.is_end = True
         self.bytes_last = bytes_cur
 
-        time_cur = int(round(time.time() * 1000))
+        time_cur = int(round(time() * 1000))
         if (time_cur - 1000) > self.time_last:
             self.time_last = time_cur
             self.bar.update(self.bytes_diff)
@@ -132,124 +408,10 @@ class UploadProgressBar(object):
         if self.is_end:
             self.bar.update(self.bytes_diff)
             self.bytes_diff = 0
-            click.echo()
-            click.echo()
+            echo()
+            echo()
             if self.end_message:
-                click.echo(self.end_message + self.end_suffix, nl=self.end_nl)
-
-
-class ErrorResponse(requests.Response):
-    def __init__(self, json_data, status_code, headers):
-        self.json_data = json_data
-        self.status_code = status_code
-        self.headers = headers
-
-    def json(self):
-        return self.json_data
-
-
-def call_api(
-    config,
-    operation,
-    request_uri,
-    headers={},
-    params=None,
-    data=None,
-    files=None,
-):
-    # Set the connect timeout to 2 seconds but extremely long (48 hour) data timeout
-    timeout = (2.05, 172800)
-
-    # Craft the URI
-    uri = "{}://{}{}{}".format(
-        config["api_scheme"], config["api_host"], config["api_prefix"], request_uri
-    )
-
-    # Craft the authentication header if required
-    if config["api_key"]:
-        headers["X-Api-Key"] = config["api_key"]
-
-    # Determine the request type and hit the API
-    disable_warnings()
-    try:
-        response = None
-        if operation == "get":
-            retry_on_code = [429, 500, 502, 503, 504]
-            for i in range(3):
-                failed = False
-                try:
-                    response = requests.get(
-                        uri,
-                        timeout=timeout,
-                        headers=headers,
-                        params=params,
-                        data=data,
-                        verify=config["verify_ssl"],
-                    )
-                    if response.status_code in retry_on_code:
-                        failed = True
-                        continue
-                    break
-                except requests.exceptions.ConnectionError:
-                    failed = True
-                    continue
-            if failed:
-                error = f"Code {response.status_code}" if response else "Timeout"
-                raise requests.exceptions.ConnectionError(
-                    f"Failed to connect after 3 tries ({error})"
-                )
-        if operation == "post":
-            response = requests.post(
-                uri,
-                timeout=timeout,
-                headers=headers,
-                params=params,
-                data=data,
-                files=files,
-                verify=config["verify_ssl"],
-            )
-        if operation == "put":
-            response = requests.put(
-                uri,
-                timeout=timeout,
-                headers=headers,
-                params=params,
-                data=data,
-                files=files,
-                verify=config["verify_ssl"],
-            )
-        if operation == "patch":
-            response = requests.patch(
-                uri,
-                timeout=timeout,
-                headers=headers,
-                params=params,
-                data=data,
-                verify=config["verify_ssl"],
-            )
-        if operation == "delete":
-            response = requests.delete(
-                uri,
-                timeout=timeout,
-                headers=headers,
-                params=params,
-                data=data,
-                verify=config["verify_ssl"],
-            )
-    except Exception as e:
-        message = "Failed to connect to the API: {}".format(e)
-        code = response.status_code if response else 504
-        response = ErrorResponse({"message": message}, code, None)
-
-    # Display debug output
-    if config["debug"]:
-        click.echo("API endpoint: {}".format(uri), err=True)
-        click.echo("Response code: {}".format(response.status_code), err=True)
-        click.echo("Response headers: {}".format(response.headers), err=True)
-        click.echo(err=True)
-
-    # Return the response object
-    return response
+                echo(self.end_message + self.end_suffix, nl=self.end_nl)
 
 
 def get_wait_retdata(response, wait_flag):

client-cli/pvc/lib/lazy_imports.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+# lazy_imports.py - Lazy module importer library
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2024 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+class LazyModule:
+    """
+    A proxy for a module that is loaded only when actually used
+    """
+    def __init__(self, name):
+        self.name = name
+        self._module = None
+
+    def __getattr__(self, attr):
+        if self._module is None:
+            import importlib
+            self._module = importlib.import_module(self.name)
+        return getattr(self._module, attr)
+
+# Create lazy module proxies
+yaml = LazyModule('yaml')
+click_advanced = LazyModule('click')  # For advanced click features not used at startup

client-cli/pvc/lib/provisioner.py

@@ -19,11 +19,6 @@
 #
 ###############################################################################
 
-from requests_toolbelt.multipart.encoder import (
-    MultipartEncoder,
-    MultipartEncoderMonitor,
-)
-
 import pvc.lib.ansiprint as ansiprint
 from pvc.lib.common import UploadProgressBar, call_api, get_wait_retdata
 
@@ -549,6 +544,12 @@ def ova_upload(config, name, ova_file, params):
     bar = UploadProgressBar(
         ova_file, end_message="Parsing file on remote side...", end_nl=False
     )
+
+    from requests_toolbelt.multipart.encoder import (
+        MultipartEncoder,
+        MultipartEncoderMonitor,
+    )
+
     upload_data = MultipartEncoder(
         fields={"file": ("filename", open(ova_file, "rb"), "application/octet-stream")}
     )

client-cli/pvc/lib/storage.py

@@ -23,10 +23,6 @@ import math
 
 from os import path
 from json import loads
-from requests_toolbelt.multipart.encoder import (
-    MultipartEncoder,
-    MultipartEncoderMonitor,
-)
 
 import pvc.lib.ansiprint as ansiprint
 from pvc.lib.common import UploadProgressBar, call_api, get_wait_retdata
@@ -1212,6 +1208,12 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file):
     bar = UploadProgressBar(
         image_file, end_message="Parsing file on remote side...", end_nl=False
     )
+
+    from requests_toolbelt.multipart.encoder import (
+        MultipartEncoder,
+        MultipartEncoderMonitor,
+    )
+
     upload_data = MultipartEncoder(
         fields={
             "file": ("filename", open(image_file, "rb"), "application/octet-stream")

client-cli/pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pvc"
+version = "0.9.107"
+dependencies = [
+    "Click",
+    "PyYAML",
+    "lxml",
+    "colorama",
+    "requests",
+    "requests-toolbelt",
+]
+
+[tool.setuptools]
+packages = ["pvc.cli", "pvc.lib"]
+
+[project.scripts]
+pvc = "pvc.cli.cli:cli"

client-cli/setup.py

@@ -1,20 +0,0 @@
-from setuptools import setup
-
-setup(
-    name="pvc",
-    version="0.9.106",
-    packages=["pvc.cli", "pvc.lib"],
-    install_requires=[
-        "Click",
-        "PyYAML",
-        "lxml",
-        "colorama",
-        "requests",
-        "requests-toolbelt",
-    ],
-    entry_points={
-        "console_scripts": [
-            "pvc = pvc.cli.cli:cli",
-        ],
-    },
-)

daemon-common/autobackup.py

@@ -469,15 +469,16 @@ def run_vm_backup(zkhandler, celery, config, vm_detail, force_full=False):
 
     if len(marked_for_deletion) > 0:
         for backup_to_delete in marked_for_deletion:
-            ret = vm.vm_worker_remove_snapshot(
-                zkhandler, None, vm_name, backup_to_delete["snapshot_name"]
-            )
-            if ret is False:
-                error_message = f"Failed to remove obsolete backup snapshot '{backup_to_delete['snapshot_name']}', leaving in tracked backups"
+            try:
+                ret = vm.vm_worker_remove_snapshot(
+                    zkhandler, None, vm_name, backup_to_delete["snapshot_name"]
+                )
+            except Exception:
+                error_message = f"Failed to remove obsolete backup snapshot '{backup_to_delete['snapshot_name']}', removing from tracked backups anyways"
                 log_err(celery, error_message)
-            else:
-                rmtree(f"{vm_backup_path}/{backup_to_delete['snapshot_name']}")
-                tracked_backups.remove(backup_to_delete)
+
+            rmtree(f"{vm_backup_path}/{backup_to_delete['snapshot_name']}")
+            tracked_backups.remove(backup_to_delete)
 
     tracked_backups = update_tracked_backups()
     return tracked_backups

daemon-common/cluster.py

@@ -496,6 +496,7 @@ def getClusterInformation(zkhandler):
 
     # Format the status data
     cluster_information = {
+        "cluster_name": zkhandler.read("base.config"),
         "cluster_health": getClusterHealthFromFaults(zkhandler, faults_data),
         "node_health": getNodeHealth(zkhandler, node_list),
         "maintenance": maintenance_state,

daemon-common/common.py

@@ -1212,3 +1212,7 @@ def get_detect_device(detect_string):
         return device
     else:
         return None
+
+
+def translate_domains_to_names(zkhandler, domain_list):
+    return list(zkhandler.read_many([("domain.name", d) for d in domain_list]))

daemon-common/node.py

@@ -142,7 +142,9 @@ def getNodeInformation(zkhandler, node_name):
     node_mem_free = int(_node_mem_free)
     node_load = float(_node_load)
     node_domains_count = int(_node_domains_count)
-    node_running_domains = _node_running_domains.split()
+    node_running_domains = common.translate_domains_to_names(
+        zkhandler, _node_running_domains.split()
+    )
 
     try:
         node_health = int(_node_health)

debian/changelog

@@ -1,3 +1,9 @@
+pvc (0.9.107-0) unstable; urgency=high
+
+  * [Worker Daemon] Fixes a bug where snapshot removal fails during autobackups
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Mon, 10 Feb 2025 23:15:21 -0500
+
 pvc (0.9.106-0) unstable; urgency=high
 
   * [API Daemon] Fixes a calculation bug when checking storage free space

debian/compat

@@ -1 +1 @@
-9
+13

debian/control

@@ -5,6 +5,7 @@ Maintainer: Joshua Boniface <joshua@boniface.me>
 Standards-Version: 3.9.8
 Homepage: https://www.boniface.me
 X-Python3-Version: >= 3.7
+Build-Depends: pybuild-plugin-pyproject, dh-python
 
 Package: pvc-daemon-node
 Architecture: all
@@ -48,7 +49,7 @@ Description: Parallel Virtual Cluster common libraries
 
 Package: pvc-client-cli
 Architecture: all
-Depends: python3-requests, python3-requests-toolbelt, python3-yaml, python3-lxml, python3-click
+Depends: python3-requests-toolbelt, python3-yaml, python3-lxml, python3-click
 Description: Parallel Virtual Cluster CLI client
  A KVM/Zookeeper/Ceph-based VM and private cloud manager
  .

debian/rules

@@ -7,13 +7,14 @@ export DH_VERBOSE = 1
 	dh $@ --with python3
 
 override_dh_python3:
-	cd $(CURDIR)/client-cli; pybuild --system=distutils --dest-dir=../debian/pvc-client-cli/
+	cd $(CURDIR)/client-cli; pybuild --system=pyproject --dest-dir=../debian/pvc-client-cli/
 	mkdir -p debian/pvc-client-cli/usr/lib/python3
 	mv debian/pvc-client-cli/usr/lib/python3*/* debian/pvc-client-cli/usr/lib/python3/
 	rm -r $(CURDIR)/client-cli/.pybuild $(CURDIR)/client-cli/pvc.egg-info
 
 override_dh_auto_clean:
-	find . -name "__pycache__" -o -name ".pybuild" -exec rm -fr {} + || true
+	find $(CURDIR) -name "__pycache__" -o -name ".pybuild" -exec rm -fr {} + || true
+	rm -r $(CURDIR)/client-cli/build || true
 
 # If you need to rebuild the Sphinx documentation
 # Add spinxdoc to the dh --with line

health-daemon/pvchealthd/Daemon.py

@@ -33,7 +33,7 @@ import os
 import signal
 
 # Daemon version
-version = "0.9.106"
+version = "0.9.107"
 
 
 ##########################################################
@@ -64,6 +64,7 @@ def entrypoint():
     logger.out("|--------------------------------------------------------------|")
     logger.out("| Parallel Virtual Cluster health daemon v{0: <20} |".format(version))
     logger.out("| Debug: {0: <53} |".format(str(config["debug"])))
+    logger.out("| Cluster: {0: <51} |".format(config["cluster_name"]))
     logger.out("| FQDN: {0: <54} |".format(config["node_fqdn"]))
     logger.out("| Host: {0: <54} |".format(config["node_hostname"]))
     logger.out("| ID: {0: <56} |".format(config["node_id"]))

node-daemon/pvcnoded/Daemon.py

@@ -49,7 +49,7 @@ import re
 import json
 
 # Daemon version
-version = "0.9.106"
+version = "0.9.107"
 
 
 ##########################################################
@@ -83,6 +83,7 @@ def entrypoint():
     logger.out("|--------------------------------------------------------------|")
     logger.out("| Parallel Virtual Cluster node daemon v{0: <22} |".format(version))
     logger.out("| Debug: {0: <53} |".format(str(config["debug"])))
+    logger.out("| Cluster: {0: <51} |".format(config["cluster_name"]))
     logger.out("| FQDN: {0: <54} |".format(config["node_fqdn"]))
     logger.out("| Host: {0: <54} |".format(config["node_hostname"]))
     logger.out("| ID: {0: <56} |".format(config["node_id"]))
@@ -301,6 +302,9 @@ def entrypoint():
     # Set up this node in Zookeeper
     pvcnoded.util.zookeeper.setup_node(logger, config, zkhandler)
 
+    # Set the cluster name in Zookeeper
+    zkhandler.write([("base.config", config["cluster_name"])])
+
     # Check that the primary node key exists and create it with us as primary if not
     try:
         current_primary = zkhandler.read("base.config.primary_node")

worker-daemon/pvcworkerd/Daemon.py

@@ -58,7 +58,7 @@ from daemon_lib.automirror import (
 )
 
 # Daemon version
-version = "0.9.106"
+version = "0.9.107"
 
 
 config = cfg.get_configuration()