Bump version to 0.9.91

Use consistent and less error-prone find rm's
Add cleanup of pycaches to CLI install
2024-01-23 10:40:59 -05:00 · 2024-01-23 10:40:48 -05:00 · 2024-01-23 10:22:50 -05:00 · 2024-01-23 09:58:32 -05:00 · 2024-01-21 14:23:44 -05:00 · 2024-01-18 13:00:48 -05:00
26 changed files with 122 additions and 4833 deletions
--- a/.version
+++ b/.version
@ -1 +1 @@
-0.9.89
+0.9.91
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -1,5 +1,20 @@
 ## PVC Changelog

+###### [v0.9.91](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.91)
+
+  * [Client CLI] Fixes a bug and improves output during cluster task events.
+  * [Client CLI] Improves the output of the task list display.
+  * [Provisioner] Fixes some missing cloud-init modules in the default debootstrap script.
+  * [Client CLI] Fixes a bug with a missing argument to the vm_define helper function.
+  * [All] Fixes inconsistent package find + rm commands to avoid errors in dpkg.
+
+###### [v0.9.90](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.90)
+
+  * [Client CLI/API Daemon] Adds additional backup metainfo and an emailed report option to autobackups.
+  * [All] Adds a live migration maximum downtime selector to help with busy VM migrations.
+  * [API Daemon] Fixes a database migration bug on Debian 10/11.
+  * [Node Daemon] Fixes a race condition when applying Zookeeper schema changes.
+
 ###### [v0.9.89](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.89)

  * [API/Worker Daemons] Fixes a bug with the Celery result backends not being properly initialized on Debian 10/11.
--- a/api-daemon/provisioner/examples/script/3-debootstrap.py
+++ b/api-daemon/provisioner/examples/script/3-debootstrap.py
@ -589,11 +589,13 @@ After=multi-user.target
                 - migrator
                 - bootcmd
                 - write-files
+                 - growpart
                 - resizefs
                 - set_hostname
                 - update_hostname
                 - update_etc_hosts
                 - ca-certs
+                 - users-groups
                 - ssh
                
                cloud_config_modules:
--- a/api-daemon/pvc-api-db-upgrade
+++ b/api-daemon/pvc-api-db-upgrade
@ -12,16 +12,7 @@ fi

 pushd /usr/share/pvc

-case "$( cat /etc/debian_version )" in
-    10.*|11.*)
-        # Debian 10 & 11
-        ./pvcapid-manage_legacy.py db upgrade
-    ;;
-    *)
-        # Debian 12+
-        export FLASK_APP=./pvcapid-manage_flask.py
-        flask db upgrade
-    ;;
-esac
+export FLASK_APP=./pvcapid-manage-flask.py
+flask db upgrade

 popd
--- a/api-daemon/pvcapid-manage-flask.py
+++ b/api-daemon/pvcapid-manage-flask.py
--- a/api-daemon/pvcapid-manage_legacy.py
+++ b/api-daemon/pvcapid-manage_legacy.py
@ -1,33 +0,0 @@
-#!/usr/bin/env python3
-
-# pvcapid-manage_legacy.py - PVC Database management tasks (Legacy)
-# Part of the Parallel Virtual Cluster (PVC) system
-#
-#    Copyright (C) 2018-2024 Joshua M. Boniface <joshua@boniface.me>
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, version 3.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-###############################################################################
-
-from flask_migrate import Migrate, MigrateCommand, Manager
-
-from pvcapid.flaskapi import app, db
-from pvcapid.models import *  # noqa F401,F403
-
-migrate = Migrate(app, db)
-manager = Manager(app)
-
-manager.add_command("db", MigrateCommand)
-
-if __name__ == "__main__":
-    manager.run()
--- a/api-daemon/pvcapid/Daemon.py
+++ b/api-daemon/pvcapid/Daemon.py
@ -27,7 +27,7 @@ from distutils.util import strtobool as dustrtobool
 import daemon_lib.config as cfg

 # Daemon version
-version = "0.9.89"
+version = "0.9.91"

 # API version
 API_VERSION = 1.0
--- a/api-daemon/swagger.html
+++ b/api-daemon/swagger.html
@ -1,13 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <title>PVC Client API Documentation</title>
-    <meta charset="utf-8"/>
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <style> body { margin: 0; padding: 0; } </style>
-  </head>
-  <body>
-    <redoc spec-url='./swagger.json' hide-loading></redoc>
-    <script src="https://rebilly.github.io/ReDoc/releases/latest/redoc.min.js"> </script>
-  </body>
-</html>
--- a/api-daemon/swagger.json
+++ b/api-daemon/swagger.json
--- a/client-cli/pvc/cli/cli.py
+++ b/client-cli/pvc/cli/cli.py
@ -687,7 +687,10 @@ def cli_cluster_task(task_id, wait_flag, format_function):

    if wait_flag:
        # First validate that this is actually a valid task that is running
+        echo(CLI_CONFIG, "Querying cluster for tasks...", newline=False)
        retcode, retdata = pvc.lib.common.task_status(CLI_CONFIG, None)
+        echo(CLI_CONFIG, " done.")
+        echo(CLI_CONFIG, "")
        if task_id in [i["id"] for i in retdata]:
            task = [i for i in retdata if i["id"] == task_id][0]
            retmsg = wait_for_celery_task(
@ -699,7 +702,10 @@ def cli_cluster_task(task_id, wait_flag, format_function):
            retmsg = f"No task with ID {task_id} found."
        finish(retcode, retmsg)
    else:
+        echo(CLI_CONFIG, "Querying cluster for tasks...", newline=False)
        retcode, retdata = pvc.lib.common.task_status(CLI_CONFIG, task_id)
+        echo(CLI_CONFIG, " done.")
+        echo(CLI_CONFIG, "")
        finish(retcode, retdata, format_function)


--- a/client-cli/pvc/cli/formatters.py
+++ b/client-cli/pvc/cli/formatters.py
@ -645,6 +645,24 @@ def cli_cluster_task_format_pretty(CLI_CONFIG, task_data):
        if _task_type_length > task_type_length:
            task_type_length = _task_type_length

+        for arg_name, arg_data in task["kwargs"].items():
+            # Skip the "run_on" argument
+            if arg_name == "run_on":
+                continue
+
+            # task_arg_name column
+            _task_arg_name_length = len(str(arg_name)) + 1
+            if _task_arg_name_length > task_arg_name_length:
+                task_arg_name_length = _task_arg_name_length
+
+    task_header_length = (
+        task_id_length + task_name_length + task_type_length + task_worker_length + 3
+    )
+    max_task_data_length = (
+        MAX_CONTENT_WIDTH - task_header_length - task_arg_name_length - 2
+    )
+
+    for task in task_data:
        updated_kwargs = list()
        for arg_name, arg_data in task["kwargs"].items():
            # Skip the "run_on" argument
@ -656,15 +674,30 @@ def cli_cluster_task_format_pretty(CLI_CONFIG, task_data):
            if _task_arg_name_length > task_arg_name_length:
                task_arg_name_length = _task_arg_name_length

-            if len(str(arg_data)) > 17:
-                arg_data = arg_data[:17] + "..."
+            if isinstance(arg_data, list):
+                for subarg_data in arg_data:
+                    if len(subarg_data) > max_task_data_length:
+                        subarg_data = (
+                            str(subarg_data[: max_task_data_length - 4]) + " ..."
+                        )

-            # task_arg_data column
-            _task_arg_data_length = len(str(arg_data)) + 1
-            if _task_arg_data_length > task_arg_data_length:
-                task_arg_data_length = _task_arg_data_length
+                    # task_arg_data column
+                    _task_arg_data_length = len(str(subarg_data)) + 1
+                    if _task_arg_data_length > task_arg_data_length:
+                        task_arg_data_length = _task_arg_data_length
+
+                    updated_kwargs.append({"name": arg_name, "data": subarg_data})
+            else:
+                if len(str(arg_data)) > 24:
+                    arg_data = str(arg_data[:24]) + " ..."
+
+                    # task_arg_data column
+                    _task_arg_data_length = len(str(arg_data)) + 1
+                    if _task_arg_data_length > task_arg_data_length:
+                        task_arg_data_length = _task_arg_data_length
+
+                updated_kwargs.append({"name": arg_name, "data": arg_data})

-            updated_kwargs.append({"name": arg_name, "data": arg_data})
        task["kwargs"] = updated_kwargs
        tasks.append(task)

--- a/client-cli/pvc/cli/waiters.py
+++ b/client-cli/pvc/cli/waiters.py
@ -115,6 +115,8 @@ def wait_for_celery_task(CLI_CONFIG, task_detail, start_late=False):
        )
        while True:
            sleep(0.5)
+            if isinstance(task_status, tuple):
+                continue
            if task_status.get("state") != "RUNNING":
                break
            if task_status.get("current") > last_task:
--- a/client-cli/pvc/lib/vm.py
+++ b/client-cli/pvc/lib/vm.py
@ -89,6 +89,7 @@ def vm_define(
    node_selector,
    node_autostart,
    migration_method,
+    migration_max_downtime,
    user_tags,
    protected_tags,
 ):
@ -96,7 +97,7 @@ def vm_define(
    Define a new VM on the cluster

    API endpoint: POST /vm
-    API arguments: xml={xml}, node={node}, limit={node_limit}, selector={node_selector}, autostart={node_autostart}, migration_method={migration_method}, user_tags={user_tags}, protected_tags={protected_tags}
+    API arguments: xml={xml}, node={node}, limit={node_limit}, selector={node_selector}, autostart={node_autostart}, migration_method={migration_method}, migration_max_downtime={migration_max_downtime}, user_tags={user_tags}, protected_tags={protected_tags}
    API schema: {"message":"{data}"}
    """
    params = {
@ -105,6 +106,7 @@ def vm_define(
        "selector": node_selector,
        "autostart": node_autostart,
        "migration_method": migration_method,
+        "migration_max_downtime": migration_max_downtime,
        "user_tags": user_tags,
        "protected_tags": protected_tags,
    }
--- a/client-cli/setup.py
+++ b/client-cli/setup.py
@ -2,7 +2,7 @@ from setuptools import setup

 setup(
    name="pvc",
-    version="0.9.89",
+    version="0.9.91",
    packages=["pvc.cli", "pvc.lib"],
    install_requires=[
        "Click",
--- a/debian/changelog
+++ b/debian/changelog
@ -1,3 +1,22 @@
+pvc (0.9.91-0) unstable; urgency=high
+
+  * [Client CLI] Fixes a bug and improves output during cluster task events.
+  * [Client CLI] Improves the output of the task list display.
+  * [Provisioner] Fixes some missing cloud-init modules in the default debootstrap script.
+  * [Client CLI] Fixes a bug with a missing argument to the vm_define helper function.
+  * [All] Fixes inconsistent package find + rm commands to avoid errors in dpkg.
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Tue, 23 Jan 2024 10:02:19 -0500
+
+pvc (0.9.90-0) unstable; urgency=high
+
+  * [Client CLI/API Daemon] Adds additional backup metainfo and an emailed report option to autobackups.
+  * [All] Adds a live migration maximum downtime selector to help with busy VM migrations.
+  * [API Daemon] Fixes a database migration bug on Debian 10/11.
+  * [Node Daemon] Fixes a race condition when applying Zookeeper schema changes.
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Thu, 11 Jan 2024 00:14:49 -0500
+
 pvc (0.9.89-0) unstable; urgency=high

  * [API/Worker Daemons] Fixes a bug with the Celery result backends not being properly initialized on Debian 10/11.
--- a/debian/pvc-client-cli.postinst
+++ b/debian/pvc-client-cli.postinst
@ -2,7 +2,12 @@

 # Generate the bash completion configuration
 if [ -d /etc/bash_completion.d ]; then
+    echo "Installing BASH completion configuration"
    _PVC_COMPLETE=source_bash pvc > /etc/bash_completion.d/pvc
 fi

+# Remove any cached CPython directories or files
+echo "Cleaning up CPython caches"
+find /usr/lib/python3/dist-packages/pvc -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true
+
 exit 0
--- a/debian/pvc-daemon-api.preinst
+++ b/debian/pvc-daemon-api.preinst
@ -1,5 +1,5 @@
 #!/bin/sh

 # Remove any cached CPython directories or files
-echo "Cleaning up existing CPython files"
-find /usr/share/pvc/pvcapid -type d -name "__pycache__" -exec rm -rf {} \; &>/dev/null || true
+echo "Cleaning up CPython caches"
+find /usr/share/pvc/pvcapid -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true
--- a/debian/pvc-daemon-common.preinst
+++ b/debian/pvc-daemon-common.preinst
@ -0,0 +1,5 @@
+#!/bin/sh
+
+# Remove any cached CPython directories or files
+echo "Cleaning up CPython caches"
+find /usr/share/pvc/daemon_lib -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true
--- a/debian/pvc-daemon-health.preinst
+++ b/debian/pvc-daemon-health.preinst
@ -1,6 +1,6 @@
 #!/bin/sh

 # Remove any cached CPython directories or files
-echo "Cleaning up existing CPython files"
-find /usr/share/pvc/pvchealthd -type d -name "__pycache__" -exec rm -rf {} \; &>/dev/null || true
-find /usr/share/pvc/plugins -type d -name "__pycache__" -exec rm -rf {} \; &>/dev/null || true
+echo "Cleaning up CPython caches"
+find /usr/share/pvc/pvchealthd -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true
+find /usr/share/pvc/plugins -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true
--- a/debian/pvc-daemon-node.preinst
+++ b/debian/pvc-daemon-node.preinst
@ -1,5 +1,5 @@
 #!/bin/sh

 # Remove any cached CPython directories or files
-echo "Cleaning up existing CPython files"
-find /usr/share/pvc/pvcnoded -type d -name "__pycache__" -exec rm -rf {} \; &>/dev/null || true
+echo "Cleaning up CPython caches"
+find /usr/share/pvc/pvcnoded -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true
--- a/debian/pvc-daemon-worker.preinst
+++ b/debian/pvc-daemon-worker.preinst
@ -1,5 +1,5 @@
 #!/bin/sh

 # Remove any cached CPython directories or files
-echo "Cleaning up existing CPython files"
-find /usr/share/pvc/pvcworkerd -type d -name "__pycache__" -exec rm -rf {} \; &>/dev/null || true
+echo "Cleaning up CPython caches"
+find /usr/share/pvc/pvcworkerd -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true
--- a/debian/rules
+++ b/debian/rules
@ -13,7 +13,7 @@ override_dh_python3:
 	rm -r $(CURDIR)/client-cli/.pybuild $(CURDIR)/client-cli/pvc.egg-info

 override_dh_auto_clean:
-	find . -name "__pycache__" -o -name ".pybuild" -exec rm -r {} \; || true
+	find . -name "__pycache__" -o -name ".pybuild" -exec rm -fr {} + || true

 # If you need to rebuild the Sphinx documentation
 # Add spinxdoc to the dh --with line
--- a/health-daemon/pvchealthd/Daemon.py
+++ b/health-daemon/pvchealthd/Daemon.py
@ -33,7 +33,7 @@ import os
 import signal

 # Daemon version
-version = "0.9.89"
+version = "0.9.91"


 ##########################################################
--- a/node-daemon/pvcnoded/Daemon.py
+++ b/node-daemon/pvcnoded/Daemon.py
@ -49,7 +49,7 @@ import re
 import json

 # Daemon version
-version = "0.9.89"
+version = "0.9.91"


 ##########################################################
@ -197,7 +197,9 @@ def entrypoint():
        os.execv(sys.argv[0], sys.argv)

    # Validate the schema
-    pvcnoded.util.zookeeper.validate_schema(logger, zkhandler)
+    with zkhandler.writelock("base.schema.version"):
+        sleep(0.5)
+        pvcnoded.util.zookeeper.validate_schema(logger, zkhandler)

    # Define a cleanup function
    def cleanup(failure=False):
--- a/node-daemon/pvcnoded/util/zookeeper.py
+++ b/node-daemon/pvcnoded/util/zookeeper.py
@ -94,7 +94,10 @@ def validate_schema(logger, zkhandler):
    # Validate our schema against the active version
    if not zkhandler.schema.validate(zkhandler, logger):
        logger.out("Found schema violations, applying", state="i")
-        zkhandler.schema.apply(zkhandler)
+        try:
+            zkhandler.schema.apply(zkhandler)
+        except Exception as e:
+            logger.out(f"Failed to apply schema updates: {e}", state="w")
    else:
        logger.out("Schema successfully validated", state="o")

--- a/worker-daemon/pvcworkerd/Daemon.py
+++ b/worker-daemon/pvcworkerd/Daemon.py
@ -44,7 +44,7 @@ from daemon_lib.vmbuilder import (
 )

 # Daemon version
-version = "0.9.89"
+version = "0.9.91"


 config = cfg.get_configuration()
Author	SHA1	Message	Date
Joshua M. Boniface	f29b4c2755	Bump version to 0.9.91	2024-01-23 10:40:59 -05:00
Joshua M. Boniface	0adec2be0d	Use consistent and less error-prone find rm's	2024-01-23 10:40:48 -05:00
Joshua M. Boniface	b994e1a26c	Add cleanup of pycaches to CLI install	2024-01-23 10:22:50 -05:00
Joshua M. Boniface	6d6420a695	Add missing value to vm_define function	2024-01-23 09:58:32 -05:00
Joshua M. Boniface	94e0287fc4	Add missing modules to default cloud-init	2024-01-21 14:23:44 -05:00
Joshua M. Boniface	2886176762	Improve handling of task arg display Shows each subarg of the task_args as its own element, if applicable, and fits the width to the terminal using MAX_CONTENT_WIDTH instead of an arbitrary value.	2024-01-18 13:00:48 -05:00
Joshua M. Boniface	4dc4c975f1	Add status messages during task query	2024-01-18 12:38:53 -05:00
Joshua M. Boniface	8f3120baf3	Avoid errors if task_status is a tuple	2024-01-18 12:30:31 -05:00
Joshua M. Boniface	86ca363697	Bump version to 0.9.90	2024-01-11 10:22:48 -05:00
Joshua M. Boniface	a5763c9d25	Fix possible race condition applying schemas Found an instance where two of these fired too close together, and caused a fatal error. Use a write lock, and then catch the schema.apply function in case it fails anyways.	2024-01-11 10:21:01 -05:00
Joshua M. Boniface	39ec427c42	Remove obsolete swagger files	2024-01-11 10:20:42 -05:00
Joshua M. Boniface	1ba21312ea	Use flask manage for all versions The legacy method does not work any longer, so just use the Flask manage script and remove the obsolete legacy one.	2024-01-11 10:20:33 -05:00
 @ -1 +1 @@
 .9.89
 .9.91