Compare commits

...

12 Commits

Author SHA1 Message Date
f29b4c2755 Bump version to 0.9.91 2024-01-23 10:40:59 -05:00
0adec2be0d Use consistent and less error-prone find rm's 2024-01-23 10:40:48 -05:00
b994e1a26c Add cleanup of pycaches to CLI install 2024-01-23 10:22:50 -05:00
6d6420a695 Add missing value to vm_define function 2024-01-23 09:58:32 -05:00
94e0287fc4 Add missing modules to default cloud-init 2024-01-21 14:23:44 -05:00
2886176762 Improve handling of task arg display
Shows each subarg of the task_args as its own element, if applicable,
and fits the width to the terminal using MAX_CONTENT_WIDTH instead of an
arbitrary value.
2024-01-18 13:00:48 -05:00
4dc4c975f1 Add status messages during task query 2024-01-18 12:38:53 -05:00
8f3120baf3 Avoid errors if task_status is a tuple 2024-01-18 12:30:31 -05:00
86ca363697 Bump version to 0.9.90 2024-01-11 10:22:48 -05:00
a5763c9d25 Fix possible race condition applying schemas
Found an instance where two of these fired too close together, and
caused a fatal error. Use a write lock, and then catch the schema.apply
function in case it fails anyways.
2024-01-11 10:21:01 -05:00
39ec427c42 Remove obsolete swagger files 2024-01-11 10:20:42 -05:00
1ba21312ea Use flask manage for all versions
The legacy method does not work any longer, so just use the Flask manage
script and remove the obsolete legacy one.
2024-01-11 10:20:33 -05:00
26 changed files with 122 additions and 4833 deletions

View File

@ -1 +1 @@
0.9.89
0.9.91

View File

@ -1,5 +1,20 @@
## PVC Changelog
###### [v0.9.91](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.91)
* [Client CLI] Fixes a bug and improves output during cluster task events.
* [Client CLI] Improves the output of the task list display.
* [Provisioner] Fixes some missing cloud-init modules in the default debootstrap script.
* [Client CLI] Fixes a bug with a missing argument to the vm_define helper function.
* [All] Fixes inconsistent package find + rm commands to avoid errors in dpkg.
###### [v0.9.90](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.90)
* [Client CLI/API Daemon] Adds additional backup metainfo and an emailed report option to autobackups.
* [All] Adds a live migration maximum downtime selector to help with busy VM migrations.
* [API Daemon] Fixes a database migration bug on Debian 10/11.
* [Node Daemon] Fixes a race condition when applying Zookeeper schema changes.
###### [v0.9.89](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.89)
* [API/Worker Daemons] Fixes a bug with the Celery result backends not being properly initialized on Debian 10/11.

View File

@ -589,11 +589,13 @@ After=multi-user.target
- migrator
- bootcmd
- write-files
- growpart
- resizefs
- set_hostname
- update_hostname
- update_etc_hosts
- ca-certs
- users-groups
- ssh
cloud_config_modules:

View File

@ -12,16 +12,7 @@ fi
pushd /usr/share/pvc
case "$( cat /etc/debian_version )" in
10.*|11.*)
# Debian 10 & 11
./pvcapid-manage_legacy.py db upgrade
;;
*)
# Debian 12+
export FLASK_APP=./pvcapid-manage_flask.py
flask db upgrade
;;
esac
export FLASK_APP=./pvcapid-manage-flask.py
flask db upgrade
popd

View File

@ -1,33 +0,0 @@
#!/usr/bin/env python3
# pvcapid-manage_legacy.py - PVC Database management tasks (Legacy)
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2024 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
from flask_migrate import Migrate, MigrateCommand, Manager
from pvcapid.flaskapi import app, db
from pvcapid.models import * # noqa F401,F403
migrate = Migrate(app, db)
manager = Manager(app)
manager.add_command("db", MigrateCommand)
if __name__ == "__main__":
manager.run()

View File

@ -27,7 +27,7 @@ from distutils.util import strtobool as dustrtobool
import daemon_lib.config as cfg
# Daemon version
version = "0.9.89"
version = "0.9.91"
# API version
API_VERSION = 1.0

View File

@ -1,13 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>PVC Client API Documentation</title>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style> body { margin: 0; padding: 0; } </style>
</head>
<body>
<redoc spec-url='./swagger.json' hide-loading></redoc>
<script src="https://rebilly.github.io/ReDoc/releases/latest/redoc.min.js"> </script>
</body>
</html>

File diff suppressed because it is too large Load Diff

View File

@ -687,7 +687,10 @@ def cli_cluster_task(task_id, wait_flag, format_function):
if wait_flag:
# First validate that this is actually a valid task that is running
echo(CLI_CONFIG, "Querying cluster for tasks...", newline=False)
retcode, retdata = pvc.lib.common.task_status(CLI_CONFIG, None)
echo(CLI_CONFIG, " done.")
echo(CLI_CONFIG, "")
if task_id in [i["id"] for i in retdata]:
task = [i for i in retdata if i["id"] == task_id][0]
retmsg = wait_for_celery_task(
@ -699,7 +702,10 @@ def cli_cluster_task(task_id, wait_flag, format_function):
retmsg = f"No task with ID {task_id} found."
finish(retcode, retmsg)
else:
echo(CLI_CONFIG, "Querying cluster for tasks...", newline=False)
retcode, retdata = pvc.lib.common.task_status(CLI_CONFIG, task_id)
echo(CLI_CONFIG, " done.")
echo(CLI_CONFIG, "")
finish(retcode, retdata, format_function)

View File

@ -645,6 +645,24 @@ def cli_cluster_task_format_pretty(CLI_CONFIG, task_data):
if _task_type_length > task_type_length:
task_type_length = _task_type_length
for arg_name, arg_data in task["kwargs"].items():
# Skip the "run_on" argument
if arg_name == "run_on":
continue
# task_arg_name column
_task_arg_name_length = len(str(arg_name)) + 1
if _task_arg_name_length > task_arg_name_length:
task_arg_name_length = _task_arg_name_length
task_header_length = (
task_id_length + task_name_length + task_type_length + task_worker_length + 3
)
max_task_data_length = (
MAX_CONTENT_WIDTH - task_header_length - task_arg_name_length - 2
)
for task in task_data:
updated_kwargs = list()
for arg_name, arg_data in task["kwargs"].items():
# Skip the "run_on" argument
@ -656,15 +674,30 @@ def cli_cluster_task_format_pretty(CLI_CONFIG, task_data):
if _task_arg_name_length > task_arg_name_length:
task_arg_name_length = _task_arg_name_length
if len(str(arg_data)) > 17:
arg_data = arg_data[:17] + "..."
if isinstance(arg_data, list):
for subarg_data in arg_data:
if len(subarg_data) > max_task_data_length:
subarg_data = (
str(subarg_data[: max_task_data_length - 4]) + " ..."
)
# task_arg_data column
_task_arg_data_length = len(str(arg_data)) + 1
if _task_arg_data_length > task_arg_data_length:
task_arg_data_length = _task_arg_data_length
# task_arg_data column
_task_arg_data_length = len(str(subarg_data)) + 1
if _task_arg_data_length > task_arg_data_length:
task_arg_data_length = _task_arg_data_length
updated_kwargs.append({"name": arg_name, "data": subarg_data})
else:
if len(str(arg_data)) > 24:
arg_data = str(arg_data[:24]) + " ..."
# task_arg_data column
_task_arg_data_length = len(str(arg_data)) + 1
if _task_arg_data_length > task_arg_data_length:
task_arg_data_length = _task_arg_data_length
updated_kwargs.append({"name": arg_name, "data": arg_data})
updated_kwargs.append({"name": arg_name, "data": arg_data})
task["kwargs"] = updated_kwargs
tasks.append(task)

View File

@ -115,6 +115,8 @@ def wait_for_celery_task(CLI_CONFIG, task_detail, start_late=False):
)
while True:
sleep(0.5)
if isinstance(task_status, tuple):
continue
if task_status.get("state") != "RUNNING":
break
if task_status.get("current") > last_task:

View File

@ -89,6 +89,7 @@ def vm_define(
node_selector,
node_autostart,
migration_method,
migration_max_downtime,
user_tags,
protected_tags,
):
@ -96,7 +97,7 @@ def vm_define(
Define a new VM on the cluster
API endpoint: POST /vm
API arguments: xml={xml}, node={node}, limit={node_limit}, selector={node_selector}, autostart={node_autostart}, migration_method={migration_method}, user_tags={user_tags}, protected_tags={protected_tags}
API arguments: xml={xml}, node={node}, limit={node_limit}, selector={node_selector}, autostart={node_autostart}, migration_method={migration_method}, migration_max_downtime={migration_max_downtime}, user_tags={user_tags}, protected_tags={protected_tags}
API schema: {"message":"{data}"}
"""
params = {
@ -105,6 +106,7 @@ def vm_define(
"selector": node_selector,
"autostart": node_autostart,
"migration_method": migration_method,
"migration_max_downtime": migration_max_downtime,
"user_tags": user_tags,
"protected_tags": protected_tags,
}

View File

@ -2,7 +2,7 @@ from setuptools import setup
setup(
name="pvc",
version="0.9.89",
version="0.9.91",
packages=["pvc.cli", "pvc.lib"],
install_requires=[
"Click",

19
debian/changelog vendored
View File

@ -1,3 +1,22 @@
pvc (0.9.91-0) unstable; urgency=high
* [Client CLI] Fixes a bug and improves output during cluster task events.
* [Client CLI] Improves the output of the task list display.
* [Provisioner] Fixes some missing cloud-init modules in the default debootstrap script.
* [Client CLI] Fixes a bug with a missing argument to the vm_define helper function.
* [All] Fixes inconsistent package find + rm commands to avoid errors in dpkg.
-- Joshua M. Boniface <joshua@boniface.me> Tue, 23 Jan 2024 10:02:19 -0500
pvc (0.9.90-0) unstable; urgency=high
* [Client CLI/API Daemon] Adds additional backup metainfo and an emailed report option to autobackups.
* [All] Adds a live migration maximum downtime selector to help with busy VM migrations.
* [API Daemon] Fixes a database migration bug on Debian 10/11.
* [Node Daemon] Fixes a race condition when applying Zookeeper schema changes.
-- Joshua M. Boniface <joshua@boniface.me> Thu, 11 Jan 2024 00:14:49 -0500
pvc (0.9.89-0) unstable; urgency=high
* [API/Worker Daemons] Fixes a bug with the Celery result backends not being properly initialized on Debian 10/11.

View File

@ -2,7 +2,12 @@
# Generate the bash completion configuration
if [ -d /etc/bash_completion.d ]; then
echo "Installing BASH completion configuration"
_PVC_COMPLETE=source_bash pvc > /etc/bash_completion.d/pvc
fi
# Remove any cached CPython directories or files
echo "Cleaning up CPython caches"
find /usr/lib/python3/dist-packages/pvc -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true
exit 0

View File

@ -1,5 +1,5 @@
#!/bin/sh
# Remove any cached CPython directories or files
echo "Cleaning up existing CPython files"
find /usr/share/pvc/pvcapid -type d -name "__pycache__" -exec rm -rf {} \; &>/dev/null || true
echo "Cleaning up CPython caches"
find /usr/share/pvc/pvcapid -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true

5
debian/pvc-daemon-common.preinst vendored Normal file
View File

@ -0,0 +1,5 @@
#!/bin/sh
# Remove any cached CPython directories or files
echo "Cleaning up CPython caches"
find /usr/share/pvc/daemon_lib -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true

View File

@ -1,6 +1,6 @@
#!/bin/sh
# Remove any cached CPython directories or files
echo "Cleaning up existing CPython files"
find /usr/share/pvc/pvchealthd -type d -name "__pycache__" -exec rm -rf {} \; &>/dev/null || true
find /usr/share/pvc/plugins -type d -name "__pycache__" -exec rm -rf {} \; &>/dev/null || true
echo "Cleaning up CPython caches"
find /usr/share/pvc/pvchealthd -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true
find /usr/share/pvc/plugins -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true

View File

@ -1,5 +1,5 @@
#!/bin/sh
# Remove any cached CPython directories or files
echo "Cleaning up existing CPython files"
find /usr/share/pvc/pvcnoded -type d -name "__pycache__" -exec rm -rf {} \; &>/dev/null || true
echo "Cleaning up CPython caches"
find /usr/share/pvc/pvcnoded -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true

View File

@ -1,5 +1,5 @@
#!/bin/sh
# Remove any cached CPython directories or files
echo "Cleaning up existing CPython files"
find /usr/share/pvc/pvcworkerd -type d -name "__pycache__" -exec rm -rf {} \; &>/dev/null || true
echo "Cleaning up CPython caches"
find /usr/share/pvc/pvcworkerd -type d -name "__pycache__" -exec rm -fr {} + &>/dev/null || true

2
debian/rules vendored
View File

@ -13,7 +13,7 @@ override_dh_python3:
rm -r $(CURDIR)/client-cli/.pybuild $(CURDIR)/client-cli/pvc.egg-info
override_dh_auto_clean:
find . -name "__pycache__" -o -name ".pybuild" -exec rm -r {} \; || true
find . -name "__pycache__" -o -name ".pybuild" -exec rm -fr {} + || true
# If you need to rebuild the Sphinx documentation
# Add spinxdoc to the dh --with line

View File

@ -33,7 +33,7 @@ import os
import signal
# Daemon version
version = "0.9.89"
version = "0.9.91"
##########################################################

View File

@ -49,7 +49,7 @@ import re
import json
# Daemon version
version = "0.9.89"
version = "0.9.91"
##########################################################
@ -197,7 +197,9 @@ def entrypoint():
os.execv(sys.argv[0], sys.argv)
# Validate the schema
pvcnoded.util.zookeeper.validate_schema(logger, zkhandler)
with zkhandler.writelock("base.schema.version"):
sleep(0.5)
pvcnoded.util.zookeeper.validate_schema(logger, zkhandler)
# Define a cleanup function
def cleanup(failure=False):

View File

@ -94,7 +94,10 @@ def validate_schema(logger, zkhandler):
# Validate our schema against the active version
if not zkhandler.schema.validate(zkhandler, logger):
logger.out("Found schema violations, applying", state="i")
zkhandler.schema.apply(zkhandler)
try:
zkhandler.schema.apply(zkhandler)
except Exception as e:
logger.out(f"Failed to apply schema updates: {e}", state="w")
else:
logger.out("Schema successfully validated", state="o")

View File

@ -44,7 +44,7 @@ from daemon_lib.vmbuilder import (
)
# Daemon version
version = "0.9.89"
version = "0.9.91"
config = cfg.get_configuration()