From 560cb609bac3efb7415e3d43ba8d0d06382c21e5 Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Sat, 15 Feb 2020 22:10:34 -0500 Subject: [PATCH] Add database management with SQLAlchemy Add management of the pvcprov database with SQLAlchemy, to allow seamless management of the database. Add automatic tasks to the postinst of the API to execute these migrations. --- api-daemon/pvc-api-db-init | 17 ++++ api-daemon/pvc-api-db-migrate | 17 ++++ api-daemon/pvc-api-db-upgrade | 16 ++++ api-daemon/pvcapid-manage.py | 35 +++++++ api-daemon/pvcapid/flaskapi.py | 9 ++ api-daemon/pvcapid/models.py | 169 +++++++++++++++++++++++++++++++++ debian/control | 2 +- debian/pvc-daemon-api.install | 3 + debian/pvc-daemon-api.postinst | 19 ++-- 9 files changed, 274 insertions(+), 13 deletions(-) create mode 100755 api-daemon/pvc-api-db-init create mode 100755 api-daemon/pvc-api-db-migrate create mode 100755 api-daemon/pvc-api-db-upgrade create mode 100755 api-daemon/pvcapid-manage.py create mode 100755 api-daemon/pvcapid/models.py diff --git a/api-daemon/pvc-api-db-init b/api-daemon/pvc-api-db-init new file mode 100755 index 00000000..a5e69128 --- /dev/null +++ b/api-daemon/pvc-api-db-init @@ -0,0 +1,17 @@ +#!/bin/bash + +# Initialize the PVC database and migrations for future upgrades +# Part of the Parallel Virtual Cluster (PVC) system + +export PVC_CONFIG_FILE="/etc/pvc/pvcapid.yaml" + +if [[ ! -f ${PVC_CONFIG_FILE} ]]; then + echo "Create a configuration file at ${PVC_CONFIG_FILE} before initializing the database." + exit 1 +fi + +pushd /usr/share/pvc +./pvcapid-manage.py db init +./pvcapid-manage.py db migrate +./pvcapid-manage.py db upgrade +popd diff --git a/api-daemon/pvc-api-db-migrate b/api-daemon/pvc-api-db-migrate new file mode 100755 index 00000000..6d26b4d0 --- /dev/null +++ b/api-daemon/pvc-api-db-migrate @@ -0,0 +1,17 @@ +#!/bin/bash + +# Apply PVC database migrations +# Part of the Parallel Virtual Cluster (PVC) system + +PVC_CONFIG_FILE="/etc/pvc/pvcapid.yaml" +PVC_SHARE_DIR="/usr/share/pvc" + +if [[ ! -f ${PVC_CONFIG_FILE} ]]; then + echo "Create a configuration file at ${PVC_CONFIG_FILE} before upgrading the database." + exit 1 +fi + +pushd ${PVC_SHARE_DIR} +./pvcapid-manage.py db migrate +./pvcapid-manage.py db upgrade +popd diff --git a/api-daemon/pvc-api-db-upgrade b/api-daemon/pvc-api-db-upgrade new file mode 100755 index 00000000..6db47c10 --- /dev/null +++ b/api-daemon/pvc-api-db-upgrade @@ -0,0 +1,16 @@ +#!/bin/bash + +# Apply PVC database migrations +# Part of the Parallel Virtual Cluster (PVC) system + +export PVC_CONFIG_FILE="/etc/pvc/pvcapid.yaml" + +if [[ ! -f ${PVC_CONFIG_FILE} ]]; then + echo "Create a configuration file at ${PVC_CONFIG_FILE} before upgrading the database." + exit 1 +fi + +pushd /usr/share/pvc +./pvcapid-manage.py db migrate +./pvcapid-manage.py db upgrade +popd diff --git a/api-daemon/pvcapid-manage.py b/api-daemon/pvcapid-manage.py new file mode 100755 index 00000000..0de43342 --- /dev/null +++ b/api-daemon/pvcapid-manage.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +# manage.py - PVC Database management tasks +# Part of the Parallel Virtual Cluster (PVC) system +# +# Copyright (C) 2018-2020 Joshua M. Boniface +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### + +import os +from flask_migrate import Migrate, MigrateCommand +from flask_script import Manager + +from pvcapid.flaskapi import app, db, config + +migrate = Migrate(app, db) +manager = Manager(app) + +manager.add_command('db', MigrateCommand) + +if __name__ == '__main__': + manager.run() diff --git a/api-daemon/pvcapid/flaskapi.py b/api-daemon/pvcapid/flaskapi.py index f5acd6e0..28e179e9 100755 --- a/api-daemon/pvcapid/flaskapi.py +++ b/api-daemon/pvcapid/flaskapi.py @@ -41,6 +41,8 @@ import pvcapid.helper as api_helper import pvcapid.provisioner as api_provisioner import pvcapid.ova as api_ova +from flask_sqlalchemy import SQLAlchemy + API_VERSION = 1.0 # Parse the configuration file @@ -105,6 +107,7 @@ except Exception as e: app = flask.Flask(__name__) app.config['CELERY_BROKER_URL'] = 'redis://{}:{}{}'.format(config['queue_host'], config['queue_port'], config['queue_path']) app.config['CELERY_RESULT_BACKEND'] = 'redis://{}:{}{}'.format(config['queue_host'], config['queue_port'], config['queue_path']) +app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://{}:{}@{}:{}/{}'.format(config['database_user'], config['database_password'], config['database_host'], config['database_port'], config['database_name']) if config['debug']: app.config['DEBUG'] = True @@ -112,6 +115,12 @@ if config['debug']: if config['auth_enabled']: app.config["SECRET_KEY"] = config['auth_secret_key'] +# Create SQLAlchemy database +db = SQLAlchemy(app) + +# Import database models +from pvcapid.models import DBSystemTemplate, DBNetworkTemplate, DBNetworkElement, DBStorageTemplate, DBStorageElement, DBUserdata, DBScript, DBProfile + # Create Flask blueprint blueprint = flask.Blueprint('api', __name__, url_prefix='/api/v1') diff --git a/api-daemon/pvcapid/models.py b/api-daemon/pvcapid/models.py new file mode 100755 index 00000000..483f9f72 --- /dev/null +++ b/api-daemon/pvcapid/models.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 + +# models.py - PVC Database models +# Part of the Parallel Virtual Cluster (PVC) system +# +# Copyright (C) 2018-2020 Joshua M. Boniface +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### + +from pvcapid.flaskapi import app, db + +class DBSystemTemplate(db.Model): + __tablename__ = 'system_template' + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text, nullable=False, unique=True) + vcpu_count = db.Column(db.Integer, nullable=False) + vram_mb = db.Column(db.Integer, nullable=False) + serial = db.Column(db.Boolean, nullable=False) + vnc = db.Column(db.Boolean, nullable=False) + vnc_bind = db.Column(db.Text) + node_limit = db.Column(db.Text) + node_selector = db.Column(db.Text) + node_autostart = db.Column(db.Boolean, nullable=False) + + def __init__(self, name, vcpu_count, vram_mb, serial, vnc, vnc_bind, node_limit, node_selector, node_autostart): + self.name = name + self.vcpu_count = vcpu_count + self.vram_mb = vram_mb + self.serial = serial + self.vnc = vnc + self.vnc_bind = vnc_bind + self.node_limit = node_limit + self.node_selector = node_selector + self.node_autostart = node_autostart + + def __repr__(self): + return ''.format(self.id) + +class DBNetworkTemplate(db.Model): + __tablename__ = 'network_template' + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text, nullable=False, unique=True) + mac_template = db.Column(db.Text) + + def __init__(self, name, mac_template): + self.name = name + self.mac_template = mac_template + + def __repr__(self): + return ''.format(self.id) + +class DBNetworkElement(db.Model): + __tablename__ = 'network' + + id = db.Column(db.Integer, primary_key=True) + network_template = db.Column(db.Integer, db.ForeignKey("network_template.id")) + vni = db.Column(db.Integer, nullable=False) + + def __init__(self, network_template, vni): + self.network_template = network_template + self.vni = vni + + def __repr__(self): + return ''.format(self.id) + +class DBStorageTemplate(db.Model): + __tablename__ = 'storage_template' + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text, nullable=False, unique=True) + + def __init__(self, name): + self.name = name + + def __repr__(self): + return ''.format(self.id) + +class DBStorageElement(db.Model): + __tablename__ = 'storage' + + id = db.Column(db.Integer, primary_key=True) + storage_template = db.Column(db.Integer, db.ForeignKey("storage_template.id")) + pool = db.Column(db.Text, nullable=False) + disk_id = db.Column(db.Text, nullable=False) + source_volume = db.Column(db.Text) + disk_size_gb = db.Column(db.Integer) + mountpoint = db.Column(db.Text) + filesystem = db.Column(db.Text) + filesystem_args = db.Column(db.Text) + + def __init__(self, storage_template, pool, disk_id, source_volume, disk_size_gb, mountpoint, filesystem, filesystem_args): + self.storage_template = storage_template + self.pool = pool + self.disk_id = disk_id + self.source_volume = source_volume + self.disk_size_gb = disk_size_gb + self.mountpoint = mountpoint + self.filesystem = filesystem + self.filesystem_args = filesystem_args + + def __repr__(self): + return ''.format(self.id) + +class DBUserdata(db.Model): + __tablename__ = 'userdata' + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text, nullable=False, unique=True) + userdata = db.Column(db.Text, nullable=False) + + def __init__(self, name, userdata): + self.name = name + self.userdata = userdata + + def __repr__(self): + return ''.format(self.id) + +class DBScript(db.Model): + __tablename__ = 'script' + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text, nullable=False, unique=True) + script = db.Column(db.Text, nullable=False) + + def __init__(self, name, script): + self.name = name + self.script = script + + def __repr__(self): + return ''.format(self.id) + +class DBProfile(db.Model): + __tablename__ = 'profile' + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text, nullable=False, unique=True) + system_template = db.Column(db.Integer, db.ForeignKey("system_template.id")) + network_template = db.Column(db.Integer, db.ForeignKey("network_template.id")) + storage_template = db.Column(db.Integer, db.ForeignKey("storage_template.id")) + userdata = db.Column(db.Integer, db.ForeignKey("userdata.id")) + script = db.Column(db.Integer, db.ForeignKey("script.id")) + arguments = db.Column(db.Text) + + def __init__(self, name, system_template, network_template, storage_template, userdata, script, arguments): + self.name = name + self.system_template = system_template + self.network_template = network_template + self.storage_template = storage_template + self.userdata = userdata + self.script = script + self.arguments = arguments + + def __repr__(self): + return ''.format(self.id) diff --git a/debian/control b/debian/control index db10f342..d48c4fa5 100644 --- a/debian/control +++ b/debian/control @@ -17,7 +17,7 @@ Description: Parallel Virtual Cluster node daemon (Python 3) Package: pvc-daemon-api Architecture: all -Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-gevent, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml +Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-gevent, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml, python3-flask-migrate, python3-flask-script Description: Parallel Virtual Cluster API daemon (Python 3) A KVM/Zookeeper/Ceph-based VM and private cloud manager . diff --git a/debian/pvc-daemon-api.install b/debian/pvc-daemon-api.install index 52e55633..57054cbf 100644 --- a/debian/pvc-daemon-api.install +++ b/debian/pvc-daemon-api.install @@ -1,4 +1,7 @@ api-daemon/pvcapid.py usr/share/pvc +api-daemon/pvcapid-manage.py usr/share/pvc +api-daemon/pvc-api-db-init usr/share/pvc +api-daemon/pvc-api-db-upgrade usr/share/pvc api-daemon/pvcapid.sample.yaml etc/pvc api-daemon/pvcapid usr/share/pvc api-daemon/pvcapid.service lib/systemd/system diff --git a/debian/pvc-daemon-api.postinst b/debian/pvc-daemon-api.postinst index 423fdb6a..6d35748e 100644 --- a/debian/pvc-daemon-api.postinst +++ b/debian/pvc-daemon-api.postinst @@ -1,20 +1,15 @@ #!/bin/sh -# Install client binary to /usr/bin via symlink -ln -s /usr/share/pvc/api.py /usr/bin/pvcapid - # Reload systemd's view of the units systemctl daemon-reload -# Restart the main daemon (or warn on first install) +# Restart the main daemon and apply database migrations (or warn on first install) if systemctl is-active --quiet pvcapid.service; then - systemctl restart pvcapid.service + systemctl stop pvcapid-worker.service + systemctl stop pvcapid.service + /usr/share/pvc/pvc-api-db-upgrade + systemctl start pvcapid.service + systemctl start pvcapid-worker.service else - echo "NOTE: The PVC client API daemon (pvcapid.service) has not been started; create a config file at /etc/pvc/pvcapid.yaml then start it." -fi -# Restart the worker daemon (or warn on first install) -if systemctl is-active --quiet pvcapid-worker.service; then - systemctl restart pvcapid-worker.service -else - echo "NOTE: The PVC provisioner worker daemon (pvcapid-worker.service) has not been started; create a config file at /etc/pvc/pvcapid.yaml then start it." + echo "NOTE: The PVC client API daemon (pvcapid.service) and the PVC provisioner worker daemon (pvcapid-worker.service) have not been started; create a config file at /etc/pvc/pvcapid.yaml, then run the database configuration (/usr/share/pvc/api-db-init) and start them manually." fi