Add database management with SQLAlchemy

Add management of the pvcprov database with SQLAlchemy, to allow
seamless management of the database. Add automatic tasks to the postinst
of the API to execute these migrations.

api-daemon/pvc-api-db-init

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Initialize the PVC database and migrations for future upgrades
+# Part of the Parallel Virtual Cluster (PVC) system
+
+export PVC_CONFIG_FILE="/etc/pvc/pvcapid.yaml"
+
+if [[ ! -f ${PVC_CONFIG_FILE} ]]; then
+    echo "Create a configuration file at ${PVC_CONFIG_FILE} before initializing the database."
+    exit 1
+fi
+
+pushd /usr/share/pvc
+./pvcapid-manage.py db init
+./pvcapid-manage.py db migrate
+./pvcapid-manage.py db upgrade
+popd

api-daemon/pvc-api-db-migrate

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Apply PVC database migrations
+# Part of the Parallel Virtual Cluster (PVC) system
+
+PVC_CONFIG_FILE="/etc/pvc/pvcapid.yaml"
+PVC_SHARE_DIR="/usr/share/pvc"
+
+if [[ ! -f ${PVC_CONFIG_FILE} ]]; then
+    echo "Create a configuration file at ${PVC_CONFIG_FILE} before upgrading the database."
+    exit 1
+fi
+
+pushd ${PVC_SHARE_DIR}
+./pvcapid-manage.py db migrate
+./pvcapid-manage.py db upgrade
+popd

api-daemon/pvc-api-db-upgrade

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Apply PVC database migrations
+# Part of the Parallel Virtual Cluster (PVC) system
+
+export PVC_CONFIG_FILE="/etc/pvc/pvcapid.yaml"
+
+if [[ ! -f ${PVC_CONFIG_FILE} ]]; then
+    echo "Create a configuration file at ${PVC_CONFIG_FILE} before upgrading the database."
+    exit 1
+fi
+
+pushd /usr/share/pvc
+./pvcapid-manage.py db migrate
+./pvcapid-manage.py db upgrade
+popd

api-daemon/pvcapid-manage.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+# manage.py - PVC Database management tasks
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import os
+from flask_migrate import Migrate, MigrateCommand
+from flask_script import Manager
+
+from pvcapid.flaskapi import app, db, config
+
+migrate = Migrate(app, db)
+manager = Manager(app)
+
+manager.add_command('db', MigrateCommand)	
+
+if __name__ == '__main__':
+    manager.run()

api-daemon/pvcapid/flaskapi.py

@@ -41,6 +41,8 @@ import pvcapid.helper as api_helper
 import pvcapid.provisioner as api_provisioner
 import pvcapid.ova as api_ova
 
+from flask_sqlalchemy import SQLAlchemy
+
 API_VERSION = 1.0
 
 # Parse the configuration file
@@ -105,6 +107,7 @@ except Exception as e:
 app = flask.Flask(__name__)
 app.config['CELERY_BROKER_URL'] = 'redis://{}:{}{}'.format(config['queue_host'], config['queue_port'], config['queue_path'])
 app.config['CELERY_RESULT_BACKEND'] = 'redis://{}:{}{}'.format(config['queue_host'], config['queue_port'], config['queue_path'])
+app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://{}:{}@{}:{}/{}'.format(config['database_user'], config['database_password'], config['database_host'], config['database_port'], config['database_name'])
 
 if config['debug']:
     app.config['DEBUG'] = True
@@ -112,6 +115,12 @@ if config['debug']:
 if config['auth_enabled']:
     app.config["SECRET_KEY"] = config['auth_secret_key']
 
+# Create SQLAlchemy database
+db = SQLAlchemy(app)
+
+# Import database models
+from pvcapid.models import DBSystemTemplate, DBNetworkTemplate, DBNetworkElement, DBStorageTemplate, DBStorageElement, DBUserdata, DBScript, DBProfile
+
 # Create Flask blueprint
 blueprint = flask.Blueprint('api', __name__, url_prefix='/api/v1')
 

api-daemon/pvcapid/models.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+
+# models.py - PVC Database models
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from pvcapid.flaskapi import app, db
+
+class DBSystemTemplate(db.Model):
+    __tablename__ = 'system_template'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.Text, nullable=False, unique=True)
+    vcpu_count = db.Column(db.Integer, nullable=False)
+    vram_mb = db.Column(db.Integer, nullable=False)
+    serial = db.Column(db.Boolean, nullable=False)
+    vnc = db.Column(db.Boolean, nullable=False)
+    vnc_bind = db.Column(db.Text)
+    node_limit = db.Column(db.Text)
+    node_selector = db.Column(db.Text)
+    node_autostart = db.Column(db.Boolean, nullable=False)
+
+    def __init__(self, name, vcpu_count, vram_mb, serial, vnc, vnc_bind, node_limit, node_selector, node_autostart):
+        self.name = name
+        self.vcpu_count = vcpu_count
+        self.vram_mb = vram_mb
+        self.serial = serial
+        self.vnc = vnc
+        self.vnc_bind = vnc_bind
+        self.node_limit = node_limit
+        self.node_selector = node_selector
+        self.node_autostart = node_autostart
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+class DBNetworkTemplate(db.Model):
+    __tablename__ = 'network_template'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.Text, nullable=False, unique=True)
+    mac_template = db.Column(db.Text)
+
+    def __init__(self, name, mac_template):
+        self.name = name
+        self.mac_template = mac_template
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+class DBNetworkElement(db.Model):
+    __tablename__ = 'network'
+
+    id = db.Column(db.Integer, primary_key=True)
+    network_template = db.Column(db.Integer, db.ForeignKey("network_template.id"))
+    vni = db.Column(db.Integer, nullable=False)
+
+    def __init__(self, network_template, vni):
+        self.network_template = network_template
+        self.vni = vni
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+class DBStorageTemplate(db.Model):
+    __tablename__ = 'storage_template'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.Text, nullable=False, unique=True)
+
+    def __init__(self, name):
+        self.name = name
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+class DBStorageElement(db.Model):
+    __tablename__ = 'storage'
+
+    id = db.Column(db.Integer, primary_key=True)
+    storage_template = db.Column(db.Integer, db.ForeignKey("storage_template.id"))
+    pool = db.Column(db.Text, nullable=False)
+    disk_id = db.Column(db.Text, nullable=False)
+    source_volume = db.Column(db.Text)
+    disk_size_gb = db.Column(db.Integer)
+    mountpoint = db.Column(db.Text)
+    filesystem = db.Column(db.Text)
+    filesystem_args = db.Column(db.Text)
+
+    def __init__(self, storage_template, pool, disk_id, source_volume, disk_size_gb, mountpoint, filesystem, filesystem_args):
+        self.storage_template = storage_template
+        self.pool = pool
+        self.disk_id = disk_id
+        self.source_volume = source_volume
+        self.disk_size_gb = disk_size_gb
+        self.mountpoint = mountpoint
+        self.filesystem = filesystem
+        self.filesystem_args = filesystem_args
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+class DBUserdata(db.Model):
+    __tablename__ = 'userdata'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.Text, nullable=False, unique=True)
+    userdata = db.Column(db.Text, nullable=False)
+
+    def __init__(self, name, userdata):
+        self.name = name
+        self.userdata = userdata
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+class DBScript(db.Model):
+    __tablename__ = 'script'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.Text, nullable=False, unique=True)
+    script = db.Column(db.Text, nullable=False)
+
+    def __init__(self, name, script):
+        self.name = name
+        self.script = script
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+class DBProfile(db.Model):
+    __tablename__ = 'profile'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.Text, nullable=False, unique=True)
+    system_template = db.Column(db.Integer, db.ForeignKey("system_template.id"))
+    network_template = db.Column(db.Integer, db.ForeignKey("network_template.id"))
+    storage_template = db.Column(db.Integer, db.ForeignKey("storage_template.id"))
+    userdata = db.Column(db.Integer, db.ForeignKey("userdata.id"))
+    script = db.Column(db.Integer, db.ForeignKey("script.id"))
+    arguments = db.Column(db.Text)
+
+    def __init__(self, name, system_template, network_template, storage_template, userdata, script, arguments):
+        self.name = name
+        self.system_template = system_template
+        self.network_template = network_template
+        self.storage_template = storage_template
+        self.userdata = userdata
+        self.script = script
+        self.arguments = arguments
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)

debian/control

@@ -17,7 +17,7 @@ Description: Parallel Virtual Cluster node daemon (Python 3)
 
 Package: pvc-daemon-api
 Architecture: all
-Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-gevent, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml
+Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-gevent, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml, python3-flask-migrate, python3-flask-script
 Description: Parallel Virtual Cluster API daemon (Python 3)
  A KVM/Zookeeper/Ceph-based VM and private cloud manager
  .

debian/pvc-daemon-api.install

@@ -1,4 +1,7 @@
 api-daemon/pvcapid.py usr/share/pvc
+api-daemon/pvcapid-manage.py usr/share/pvc
+api-daemon/pvc-api-db-init usr/share/pvc
+api-daemon/pvc-api-db-upgrade usr/share/pvc
 api-daemon/pvcapid.sample.yaml etc/pvc
 api-daemon/pvcapid usr/share/pvc
 api-daemon/pvcapid.service lib/systemd/system

debian/pvc-daemon-api.postinst

@@ -1,20 +1,15 @@
 #!/bin/sh
 
-# Install client binary to /usr/bin via symlink
-ln -s /usr/share/pvc/api.py /usr/bin/pvcapid
-
 # Reload systemd's view of the units
 systemctl daemon-reload
 
-# Restart the main daemon (or warn on first install)
+# Restart the main daemon and apply database migrations (or warn on first install)
 if systemctl is-active --quiet pvcapid.service; then
-    systemctl restart pvcapid.service
+    systemctl stop pvcapid-worker.service
+    systemctl stop pvcapid.service
+    /usr/share/pvc/pvc-api-db-upgrade
+    systemctl start pvcapid.service
+    systemctl start pvcapid-worker.service
 else
-    echo "NOTE: The PVC client API daemon (pvcapid.service) has not been started; create a config file at /etc/pvc/pvcapid.yaml then start it."
+    echo "NOTE: The PVC client API daemon (pvcapid.service) and the PVC provisioner worker daemon (pvcapid-worker.service) have not been started; create a config file at /etc/pvc/pvcapid.yaml, then run the database configuration (/usr/share/pvc/api-db-init) and start them manually."
-fi
-# Restart the worker daemon (or warn on first install)
-if systemctl is-active --quiet pvcapid-worker.service; then
-    systemctl restart pvcapid-worker.service
-else
-    echo "NOTE: The PVC provisioner worker daemon (pvcapid-worker.service) has not been started; create a config file at /etc/pvc/pvcapid.yaml then start it."
 fi