Add database management with SQLAlchemy

Add management of the pvcprov database with SQLAlchemy, to allow
seamless management of the database. Add automatic tasks to the postinst
of the API to execute these migrations.
This commit is contained in:
Joshua Boniface 2020-02-15 22:10:34 -05:00
parent 670596ed8e
commit 560cb609ba
9 changed files with 274 additions and 13 deletions

17
api-daemon/pvc-api-db-init Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
# Initialize the PVC database and migrations for future upgrades
# Part of the Parallel Virtual Cluster (PVC) system
export PVC_CONFIG_FILE="/etc/pvc/pvcapid.yaml"
if [[ ! -f ${PVC_CONFIG_FILE} ]]; then
echo "Create a configuration file at ${PVC_CONFIG_FILE} before initializing the database."
exit 1
fi
pushd /usr/share/pvc
./pvcapid-manage.py db init
./pvcapid-manage.py db migrate
./pvcapid-manage.py db upgrade
popd

17
api-daemon/pvc-api-db-migrate Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
# Apply PVC database migrations
# Part of the Parallel Virtual Cluster (PVC) system
PVC_CONFIG_FILE="/etc/pvc/pvcapid.yaml"
PVC_SHARE_DIR="/usr/share/pvc"
if [[ ! -f ${PVC_CONFIG_FILE} ]]; then
echo "Create a configuration file at ${PVC_CONFIG_FILE} before upgrading the database."
exit 1
fi
pushd ${PVC_SHARE_DIR}
./pvcapid-manage.py db migrate
./pvcapid-manage.py db upgrade
popd

16
api-daemon/pvc-api-db-upgrade Executable file
View File

@ -0,0 +1,16 @@
#!/bin/bash
# Apply PVC database migrations
# Part of the Parallel Virtual Cluster (PVC) system
export PVC_CONFIG_FILE="/etc/pvc/pvcapid.yaml"
if [[ ! -f ${PVC_CONFIG_FILE} ]]; then
echo "Create a configuration file at ${PVC_CONFIG_FILE} before upgrading the database."
exit 1
fi
pushd /usr/share/pvc
./pvcapid-manage.py db migrate
./pvcapid-manage.py db upgrade
popd

35
api-daemon/pvcapid-manage.py Executable file
View File

@ -0,0 +1,35 @@
#!/usr/bin/env python3
# manage.py - PVC Database management tasks
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
import os
from flask_migrate import Migrate, MigrateCommand
from flask_script import Manager
from pvcapid.flaskapi import app, db, config
migrate = Migrate(app, db)
manager = Manager(app)
manager.add_command('db', MigrateCommand)
if __name__ == '__main__':
manager.run()

View File

@ -41,6 +41,8 @@ import pvcapid.helper as api_helper
import pvcapid.provisioner as api_provisioner
import pvcapid.ova as api_ova
from flask_sqlalchemy import SQLAlchemy
API_VERSION = 1.0
# Parse the configuration file
@ -105,6 +107,7 @@ except Exception as e:
app = flask.Flask(__name__)
app.config['CELERY_BROKER_URL'] = 'redis://{}:{}{}'.format(config['queue_host'], config['queue_port'], config['queue_path'])
app.config['CELERY_RESULT_BACKEND'] = 'redis://{}:{}{}'.format(config['queue_host'], config['queue_port'], config['queue_path'])
app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://{}:{}@{}:{}/{}'.format(config['database_user'], config['database_password'], config['database_host'], config['database_port'], config['database_name'])
if config['debug']:
app.config['DEBUG'] = True
@ -112,6 +115,12 @@ if config['debug']:
if config['auth_enabled']:
app.config["SECRET_KEY"] = config['auth_secret_key']
# Create SQLAlchemy database
db = SQLAlchemy(app)
# Import database models
from pvcapid.models import DBSystemTemplate, DBNetworkTemplate, DBNetworkElement, DBStorageTemplate, DBStorageElement, DBUserdata, DBScript, DBProfile
# Create Flask blueprint
blueprint = flask.Blueprint('api', __name__, url_prefix='/api/v1')

169
api-daemon/pvcapid/models.py Executable file
View File

@ -0,0 +1,169 @@
#!/usr/bin/env python3
# models.py - PVC Database models
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
from pvcapid.flaskapi import app, db
class DBSystemTemplate(db.Model):
__tablename__ = 'system_template'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Text, nullable=False, unique=True)
vcpu_count = db.Column(db.Integer, nullable=False)
vram_mb = db.Column(db.Integer, nullable=False)
serial = db.Column(db.Boolean, nullable=False)
vnc = db.Column(db.Boolean, nullable=False)
vnc_bind = db.Column(db.Text)
node_limit = db.Column(db.Text)
node_selector = db.Column(db.Text)
node_autostart = db.Column(db.Boolean, nullable=False)
def __init__(self, name, vcpu_count, vram_mb, serial, vnc, vnc_bind, node_limit, node_selector, node_autostart):
self.name = name
self.vcpu_count = vcpu_count
self.vram_mb = vram_mb
self.serial = serial
self.vnc = vnc
self.vnc_bind = vnc_bind
self.node_limit = node_limit
self.node_selector = node_selector
self.node_autostart = node_autostart
def __repr__(self):
return '<id {}>'.format(self.id)
class DBNetworkTemplate(db.Model):
__tablename__ = 'network_template'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Text, nullable=False, unique=True)
mac_template = db.Column(db.Text)
def __init__(self, name, mac_template):
self.name = name
self.mac_template = mac_template
def __repr__(self):
return '<id {}>'.format(self.id)
class DBNetworkElement(db.Model):
__tablename__ = 'network'
id = db.Column(db.Integer, primary_key=True)
network_template = db.Column(db.Integer, db.ForeignKey("network_template.id"))
vni = db.Column(db.Integer, nullable=False)
def __init__(self, network_template, vni):
self.network_template = network_template
self.vni = vni
def __repr__(self):
return '<id {}>'.format(self.id)
class DBStorageTemplate(db.Model):
__tablename__ = 'storage_template'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Text, nullable=False, unique=True)
def __init__(self, name):
self.name = name
def __repr__(self):
return '<id {}>'.format(self.id)
class DBStorageElement(db.Model):
__tablename__ = 'storage'
id = db.Column(db.Integer, primary_key=True)
storage_template = db.Column(db.Integer, db.ForeignKey("storage_template.id"))
pool = db.Column(db.Text, nullable=False)
disk_id = db.Column(db.Text, nullable=False)
source_volume = db.Column(db.Text)
disk_size_gb = db.Column(db.Integer)
mountpoint = db.Column(db.Text)
filesystem = db.Column(db.Text)
filesystem_args = db.Column(db.Text)
def __init__(self, storage_template, pool, disk_id, source_volume, disk_size_gb, mountpoint, filesystem, filesystem_args):
self.storage_template = storage_template
self.pool = pool
self.disk_id = disk_id
self.source_volume = source_volume
self.disk_size_gb = disk_size_gb
self.mountpoint = mountpoint
self.filesystem = filesystem
self.filesystem_args = filesystem_args
def __repr__(self):
return '<id {}>'.format(self.id)
class DBUserdata(db.Model):
__tablename__ = 'userdata'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Text, nullable=False, unique=True)
userdata = db.Column(db.Text, nullable=False)
def __init__(self, name, userdata):
self.name = name
self.userdata = userdata
def __repr__(self):
return '<id {}>'.format(self.id)
class DBScript(db.Model):
__tablename__ = 'script'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Text, nullable=False, unique=True)
script = db.Column(db.Text, nullable=False)
def __init__(self, name, script):
self.name = name
self.script = script
def __repr__(self):
return '<id {}>'.format(self.id)
class DBProfile(db.Model):
__tablename__ = 'profile'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Text, nullable=False, unique=True)
system_template = db.Column(db.Integer, db.ForeignKey("system_template.id"))
network_template = db.Column(db.Integer, db.ForeignKey("network_template.id"))
storage_template = db.Column(db.Integer, db.ForeignKey("storage_template.id"))
userdata = db.Column(db.Integer, db.ForeignKey("userdata.id"))
script = db.Column(db.Integer, db.ForeignKey("script.id"))
arguments = db.Column(db.Text)
def __init__(self, name, system_template, network_template, storage_template, userdata, script, arguments):
self.name = name
self.system_template = system_template
self.network_template = network_template
self.storage_template = storage_template
self.userdata = userdata
self.script = script
self.arguments = arguments
def __repr__(self):
return '<id {}>'.format(self.id)

2
debian/control vendored
View File

@ -17,7 +17,7 @@ Description: Parallel Virtual Cluster node daemon (Python 3)
Package: pvc-daemon-api
Architecture: all
Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-gevent, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml
Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-gevent, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml, python3-flask-migrate, python3-flask-script
Description: Parallel Virtual Cluster API daemon (Python 3)
A KVM/Zookeeper/Ceph-based VM and private cloud manager
.

View File

@ -1,4 +1,7 @@
api-daemon/pvcapid.py usr/share/pvc
api-daemon/pvcapid-manage.py usr/share/pvc
api-daemon/pvc-api-db-init usr/share/pvc
api-daemon/pvc-api-db-upgrade usr/share/pvc
api-daemon/pvcapid.sample.yaml etc/pvc
api-daemon/pvcapid usr/share/pvc
api-daemon/pvcapid.service lib/systemd/system

View File

@ -1,20 +1,15 @@
#!/bin/sh
# Install client binary to /usr/bin via symlink
ln -s /usr/share/pvc/api.py /usr/bin/pvcapid
# Reload systemd's view of the units
systemctl daemon-reload
# Restart the main daemon (or warn on first install)
# Restart the main daemon and apply database migrations (or warn on first install)
if systemctl is-active --quiet pvcapid.service; then
systemctl restart pvcapid.service
systemctl stop pvcapid-worker.service
systemctl stop pvcapid.service
/usr/share/pvc/pvc-api-db-upgrade
systemctl start pvcapid.service
systemctl start pvcapid-worker.service
else
echo "NOTE: The PVC client API daemon (pvcapid.service) has not been started; create a config file at /etc/pvc/pvcapid.yaml then start it."
fi
# Restart the worker daemon (or warn on first install)
if systemctl is-active --quiet pvcapid-worker.service; then
systemctl restart pvcapid-worker.service
else
echo "NOTE: The PVC provisioner worker daemon (pvcapid-worker.service) has not been started; create a config file at /etc/pvc/pvcapid.yaml then start it."
echo "NOTE: The PVC client API daemon (pvcapid.service) and the PVC provisioner worker daemon (pvcapid-worker.service) have not been started; create a config file at /etc/pvc/pvcapid.yaml, then run the database configuration (/usr/share/pvc/api-db-init) and start them manually."
fi