diff --git a/api-daemon/pvc-api-db-init b/api-daemon/pvc-api-db-init new file mode 100755 index 00000000..a5e69128 --- /dev/null +++ b/api-daemon/pvc-api-db-init @@ -0,0 +1,17 @@ +#!/bin/bash + +# Initialize the PVC database and migrations for future upgrades +# Part of the Parallel Virtual Cluster (PVC) system + +export PVC_CONFIG_FILE="/etc/pvc/pvcapid.yaml" + +if [[ ! -f ${PVC_CONFIG_FILE} ]]; then + echo "Create a configuration file at ${PVC_CONFIG_FILE} before initializing the database." + exit 1 +fi + +pushd /usr/share/pvc +./pvcapid-manage.py db init +./pvcapid-manage.py db migrate +./pvcapid-manage.py db upgrade +popd diff --git a/api-daemon/pvc-api-db-migrate b/api-daemon/pvc-api-db-migrate new file mode 100755 index 00000000..6d26b4d0 --- /dev/null +++ b/api-daemon/pvc-api-db-migrate @@ -0,0 +1,17 @@ +#!/bin/bash + +# Apply PVC database migrations +# Part of the Parallel Virtual Cluster (PVC) system + +PVC_CONFIG_FILE="/etc/pvc/pvcapid.yaml" +PVC_SHARE_DIR="/usr/share/pvc" + +if [[ ! -f ${PVC_CONFIG_FILE} ]]; then + echo "Create a configuration file at ${PVC_CONFIG_FILE} before upgrading the database." + exit 1 +fi + +pushd ${PVC_SHARE_DIR} +./pvcapid-manage.py db migrate +./pvcapid-manage.py db upgrade +popd diff --git a/api-daemon/pvc-api-db-upgrade b/api-daemon/pvc-api-db-upgrade new file mode 100755 index 00000000..6db47c10 --- /dev/null +++ b/api-daemon/pvc-api-db-upgrade @@ -0,0 +1,16 @@ +#!/bin/bash + +# Apply PVC database migrations +# Part of the Parallel Virtual Cluster (PVC) system + +export PVC_CONFIG_FILE="/etc/pvc/pvcapid.yaml" + +if [[ ! -f ${PVC_CONFIG_FILE} ]]; then + echo "Create a configuration file at ${PVC_CONFIG_FILE} before upgrading the database." + exit 1 +fi + +pushd /usr/share/pvc +./pvcapid-manage.py db migrate +./pvcapid-manage.py db upgrade +popd diff --git a/api-daemon/pvcapid-manage.py b/api-daemon/pvcapid-manage.py new file mode 100755 index 00000000..0de43342 --- /dev/null +++ b/api-daemon/pvcapid-manage.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +# manage.py - PVC Database management tasks +# Part of the Parallel Virtual Cluster (PVC) system +# +# Copyright (C) 2018-2020 Joshua M. Boniface +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### + +import os +from flask_migrate import Migrate, MigrateCommand +from flask_script import Manager + +from pvcapid.flaskapi import app, db, config + +migrate = Migrate(app, db) +manager = Manager(app) + +manager.add_command('db', MigrateCommand) + +if __name__ == '__main__': + manager.run() diff --git a/api-daemon/pvcapid/flaskapi.py b/api-daemon/pvcapid/flaskapi.py index f5acd6e0..28e179e9 100755 --- a/api-daemon/pvcapid/flaskapi.py +++ b/api-daemon/pvcapid/flaskapi.py @@ -41,6 +41,8 @@ import pvcapid.helper as api_helper import pvcapid.provisioner as api_provisioner import pvcapid.ova as api_ova +from flask_sqlalchemy import SQLAlchemy + API_VERSION = 1.0 # Parse the configuration file @@ -105,6 +107,7 @@ except Exception as e: app = flask.Flask(__name__) app.config['CELERY_BROKER_URL'] = 'redis://{}:{}{}'.format(config['queue_host'], config['queue_port'], config['queue_path']) app.config['CELERY_RESULT_BACKEND'] = 'redis://{}:{}{}'.format(config['queue_host'], config['queue_port'], config['queue_path']) +app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://{}:{}@{}:{}/{}'.format(config['database_user'], config['database_password'], config['database_host'], config['database_port'], config['database_name']) if config['debug']: app.config['DEBUG'] = True @@ -112,6 +115,12 @@ if config['debug']: if config['auth_enabled']: app.config["SECRET_KEY"] = config['auth_secret_key'] +# Create SQLAlchemy database +db = SQLAlchemy(app) + +# Import database models +from pvcapid.models import DBSystemTemplate, DBNetworkTemplate, DBNetworkElement, DBStorageTemplate, DBStorageElement, DBUserdata, DBScript, DBProfile + # Create Flask blueprint blueprint = flask.Blueprint('api', __name__, url_prefix='/api/v1') diff --git a/api-daemon/pvcapid/models.py b/api-daemon/pvcapid/models.py new file mode 100755 index 00000000..483f9f72 --- /dev/null +++ b/api-daemon/pvcapid/models.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 + +# models.py - PVC Database models +# Part of the Parallel Virtual Cluster (PVC) system +# +# Copyright (C) 2018-2020 Joshua M. Boniface +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### + +from pvcapid.flaskapi import app, db + +class DBSystemTemplate(db.Model): + __tablename__ = 'system_template' + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text, nullable=False, unique=True) + vcpu_count = db.Column(db.Integer, nullable=False) + vram_mb = db.Column(db.Integer, nullable=False) + serial = db.Column(db.Boolean, nullable=False) + vnc = db.Column(db.Boolean, nullable=False) + vnc_bind = db.Column(db.Text) + node_limit = db.Column(db.Text) + node_selector = db.Column(db.Text) + node_autostart = db.Column(db.Boolean, nullable=False) + + def __init__(self, name, vcpu_count, vram_mb, serial, vnc, vnc_bind, node_limit, node_selector, node_autostart): + self.name = name + self.vcpu_count = vcpu_count + self.vram_mb = vram_mb + self.serial = serial + self.vnc = vnc + self.vnc_bind = vnc_bind + self.node_limit = node_limit + self.node_selector = node_selector + self.node_autostart = node_autostart + + def __repr__(self): + return ''.format(self.id) + +class DBNetworkTemplate(db.Model): + __tablename__ = 'network_template' + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text, nullable=False, unique=True) + mac_template = db.Column(db.Text) + + def __init__(self, name, mac_template): + self.name = name + self.mac_template = mac_template + + def __repr__(self): + return ''.format(self.id) + +class DBNetworkElement(db.Model): + __tablename__ = 'network' + + id = db.Column(db.Integer, primary_key=True) + network_template = db.Column(db.Integer, db.ForeignKey("network_template.id")) + vni = db.Column(db.Integer, nullable=False) + + def __init__(self, network_template, vni): + self.network_template = network_template + self.vni = vni + + def __repr__(self): + return ''.format(self.id) + +class DBStorageTemplate(db.Model): + __tablename__ = 'storage_template' + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text, nullable=False, unique=True) + + def __init__(self, name): + self.name = name + + def __repr__(self): + return ''.format(self.id) + +class DBStorageElement(db.Model): + __tablename__ = 'storage' + + id = db.Column(db.Integer, primary_key=True) + storage_template = db.Column(db.Integer, db.ForeignKey("storage_template.id")) + pool = db.Column(db.Text, nullable=False) + disk_id = db.Column(db.Text, nullable=False) + source_volume = db.Column(db.Text) + disk_size_gb = db.Column(db.Integer) + mountpoint = db.Column(db.Text) + filesystem = db.Column(db.Text) + filesystem_args = db.Column(db.Text) + + def __init__(self, storage_template, pool, disk_id, source_volume, disk_size_gb, mountpoint, filesystem, filesystem_args): + self.storage_template = storage_template + self.pool = pool + self.disk_id = disk_id + self.source_volume = source_volume + self.disk_size_gb = disk_size_gb + self.mountpoint = mountpoint + self.filesystem = filesystem + self.filesystem_args = filesystem_args + + def __repr__(self): + return ''.format(self.id) + +class DBUserdata(db.Model): + __tablename__ = 'userdata' + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text, nullable=False, unique=True) + userdata = db.Column(db.Text, nullable=False) + + def __init__(self, name, userdata): + self.name = name + self.userdata = userdata + + def __repr__(self): + return ''.format(self.id) + +class DBScript(db.Model): + __tablename__ = 'script' + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text, nullable=False, unique=True) + script = db.Column(db.Text, nullable=False) + + def __init__(self, name, script): + self.name = name + self.script = script + + def __repr__(self): + return ''.format(self.id) + +class DBProfile(db.Model): + __tablename__ = 'profile' + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text, nullable=False, unique=True) + system_template = db.Column(db.Integer, db.ForeignKey("system_template.id")) + network_template = db.Column(db.Integer, db.ForeignKey("network_template.id")) + storage_template = db.Column(db.Integer, db.ForeignKey("storage_template.id")) + userdata = db.Column(db.Integer, db.ForeignKey("userdata.id")) + script = db.Column(db.Integer, db.ForeignKey("script.id")) + arguments = db.Column(db.Text) + + def __init__(self, name, system_template, network_template, storage_template, userdata, script, arguments): + self.name = name + self.system_template = system_template + self.network_template = network_template + self.storage_template = storage_template + self.userdata = userdata + self.script = script + self.arguments = arguments + + def __repr__(self): + return ''.format(self.id) diff --git a/debian/control b/debian/control index db10f342..d48c4fa5 100644 --- a/debian/control +++ b/debian/control @@ -17,7 +17,7 @@ Description: Parallel Virtual Cluster node daemon (Python 3) Package: pvc-daemon-api Architecture: all -Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-gevent, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml +Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-gevent, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml, python3-flask-migrate, python3-flask-script Description: Parallel Virtual Cluster API daemon (Python 3) A KVM/Zookeeper/Ceph-based VM and private cloud manager . diff --git a/debian/pvc-daemon-api.install b/debian/pvc-daemon-api.install index 52e55633..57054cbf 100644 --- a/debian/pvc-daemon-api.install +++ b/debian/pvc-daemon-api.install @@ -1,4 +1,7 @@ api-daemon/pvcapid.py usr/share/pvc +api-daemon/pvcapid-manage.py usr/share/pvc +api-daemon/pvc-api-db-init usr/share/pvc +api-daemon/pvc-api-db-upgrade usr/share/pvc api-daemon/pvcapid.sample.yaml etc/pvc api-daemon/pvcapid usr/share/pvc api-daemon/pvcapid.service lib/systemd/system diff --git a/debian/pvc-daemon-api.postinst b/debian/pvc-daemon-api.postinst index 423fdb6a..6d35748e 100644 --- a/debian/pvc-daemon-api.postinst +++ b/debian/pvc-daemon-api.postinst @@ -1,20 +1,15 @@ #!/bin/sh -# Install client binary to /usr/bin via symlink -ln -s /usr/share/pvc/api.py /usr/bin/pvcapid - # Reload systemd's view of the units systemctl daemon-reload -# Restart the main daemon (or warn on first install) +# Restart the main daemon and apply database migrations (or warn on first install) if systemctl is-active --quiet pvcapid.service; then - systemctl restart pvcapid.service + systemctl stop pvcapid-worker.service + systemctl stop pvcapid.service + /usr/share/pvc/pvc-api-db-upgrade + systemctl start pvcapid.service + systemctl start pvcapid-worker.service else - echo "NOTE: The PVC client API daemon (pvcapid.service) has not been started; create a config file at /etc/pvc/pvcapid.yaml then start it." -fi -# Restart the worker daemon (or warn on first install) -if systemctl is-active --quiet pvcapid-worker.service; then - systemctl restart pvcapid-worker.service -else - echo "NOTE: The PVC provisioner worker daemon (pvcapid-worker.service) has not been started; create a config file at /etc/pvc/pvcapid.yaml then start it." + echo "NOTE: The PVC client API daemon (pvcapid.service) and the PVC provisioner worker daemon (pvcapid-worker.service) have not been started; create a config file at /etc/pvc/pvcapid.yaml, then run the database configuration (/usr/share/pvc/api-db-init) and start them manually." fi