Bump version to 0.9.6

Prevents issues with converting None to integers and such.

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,68 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+# ******** NOTE ********
+
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '17 22 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.gitlab-ci.yml

@@ -2,12 +2,14 @@ stages:
    - build
    - deploy
 
-build:
+build_releases:
   stage: build
   before_script: 
     - git submodule update --init
   script:
     - /bin/bash build-deb.sh
-    - /usr/local/bin/deploy-package
+    - /usr/local/bin/deploy-package -C pvc
   only:
-    - master
+    - tags
+  except:
+    - branches

.hooks/pre-commit

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+pushd $( git rev-parse --show-toplevel ) &>/dev/null
+
+ex=0
+
+# Linting
+echo -n "Linting... "
+./lint
+if [[ $? -ne 0 ]]; then
+    echo "Aborting commit due to linting errors."
+    ex=1
+fi
+
+echo
+popd &>/dev/null
+exit $ex

README.md

@@ -1,27 +1,83 @@
-# PVC - The Parallel Virtual Cluster suite
+# PVC - The Parallel Virtual Cluster system
 
 <p align="center">
 <img alt="Logo banner" src="https://git.bonifacelabs.ca/uploads/-/system/project/avatar/135/pvc_logo.png"/>
 <br/><br/>
 <a href="https://github.com/parallelvirtualcluster/pvc"><img alt="License" src="https://img.shields.io/github/license/parallelvirtualcluster/pvc"/></a>
 <a href="https://github.com/parallelvirtualcluster/pvc/releases"><img alt="Release" src="https://img.shields.io/github/release-pre/parallelvirtualcluster/pvc"/></a>
-<a href="https://git.bonifacelabs.ca/parallelvirtualcluster/pvc/pipelines"><img alt="Pipeline Status" src="https://git.bonifacelabs.ca/parallelvirtualcluster/pvc/badges/master/pipeline.svg"/></a>
 <a href="https://parallelvirtualcluster.readthedocs.io/en/latest/?badge=latest"><img alt="Documentation Status" src="https://readthedocs.org/projects/parallelvirtualcluster/badge/?version=latest"/></a>
 </p>
 
-PVC is a suite of Python 3 tools to manage virtualized clusters. It provides a fully-functional private cloud based on four key principles:
+**NOTICE FOR GITHUB**: This repository is a read-only mirror of the PVC repositories from my personal GitLab instance. Pull requests submitted here will not be merged. Issues submitted here will however be treated as authoritative.
 
-1. Be Free Software Forever (or Bust)
-2. Be Opinionated and Efficient and Pick The Best Software
-3. Be Scalable and Redundant but Not Hyperscale
-4. Be Simple To Use, Configure, and Maintain
+PVC is a KVM+Ceph+Zookeeper-based, Free Software, scalable, redundant, self-healing, and self-managing private cloud solution designed with administrator simplicity in mind. It is built from the ground-up to be redundant at the host layer, allowing the cluster to gracefully handle the loss of nodes or their components, both due to hardware failure or due to maintenance. It is able to scale from a minimum of 3 nodes up to 12 or more nodes, while retaining performance and flexibility, allowing the administrator to build a small cluster today and grow it as needed.
 
-It is designed to be an administrator-friendly but extremely powerful and rich modern private cloud system, but without the feature bloat and complexity of tools like OpenStack. With PVC, an administrator can provision, manage, and update a cluster of dozens or more hypervisors running thousands of VMs using a simple CLI tool, HTTP API, or [eventually] web interface. PVC is based entirely on Debian GNU/Linux and Free-and-Open-Source tools, providing the glue to bootstrap, provision and manage the cluster, then getting out of the administrators' way.
+The major goal of PVC is to be administrator friendly, providing the power of Enterprise-grade private clouds like OpenStack, Nutanix, and VMWare to homelabbers, SMBs, and small ISPs, without the cost or complexity. It believes in picking the best tool for a job and abstracting it behind the cluster as a whole, freeing the administrator from the boring and time-consuming task of selecting the best component, and letting them get on with the things that really matter. Administration can be done from a simple CLI or via a RESTful API capable of building full-featured web frontends or additional applications, taking a self-documenting approach to keep the administrator learning curvet as low as possible. Setup is easy and straightforward with an [ISO-based node installer](https://git.bonifacelabs.ca/parallelvirtualcluster/pvc-installer) and [Ansible role framework](https://git.bonifacelabs.ca/parallelvirtualcluster/pvc-ansible) designed to get a cluster up and running as quickly as possible. Build your cloud in an hour, grow it as you need, and never worry about it: just add physical servers.
 
-Your cloud, the best way; just add physical servers.
+## Getting Started
 
-[See the documentation here](https://parallelvirtualcluster.readthedocs.io/en/latest/)
+To get started with PVC, please see the [About](https://parallelvirtualcluster.readthedocs.io/en/latest/about/) page for general information about the project, and the [Getting Started](https://parallelvirtualcluster.readthedocs.io/en/latest/getting-started/) page for details on configuring your cluster.
 
-[See the API reference here](https://parallelvirtualcluster.readthedocs.io/en/latest/manuals/api-reference.html)
+## Changelog
+
+#### v0.9.6
+
+  * Fixes bug with migrations
+
+#### v0.9.5
+
+  * Fixes bug with line count in log follow
+  * Fixes bug with disk stat output being None
+  * Adds short pretty health output
+  * Documentation updates
+
+#### v0.9.4
+
+  * Fixes major bug in OVA parser
+
+#### v0.9.3
+
+  * Fixes bugs with image & OVA upload parsing
+
+#### v0.9.2
+
+  * Major linting of the codebase with flake8; adds linting tools
+  * Implements CLI-based modification of VM vCPUs, memory, networks, and disks without directly editing XML
+  * Fixes bug where `pvc vm log -f` would show all 1000 lines before starting
+  * Fixes bug in default provisioner libvirt schema (`drive` -> `driver` typo)
+
+#### v0.9.1
+
+  * Added per-VM migration method feature
+  * Fixed bug with provisioner system template listing
+
+#### v0.9.0
+
+Numerous small improvements and bugfixes. This release is suitable for general use and is pre-release-quality software.
+
+This release introduces an updated version scheme; all future stable releases until 1.0.0 is ready will be made under this 0.9.z naming. This does not represent semantic versioning and all changes (feature, improvement, or bugfix) will be considered for inclusion in this release train.
+
+#### v0.8
+
+Numerous improvements and bugfixes. This release is suitable for general use and is pre-release-quality software.
+
+#### v0.7
+
+Numerous improvements and bugfixes, revamped documentation. This release is suitable for general use and is beta-quality software.
+
+#### v0.6
+
+Numerous improvements and bugfixes, full implementation of the provisioner, full implementation of the API CLI client (versus direct CLI client). This release is suitable for general use and is beta-quality software.
+
+#### v0.5
+
+First public release; fully implements the VM, network, and storage managers, the HTTP API, and the pvc-ansible framework for deploying and bootstrapping a cluster. This release is suitable for general use, though it is still alpha-quality software and should be expected to change significantly until 1.0 is released.
+
+#### v0.4
+
+Full implementation of virtual management and virtual networking functionality. Partial implementation of storage functionality.
+
+#### v0.3
+
+Basic implementation of virtual management functionality.
 
-**NOTICE FOR GITHUB**: This repository is a read-only mirror of the PVC repositories. Pull requests submitted here will not be merged.

api-daemon/daemon_lib

@@ -0,0 +1 @@
+../daemon-common

api-daemon/migrations/README

@@ -0,0 +1 @@
+Generic single-database configuration.

api-daemon/migrations/alembic.ini

@@ -0,0 +1,45 @@
+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+script_location = .
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

api-daemon/migrations/env.py

@@ -0,0 +1,88 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+from flask import current_app
+import logging
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+logger = logging.getLogger('alembic.env')
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+config.set_main_option('sqlalchemy.url',
+                       current_app.config.get('SQLALCHEMY_DATABASE_URI'))
+target_metadata = current_app.extensions['migrate'].db.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(url=url)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+
+    # this callback is used to prevent an auto-migration from being generated
+    # when there are no changes to the schema
+    # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
+    def process_revision_directives(context, revision, directives):
+        if getattr(config.cmd_opts, 'autogenerate', False):
+            script = directives[0]
+            if script.upgrade_ops.is_empty():
+                directives[:] = []
+                logger.info('No changes in schema detected.')
+
+    engine = engine_from_config(config.get_section(config.config_ini_section),
+                                prefix='sqlalchemy.',
+                                poolclass=pool.NullPool)
+
+    connection = engine.connect()
+    context.configure(connection=connection,
+                      target_metadata=target_metadata,
+                      process_revision_directives=process_revision_directives,
+                      **current_app.extensions['migrate'].configure_args)
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    finally:
+        connection.close()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

api-daemon/migrations/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

api-daemon/migrations/versions/2d1daa722a0a_pvc_version_0_6.py

@@ -0,0 +1,112 @@
+"""PVC version 0.6
+
+Revision ID: 2d1daa722a0a
+Revises:
+Create Date: 2020-02-15 23:14:14.733134
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2d1daa722a0a'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('network_template',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.Text(), nullable=False),
+    sa.Column('mac_template', sa.Text(), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('script',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.Text(), nullable=False),
+    sa.Column('script', sa.Text(), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('storage_template',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.Text(), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('system_template',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.Text(), nullable=False),
+    sa.Column('vcpu_count', sa.Integer(), nullable=False),
+    sa.Column('vram_mb', sa.Integer(), nullable=False),
+    sa.Column('serial', sa.Boolean(), nullable=False),
+    sa.Column('vnc', sa.Boolean(), nullable=False),
+    sa.Column('vnc_bind', sa.Text(), nullable=True),
+    sa.Column('node_limit', sa.Text(), nullable=True),
+    sa.Column('node_selector', sa.Text(), nullable=True),
+    sa.Column('node_autostart', sa.Boolean(), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('userdata',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.Text(), nullable=False),
+    sa.Column('userdata', sa.Text(), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('network',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('network_template', sa.Integer(), nullable=True),
+    sa.Column('vni', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['network_template'], ['network_template.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('profile',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.Text(), nullable=False),
+    sa.Column('system_template', sa.Integer(), nullable=True),
+    sa.Column('network_template', sa.Integer(), nullable=True),
+    sa.Column('storage_template', sa.Integer(), nullable=True),
+    sa.Column('userdata', sa.Integer(), nullable=True),
+    sa.Column('script', sa.Integer(), nullable=True),
+    sa.Column('arguments', sa.Text(), nullable=True),
+    sa.ForeignKeyConstraint(['network_template'], ['network_template.id'], ),
+    sa.ForeignKeyConstraint(['script'], ['script.id'], ),
+    sa.ForeignKeyConstraint(['storage_template'], ['storage_template.id'], ),
+    sa.ForeignKeyConstraint(['system_template'], ['system_template.id'], ),
+    sa.ForeignKeyConstraint(['userdata'], ['userdata.id'], ),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('storage',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('storage_template', sa.Integer(), nullable=True),
+    sa.Column('pool', sa.Text(), nullable=False),
+    sa.Column('disk_id', sa.Text(), nullable=False),
+    sa.Column('source_volume', sa.Text(), nullable=True),
+    sa.Column('disk_size_gb', sa.Integer(), nullable=True),
+    sa.Column('mountpoint', sa.Text(), nullable=True),
+    sa.Column('filesystem', sa.Text(), nullable=True),
+    sa.Column('filesystem_args', sa.Text(), nullable=True),
+    sa.ForeignKeyConstraint(['storage_template'], ['storage_template.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('storage')
+    op.drop_table('profile')
+    op.drop_table('network')
+    op.drop_table('userdata')
+    op.drop_table('system_template')
+    op.drop_table('storage_template')
+    op.drop_table('script')
+    op.drop_table('network_template')
+    # ### end Alembic commands ###

api-daemon/migrations/versions/3bc6117ea44d_pvc_version_0_7.py

@@ -0,0 +1,33 @@
+"""PVC version 0.7
+
+Revision ID: 3bc6117ea44d
+Revises: 88c8514684f7
+Create Date: 2020-08-24 14:34:36.919308
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3bc6117ea44d'
+down_revision = '88c8514684f7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('storage_benchmarks',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('job', sa.Text(), nullable=False),
+    sa.Column('result', sa.Text(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('storage_benchmarks')
+    # ### end Alembic commands ###

api-daemon/migrations/versions/3efe890e1d87_pvc_version_0_9_0.py

@@ -0,0 +1,28 @@
+"""PVC version 0.9.0
+
+Revision ID: 3efe890e1d87
+Revises: 3bc6117ea44d
+Create Date: 2020-10-29 11:49:58.756626
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3efe890e1d87'
+down_revision = '3bc6117ea44d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('system_template', sa.Column('migration_method', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('system_template', 'migration_method')
+    # ### end Alembic commands ###

api-daemon/migrations/versions/88c8514684f7_pvc_version_0_7.py

@@ -0,0 +1,76 @@
+"""PVC version 0.7
+
+Revision ID: 88c8514684f7
+Revises: 2d1daa722a0a
+Create Date: 2020-02-16 19:49:50.126265
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '88c8514684f7'
+down_revision = '2d1daa722a0a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('ova',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.Text(), nullable=False),
+    sa.Column('ovf', sa.Text(), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('ova_volume',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('ova', sa.Integer(), nullable=False),
+    sa.Column('pool', sa.Text(), nullable=False),
+    sa.Column('volume_name', sa.Text(), nullable=False),
+    sa.Column('volume_format', sa.Text(), nullable=False),
+    sa.Column('disk_id', sa.Text(), nullable=False),
+    sa.Column('disk_size_gb', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['ova'], ['ova.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.alter_column('network', 'network_template',
+               existing_type=sa.INTEGER(),
+               nullable=False)
+    op.add_column('network_template', sa.Column('ova', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'network_template', 'ova', ['ova'], ['id'])
+    op.add_column('profile', sa.Column('ova', sa.Integer(), nullable=True))
+    op.add_column('profile', sa.Column('profile_type', sa.Text(), nullable=False))
+    op.create_foreign_key(None, 'profile', 'ova', ['ova'], ['id'])
+    op.alter_column('storage', 'storage_template',
+               existing_type=sa.INTEGER(),
+               nullable=False)
+    op.add_column('storage_template', sa.Column('ova', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'storage_template', 'ova', ['ova'], ['id'])
+    op.add_column('system_template', sa.Column('ova', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'system_template', 'ova', ['ova'], ['id'])
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'system_template', type_='foreignkey')
+    op.drop_column('system_template', 'ova')
+    op.drop_constraint(None, 'storage_template', type_='foreignkey')
+    op.drop_column('storage_template', 'ova')
+    op.alter_column('storage', 'storage_template',
+               existing_type=sa.INTEGER(),
+               nullable=True)
+    op.drop_constraint(None, 'profile', type_='foreignkey')
+    op.drop_column('profile', 'profile_type')
+    op.drop_column('profile', 'ova')
+    op.drop_constraint(None, 'network_template', type_='foreignkey')
+    op.drop_column('network_template', 'ova')
+    op.alter_column('network', 'network_template',
+               existing_type=sa.INTEGER(),
+               nullable=True)
+    op.drop_table('ova_volume')
+    op.drop_table('ova')
+    # ### end Alembic commands ###

api-daemon/provisioner/examples/debootstrap_script.py

@@ -50,15 +50,15 @@ def install(**kwargs):
     # failures of these gracefully, should administrators forget to specify them.
     try:
         deb_release = kwargs['deb_release']
-    except:
+    except Exception:
         deb_release = "stable"
     try:
         deb_mirror = kwargs['deb_mirror']
-    except:
+    except Exception:
         deb_mirror = "http://ftp.debian.org/debian"
     try:
         deb_packages = kwargs['deb_packages'].split(',')
-    except:
+    except Exception:
         deb_packages = ["linux-image-amd64", "grub-pc", "cloud-init", "python3-cffi-backend", "wget"]
 
     # We need to know our root disk
@@ -109,6 +109,7 @@ def install(**kwargs):
 
         # The root, var, and log volumes have specific values
         if disk['mountpoint'] == "/":
+            root_disk['scsi_id'] = disk_id
             dump = 0
             cpass = 1
         elif disk['mountpoint'] == '/var' or disk['mountpoint'] == '/var/log':
@@ -184,12 +185,12 @@ interface "ens2" {
 GRUB_DEFAULT=0
 GRUB_TIMEOUT=1
 GRUB_DISTRIBUTOR="PVC Virtual Machine"
-GRUB_CMDLINE_LINUX_DEFAULT="root=/dev/{root_disk} console=tty0 console=ttyS0,115200n8"
+GRUB_CMDLINE_LINUX_DEFAULT="root=/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0-0-0-{root_disk} console=tty0 console=ttyS0,115200n8"
 GRUB_CMDLINE_LINUX=""
 GRUB_TERMINAL=console
 GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
 GRUB_DISABLE_LINUX_UUID=false
-""".format(root_disk=root_disk['disk_id'])
+""".format(root_disk=root_disk['scsi_id'])
         fh.write(data)
 
     # Chroot, do some in-root tasks, then exit the chroot
@@ -204,7 +205,7 @@ GRUB_DISABLE_LINUX_UUID=false
     os.system(
         "grub-install --force /dev/rbd/{}/{}_{}".format(root_disk['pool'], vm_name, root_disk['disk_id'])
     )
-    os.system( 
+    os.system(
         "update-grub"
     )
     # Set a really dumb root password [TEMPORARY]

api-daemon/provisioner/examples/dummy_script.py

@@ -30,8 +30,6 @@
 # This script will run under root privileges as the provisioner does. Be careful
 # with that.
 
-import os
-
 # Installation function - performs a debootstrap install of a Debian system
 # Note that the only arguments are keyword arguments.
 def install(**kwargs):

api-daemon/pvc-api-db-upgrade

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Apply PVC database migrations
+# Part of the Parallel Virtual Cluster (PVC) system
+
+export PVC_CONFIG_FILE="/etc/pvc/pvcapid.yaml"
+
+if [[ ! -f ${PVC_CONFIG_FILE} ]]; then
+    echo "Create a configuration file at ${PVC_CONFIG_FILE} before upgrading the database."
+    exit 1
+fi
+
+pushd /usr/share/pvc
+./pvcapid-manage.py db upgrade
+popd

api-daemon/pvcapid-manage.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+# manage.py - PVC Database management tasks
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from flask_migrate import Migrate, MigrateCommand
+from flask_script import Manager
+
+from pvcapid.flaskapi import app, db
+from pvcapid.models import *  # noqa F401,F403
+
+migrate = Migrate(app, db)
+manager = Manager(app)
+
+manager.add_command('db', MigrateCommand)
+
+if __name__ == '__main__':
+    manager.run()

api-daemon/pvcapid-worker.service

@@ -0,0 +1,16 @@
+# Parallel Virtual Cluster Provisioner API provisioner worker unit file
+
+[Unit]
+Description = Parallel Virtual Cluster API provisioner worker
+After = network-online.target
+
+[Service]
+Type = simple
+WorkingDirectory = /usr/share/pvc
+Environment = PYTHONUNBUFFERED=true
+Environment = PVC_CONFIG_FILE=/etc/pvc/pvcapid.yaml
+ExecStart = /usr/bin/celery worker -A pvcapid.flaskapi.celery --concurrency 1 --loglevel INFO
+Restart = on-failure
+
+[Install]
+WantedBy = multi-user.target

api-daemon/pvcapid.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# pvcd.py - Node daemon startup stub
+# pvcapid.py - API daemon startup stub
 # Part of the Parallel Virtual Cluster (PVC) system
 #
 #    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
@@ -20,4 +20,4 @@
 #
 ###############################################################################
 
-import pvcd.Daemon
+import pvcapid.Daemon  # noqa: F401

api-daemon/pvcapid.sample.yaml

@@ -1,11 +1,11 @@
 ---
-# pvc-api client configuration file example
+# pvcapid configuration file example
 #
-# This configuration file specifies details for the PVC API client running on
+# This configuration file specifies details for the PVC API daemon running on
 # this machine. Default values are not supported; the values in this sample
 # configuration are considered defaults and can be used as-is.
 #
-# Copy this example to /etc/pvc/pvc-api.conf and edit to your needs
+# Copy this example to /etc/pvc/pvcapid.conf and edit to your needs
 
 pvc:
     # debug: Enable/disable API debug mode
@@ -49,12 +49,12 @@ pvc:
             host: localhost
             # port: PostgreSQL port, invariably '5432'
             port: 5432
-            # name: PostgreSQL database name, invariably 'pvcprov'
-            name: pvcprov
-            # user: PostgreSQL username, invariable 'pvcprov'
-            user: pvcprov
+            # name: PostgreSQL database name, invariably 'pvcapi'
+            name: pvcapi
+            # user: PostgreSQL username, invariable 'pvcapi'
+            user: pvcapi
             # pass: PostgreSQL user password, randomly generated
-            pass: pvcprov
+            pass: pvcapi
         # queue: Celery backend queue using the PVC Zookeeper cluster
         queue:
             # host: Redis hostname, usually 'localhost'
@@ -70,7 +70,7 @@ pvc:
             storage_hosts:
               - pvchv1
               - pvchv2
-              - pvchv2
+              - pvchv3
             # storage_domain: The storage domain name, concatenated with the coordinators list names
             #                 to form monitor access strings
             storage_domain: "pvc.storage"

api-daemon/pvcapid.service

@@ -8,8 +8,8 @@ After = network-online.target
 Type = simple
 WorkingDirectory = /usr/share/pvc
 Environment = PYTHONUNBUFFERED=true
-Environment = PVC_CONFIG_FILE=/etc/pvc/pvc-api.yaml
-ExecStart = /usr/share/pvc/pvc-api.py
+Environment = PVC_CONFIG_FILE=/etc/pvc/pvcapid.yaml
+ExecStart = /usr/share/pvc/pvcapid.py
 Restart = on-failure
 
 [Install]

api-daemon/pvcapid/Daemon.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+# Daemon.py - PVC HTTP API daemon
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import pvcapid.flaskapi as pvc_api
+
+##########################################################
+# Entrypoint
+##########################################################
+
+if pvc_api.config['ssl_enabled']:
+    context = (pvc_api.config['ssl_cert_file'], pvc_api.config['ssl_key_file'])
+else:
+    context = None
+
+print('Starting PVC API daemon at {}:{} with SSL={}, Authentication={}'.format(pvc_api.config['listen_address'], pvc_api.config['listen_port'], pvc_api.config['ssl_enabled'], pvc_api.config['auth_enabled']))
+pvc_api.app.run(pvc_api.config['listen_address'], pvc_api.config['listen_port'], threaded=True, ssl_context=context)

api-daemon/pvcapid/benchmark.py

@@ -0,0 +1,466 @@
+#!/usr/bin/env python3
+
+# benchmark.py - PVC API Benchmark functions
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import psycopg2
+import psycopg2.extras
+
+from distutils.util import strtobool as dustrtobool
+
+import daemon_lib.common as pvc_common
+import daemon_lib.ceph as pvc_ceph
+
+config = None  # Set in this namespace by flaskapi
+
+
+def strtobool(stringv):
+    if stringv is None:
+        return False
+    if isinstance(stringv, bool):
+        return bool(stringv)
+    try:
+        return bool(dustrtobool(stringv))
+    except Exception:
+        return False
+
+
+#
+# Exceptions (used by Celery tasks)
+#
+class BenchmarkError(Exception):
+    """
+    An exception that results from the Benchmark job.
+    """
+    def __init__(self, message, cur_time=None, db_conn=None, db_cur=None, zk_conn=None):
+        self.message = message
+        if cur_time is not None:
+            # Clean up our dangling result
+            query = "DELETE FROM storage_benchmarks WHERE job = %s;"
+            args = (cur_time,)
+            db_cur.execute(query, args)
+            db_conn.commit()
+            # Close the database connections cleanly
+            close_database(db_conn, db_cur)
+            pvc_common.stopZKConnection(zk_conn)
+
+    def __str__(self):
+        return str(self.message)
+
+#
+# Common functions
+#
+
+
+# Database connections
+def open_database(config):
+    conn = psycopg2.connect(
+        host=config['database_host'],
+        port=config['database_port'],
+        dbname=config['database_name'],
+        user=config['database_user'],
+        password=config['database_password']
+    )
+    cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+    return conn, cur
+
+
+def close_database(conn, cur, failed=False):
+    if not failed:
+        conn.commit()
+    cur.close()
+    conn.close()
+
+
+def list_benchmarks(job=None):
+    if job is not None:
+        query = "SELECT * FROM {} WHERE job = %s;".format('storage_benchmarks')
+        args = (job, )
+    else:
+        query = "SELECT * FROM {} ORDER BY id DESC;".format('storage_benchmarks')
+        args = ()
+
+    conn, cur = open_database(config)
+    cur.execute(query, args)
+    orig_data = cur.fetchall()
+    data = list()
+    for benchmark in orig_data:
+        benchmark_data = dict()
+        benchmark_data['id'] = benchmark['id']
+        benchmark_data['job'] = benchmark['job']
+        benchmark_data['benchmark_result'] = benchmark['result']
+        # Append the new data to our actual output structure
+        data.append(benchmark_data)
+    close_database(conn, cur)
+    if data:
+        return data, 200
+    else:
+        return {'message': 'No benchmark found.'}, 404
+
+
+def run_benchmark(self, pool):
+    # Runtime imports
+    import time
+    import json
+    from datetime import datetime
+
+    time.sleep(2)
+
+    cur_time = datetime.now().isoformat(timespec='seconds')
+
+    print("Starting storage benchmark '{}' on pool '{}'".format(cur_time, pool))
+
+    # Phase 0 - connect to databases
+    try:
+        db_conn, db_cur = open_database(config)
+    except Exception:
+        print('FATAL - failed to connect to Postgres')
+        raise Exception
+
+    try:
+        zk_conn = pvc_common.startZKConnection(config['coordinators'])
+    except Exception:
+        print('FATAL - failed to connect to Zookeeper')
+        raise Exception
+
+    print("Storing running status for job '{}' in database".format(cur_time))
+    try:
+        query = "INSERT INTO storage_benchmarks (job, result) VALUES (%s, %s);"
+        args = (cur_time, "Running",)
+        db_cur.execute(query, args)
+        db_conn.commit()
+    except Exception as e:
+        raise BenchmarkError("Failed to store running status: {}".format(e), cur_time=cur_time, db_conn=db_conn, db_cur=db_cur, zk_conn=zk_conn)
+
+    # Phase 1 - volume preparation
+    self.update_state(state='RUNNING', meta={'current': 1, 'total': 3, 'status': 'Creating benchmark volume'})
+    time.sleep(1)
+
+    volume = 'pvcbenchmark'
+
+    # Create the RBD volume
+    retcode, retmsg = pvc_ceph.add_volume(zk_conn, pool, volume, "8G")
+    if not retcode:
+        raise BenchmarkError('Failed to create volume "{}": {}'.format(volume, retmsg), cur_time=cur_time, db_conn=db_conn, db_cur=db_cur, zk_conn=zk_conn)
+    else:
+        print(retmsg)
+
+    # Phase 2 - benchmark run
+    self.update_state(state='RUNNING', meta={'current': 2, 'total': 3, 'status': 'Running fio benchmarks on volume'})
+    time.sleep(1)
+
+    # We run a total of 8 tests, to give a generalized idea of performance on the cluster:
+    #   1. A sequential read test of 8GB with a 4M block size
+    #   2. A sequential write test of 8GB with a 4M block size
+    #   3. A random read test of 8GB with a 4M block size
+    #   4. A random write test of 8GB with a 4M block size
+    #   5. A random read test of 8GB with a 256k block size
+    #   6. A random write test of 8GB with a 256k block size
+    #   7. A random read test of 8GB with a 4k block size
+    #   8. A random write test of 8GB with a 4k block size
+    # Taken together, these 8 results should give a very good indication of the overall storage performance
+    # for a variety of workloads.
+    test_matrix = {
+        'seq_read': {
+            'direction': 'read',
+            'bs': '4M',
+            'rw': 'read'
+        },
+        'seq_write': {
+            'direction': 'write',
+            'bs': '4M',
+            'rw': 'write'
+        },
+        'rand_read_4M': {
+            'direction': 'read',
+            'bs': '4M',
+            'rw': 'randread'
+        },
+        'rand_write_4M': {
+            'direction': 'write',
+            'bs': '4M',
+            'rw': 'randwrite'
+        },
+        'rand_read_256K': {
+            'direction': 'read',
+            'bs': '256K',
+            'rw': 'randread'
+        },
+        'rand_write_256K': {
+            'direction': 'write',
+            'bs': '256K',
+            'rw': 'randwrite'
+        },
+        'rand_read_4K': {
+            'direction': 'read',
+            'bs': '4K',
+            'rw': 'randread'
+        },
+        'rand_write_4K': {
+            'direction': 'write',
+            'bs': '4K',
+            'rw': 'randwrite'
+        }
+    }
+    parsed_results = dict()
+    for test in test_matrix:
+        print("Running test '{}'".format(test))
+        fio_cmd = """
+            fio \
+                --output-format=terse \
+                --terse-version=5 \
+                --ioengine=rbd \
+                --pool={pool} \
+                --rbdname={volume} \
+                --direct=1 \
+                --randrepeat=1 \
+                --iodepth=64 \
+                --size=8G \
+                --name={test} \
+                --bs={bs} \
+                --readwrite={rw}
+        """.format(
+            pool=pool,
+            volume=volume,
+            test=test,
+            bs=test_matrix[test]['bs'],
+            rw=test_matrix[test]['rw'])
+
+        retcode, stdout, stderr = pvc_common.run_os_command(fio_cmd)
+        if retcode:
+            raise BenchmarkError("Failed to run fio test: {}".format(stderr), cur_time=cur_time, db_conn=db_conn, db_cur=db_cur, zk_conn=zk_conn)
+
+        # Parse the terse results to avoid storing tons of junk
+        # Reference: https://fio.readthedocs.io/en/latest/fio_doc.html#terse-output
+        # This is written out broken up because the man page didn't bother to do this, and I'm putting it here for posterity.
+        # Example Read test (line breaks to match man ref):
+        #    I 5;fio-3.12;test;0;0; (5) [0, 1, 2, 3, 4]
+        #    R 8388608;2966268;724;2828; (4) [5, 6, 7, 8]
+        #      0;0;0.000000;0.000000; (4) [9, 10, 11, 12]
+        #      0;0;0.000000;0.000000; (4) [13, 14, 15, 16]
+        #      0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0; (20) [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,33, 34, 35, 36]
+        #      0;0;0.000000;0.000000; (4) [37, 38, 39, 40]
+        #      2842624;3153920;100.000000%;2967142.400000;127226.797479;5; (6) [41, 42, 43, 44, 45, 46]
+        #      694;770;724.400000;31.061230;5; (5) [47, 48, 49, 50, 51]
+        #    W 0;0;0;0; (4) [52, 53, 54, 55]
+        #      0;0;0.000000;0.000000; (4) [56, 57, 58, 59]
+        #      0;0;0.000000;0.000000; (4) [60, 61, 62, 63]
+        #      0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0; (20) [64, 65, 66, 67, 68. 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83]
+        #      0;0;0.000000;0.000000; (4) [84, 85, 86, 87]
+        #      0;0;0.000000%;0.000000;0.000000;0; (6) [88, 89, 90, 91, 92, 93]
+        #      0;0;0.000000;0.000000;0; (5) [94, 95, 96, 97, 98]
+        #    T 0;0;0;0; (4) [99, 100, 101, 102]
+        #      0;0;0.000000;0.000000; (4) [103, 104, 105, 106]
+        #      0;0;0.000000;0.000000; (4) [107, 108, 109, 110]
+        #      0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0; (20) [111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130]
+        #      0;0;0.000000;0.000000; (4) [131, 132, 133, 134]
+        #      0;0;0.000000%;0.000000;0.000000;0; (6) [135, 136, 137, 138, 139, 140]
+        #      0;0;0.000000;0.000000;0; (5) [141, 142, 143, 144, 145]
+        #    C 0.495225%;0.000000%;2083;0;13; (5) [146, 147, 148, 149, 150]
+        #    D 0.1%;0.1%;0.2%;0.4%;0.8%;1.6%;96.9%; (7) [151, 152, 153, 154, 155, 156, 157]
+        #    U 0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%; (10) [158, 159, 160, 161, 162, 163, 164, 165, 166, 167]
+        #    M 0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%; (12) [168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178. 179]
+        #    B dm-0;0;110;0;0;0;4;4;0.15%; (9) [180, 181, 182, 183, 184, 185, 186, 187, 188]
+        #      slaves;0;118;0;28;0;23;0;0.00%; (9) [189, 190, 191, 192, 193, 194, 195, 196, 197]
+        #      sde;0;118;0;28;0;23;0;0.00% (9) [198, 199, 200, 201, 202, 203, 204, 205, 206]
+        # Example Write test:
+        #    I 5;fio-3.12;test;0;0; (5)
+        #    R 0;0;0;0; (4)
+        #      0;0;0.000000;0.000000; (4)
+        #      0;0;0.000000;0.000000; (4)
+        #      0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0; (20)
+        #      0;0;0.000000;0.000000; (4)
+        #      0;0;0.000000%;0.000000;0.000000;0; (6)
+        #      0;0;0.000000;0.000000;0; (5)
+        #    W 8388608;1137438;277;7375; (4)
+        #      0;0;0.000000;0.000000; (4)
+        #      0;0;0.000000;0.000000; (4)
+        #      0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0; (20)
+        #      0;0;0.000000;0.000000; (4)
+        #      704512;1400832;99.029573%;1126400.000000;175720.860374;14; (6)
+        #      172;342;275.000000;42.900601;14; (5)
+        #    T 0;0;0;0; (4)
+        #      0;0;0.000000;0.000000; (4)
+        #      0;0;0.000000;0.000000; (4)
+        #      0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0;0%=0; (20)
+        #      0;0;0.000000;0.000000; (4)
+        #      0;0;0.000000%;0.000000;0.000000;0; (6)
+        #      0;0;0.000000;0.000000;0; (5)
+        #    C 12.950909%;1.912124%;746;0;95883; (5)
+        #    D 0.1%;0.1%;0.2%;0.4%;0.8%;1.6%;96.9%; (7)
+        #    U 0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%; (10)
+        #    M 0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%; (12)
+        #    B dm-0;0;196;0;0;0;12;12;0.16%; (9)
+        #      slaves;0;207;0;95;0;39;16;0.21%; (9)
+        #      sde;0;207;0;95;0;39;16;0.21% (9)
+        results = stdout.split(';')
+        if test_matrix[test]['direction'] == 'read':
+            # Stats
+            #         5:   Total IO (KiB)
+            #         6:   bandwidth (KiB/sec)
+            #         7:   IOPS
+            #         8:   runtime (msec)
+            # Total latency
+            #         37:  min
+            #         38:  max
+            #         39:  mean
+            #         40:  stdev
+            # Bandwidth
+            #         41:  min
+            #         42:  max
+            #         44:  mean
+            #         45:  stdev
+            #         46:  # samples
+            # IOPS
+            #         47:  min
+            #         48:  max
+            #         49:  mean
+            #         50:  stdev
+            #         51:  # samples
+            # CPU
+            #         146: user
+            #         147: system
+            #         148: ctx switches
+            #         149: maj faults
+            #         150: min faults
+            parsed_results[test] = {
+                "overall": {
+                    "iosize": results[5],
+                    "bandwidth": results[6],
+                    "iops": results[7],
+                    "runtime": results[8]
+                },
+                "latency": {
+                    "min": results[37],
+                    "max": results[38],
+                    "mean": results[39],
+                    "stdev": results[40]
+                },
+                "bandwidth": {
+                    "min": results[41],
+                    "max": results[42],
+                    "mean": results[44],
+                    "stdev": results[45],
+                    "numsamples": results[46],
+                },
+                "iops": {
+                    "min": results[47],
+                    "max": results[48],
+                    "mean": results[49],
+                    "stdev": results[50],
+                    "numsamples": results[51]
+                },
+                "cpu": {
+                    "user": results[146],
+                    "system": results[147],
+                    "ctxsw": results[148],
+                    "majfault": results[149],
+                    "minfault": results[150]
+                }
+            }
+
+        if test_matrix[test]['direction'] == 'write':
+            # Stats
+            #         52:  Total IO (KiB)
+            #         53:  bandwidth (KiB/sec)
+            #         54:  IOPS
+            #         55:  runtime (msec)
+            # Total latency
+            #         84:  min
+            #         85:  max
+            #         86:  mean
+            #         87:  stdev
+            # Bandwidth
+            #         88:  min
+            #         89:  max
+            #         91:  mean
+            #         92:  stdev
+            #         93:  # samples
+            # IOPS
+            #         94:  min
+            #         95:  max
+            #         96:  mean
+            #         97:  stdev
+            #         98:  # samples
+            # CPU
+            #         146: user
+            #         147: system
+            #         148: ctx switches
+            #         149: maj faults
+            #         150: min faults
+            parsed_results[test] = {
+                "overall": {
+                    "iosize": results[52],
+                    "bandwidth": results[53],
+                    "iops": results[54],
+                    "runtime": results[55]
+                },
+                "latency": {
+                    "min": results[84],
+                    "max": results[85],
+                    "mean": results[86],
+                    "stdev": results[87]
+                },
+                "bandwidth": {
+                    "min": results[88],
+                    "max": results[89],
+                    "mean": results[91],
+                    "stdev": results[92],
+                    "numsamples": results[93],
+                },
+                "iops": {
+                    "min": results[94],
+                    "max": results[95],
+                    "mean": results[96],
+                    "stdev": results[97],
+                    "numsamples": results[98]
+                },
+                "cpu": {
+                    "user": results[146],
+                    "system": results[147],
+                    "ctxsw": results[148],
+                    "majfault": results[149],
+                    "minfault": results[150]
+                }
+            }
+
+    # Phase 3 - cleanup
+    self.update_state(state='RUNNING', meta={'current': 3, 'total': 3, 'status': 'Cleaning up and storing results'})
+    time.sleep(1)
+
+    # Remove the RBD volume
+    retcode, retmsg = pvc_ceph.remove_volume(zk_conn, pool, volume)
+    if not retcode:
+        raise BenchmarkError('Failed to remove volume "{}": {}'.format(volume, retmsg), cur_time=cur_time, db_conn=db_conn, db_cur=db_cur, zk_conn=zk_conn)
+    else:
+        print(retmsg)
+
+    print("Storing result of tests for job '{}' in database".format(cur_time))
+    try:
+        query = "UPDATE storage_benchmarks SET result = %s WHERE job = %s;"
+        args = (json.dumps(parsed_results), cur_time)
+        db_cur.execute(query, args)
+        db_conn.commit()
+    except Exception as e:
+        raise BenchmarkError("Failed to store test results: {}".format(e), cur_time=cur_time, db_conn=db_conn, db_cur=db_cur, zk_conn=zk_conn)
+
+    close_database(db_conn, db_cur)
+    pvc_common.stopZKConnection(zk_conn)
+    return {'status': "Storage benchmark '{}' completed successfully.", 'current': 3, 'total': 3}

api-daemon/pvcapid/helper.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# pvcapi_helper.py - PVC HTTP API functions
+# helper.py - PVC HTTP API helper functions
 # Part of the Parallel Virtual Cluster (PVC) system
 #
 #    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
@@ -21,17 +21,32 @@
 ###############################################################################
 
 import flask
-import json
 import lxml.etree as etree
 
-from distutils.util import strtobool
+from distutils.util import strtobool as dustrtobool
+
+from werkzeug.formparser import parse_form_data
+
+import daemon_lib.common as pvc_common
+import daemon_lib.cluster as pvc_cluster
+import daemon_lib.node as pvc_node
+import daemon_lib.vm as pvc_vm
+import daemon_lib.network as pvc_network
+import daemon_lib.ceph as pvc_ceph
+
+config = None  # Set in this namespace by flaskapi
+
+
+def strtobool(stringv):
+    if stringv is None:
+        return False
+    if isinstance(stringv, bool):
+        return bool(stringv)
+    try:
+        return bool(dustrtobool(stringv))
+    except Exception:
+        return False
 
-import client_lib.common as pvc_common
-import client_lib.cluster as pvc_cluster
-import client_lib.node as pvc_node
-import client_lib.vm as pvc_vm
-import client_lib.network as pvc_network
-import client_lib.ceph as pvc_ceph
 
 #
 # Initialization function
@@ -70,6 +85,7 @@ def initialize_cluster():
 
     return True
 
+
 #
 # Cluster functions
 #
@@ -83,6 +99,7 @@ def cluster_status():
 
     return retdata, 200
 
+
 def cluster_maintenance(maint_state='false'):
     """
     Set the cluster in or out of maintenance state
@@ -101,15 +118,16 @@ def cluster_maintenance(maint_state='false'):
 
     return retdata, retcode
 
+
 #
 # Node functions
 #
-def node_list(limit=None, is_fuzzy=True):
+def node_list(limit=None, daemon_state=None, coordinator_state=None, domain_state=None, is_fuzzy=True):
     """
     Return a list of nodes with limit LIMIT.
     """
     zk_conn = pvc_common.startZKConnection(config['coordinators'])
-    retflag, retdata = pvc_node.get_list(zk_conn, limit, is_fuzzy=is_fuzzy)
+    retflag, retdata = pvc_node.get_list(zk_conn, limit, daemon_state=daemon_state, coordinator_state=coordinator_state, domain_state=domain_state, is_fuzzy=is_fuzzy)
     pvc_common.stopZKConnection(zk_conn)
 
     if retflag:
@@ -132,6 +150,7 @@ def node_list(limit=None, is_fuzzy=True):
 
     return retdata, retcode
 
+
 def node_daemon_state(node):
     """
     Return the daemon state of node NODE.
@@ -160,6 +179,7 @@ def node_daemon_state(node):
 
     return retdata, retcode
 
+
 def node_coordinator_state(node):
     """
     Return the coordinator state of node NODE.
@@ -188,6 +208,7 @@ def node_coordinator_state(node):
 
     return retdata, retcode
 
+
 def node_domain_state(node):
     """
     Return the domain state of node NODE.
@@ -213,11 +234,12 @@ def node_domain_state(node):
 
     return retdata, retcode
 
+
 def node_secondary(node):
     """
     Take NODE out of primary router mode.
     """
-    zk_conn = pvc_common.startZKConnection(config['coordinators']) 
+    zk_conn = pvc_common.startZKConnection(config['coordinators'])
     retflag, retdata = pvc_node.secondary_node(zk_conn, node)
     pvc_common.stopZKConnection(zk_conn)
 
@@ -231,11 +253,12 @@ def node_secondary(node):
     }
     return output, retcode
 
+
 def node_primary(node):
     """
     Set NODE to primary router mode.
     """
-    zk_conn = pvc_common.startZKConnection(config['coordinators']) 
+    zk_conn = pvc_common.startZKConnection(config['coordinators'])
     retflag, retdata = pvc_node.primary_node(zk_conn, node)
     pvc_common.stopZKConnection(zk_conn)
 
@@ -249,6 +272,7 @@ def node_primary(node):
     }
     return output, retcode
 
+
 def node_flush(node, wait):
     """
     Flush NODE of running VMs.
@@ -267,6 +291,7 @@ def node_flush(node, wait):
     }
     return output, retcode
 
+
 def node_ready(node, wait):
     """
     Restore NODE to active service.
@@ -285,6 +310,7 @@ def node_ready(node, wait):
     }
     return output, retcode
 
+
 #
 # VM functions
 #
@@ -298,6 +324,7 @@ def vm_is_migrated(vm):
 
     return retdata
 
+
 def vm_state(vm):
     """
     Return the state of virtual machine VM.
@@ -330,6 +357,7 @@ def vm_state(vm):
 
     return retdata, retcode
 
+
 def vm_node(vm):
     """
     Return the current node of virtual machine VM.
@@ -363,6 +391,7 @@ def vm_node(vm):
 
     return retdata, retcode
 
+
 def vm_console(vm, lines=None):
     """
     Return the current console log for VM.
@@ -391,6 +420,7 @@ def vm_console(vm, lines=None):
 
     return retdata, retcode
 
+
 def vm_list(node=None, state=None, limit=None, is_fuzzy=True):
     """
     Return a list of VMs with limit LIMIT.
@@ -419,7 +449,8 @@ def vm_list(node=None, state=None, limit=None, is_fuzzy=True):
 
     return retdata, retcode
 
-def vm_define(xml, node, limit, selector, autostart):
+
+def vm_define(xml, node, limit, selector, autostart, migration_method):
     """
     Define a VM from Libvirt XML in the PVC cluster.
     """
@@ -428,10 +459,10 @@ def vm_define(xml, node, limit, selector, autostart):
         xml_data = etree.fromstring(xml)
         new_cfg = etree.tostring(xml_data, pretty_print=True).decode('utf8')
     except Exception as e:
-        return {'message': 'Error: XML is malformed or incorrect: {}'.format(e)}, 400
+        return {'message': 'XML is malformed or incorrect: {}'.format(e)}, 400
 
     zk_conn = pvc_common.startZKConnection(config['coordinators'])
-    retflag, retdata = pvc_vm.define_vm(zk_conn, new_cfg, node, limit, selector, autostart, profile=None)
+    retflag, retdata = pvc_vm.define_vm(zk_conn, new_cfg, node, limit, selector, autostart, migration_method, profile=None)
     pvc_common.stopZKConnection(zk_conn)
 
     if retflag:
@@ -444,6 +475,7 @@ def vm_define(xml, node, limit, selector, autostart):
     }
     return output, retcode
 
+
 def get_vm_meta(vm):
     """
     Get metadata of a VM.
@@ -463,7 +495,8 @@ def get_vm_meta(vm):
                 'name': vm,
                 'node_limit': retdata['node_limit'],
                 'node_selector': retdata['node_selector'],
-                'node_autostart': retdata['node_autostart']
+                'node_autostart': retdata['node_autostart'],
+                'migration_method': retdata['migration_method']
             }
         else:
             retcode = 404
@@ -478,7 +511,8 @@ def get_vm_meta(vm):
 
     return retdata, retcode
 
-def update_vm_meta(vm, limit, selector, autostart, provisioner_profile):
+
+def update_vm_meta(vm, limit, selector, autostart, provisioner_profile, migration_method):
     """
     Update metadata of a VM.
     """
@@ -486,9 +520,9 @@ def update_vm_meta(vm, limit, selector, autostart, provisioner_profile):
     if autostart is not None:
         try:
             autostart = bool(strtobool(autostart))
-        except:
+        except Exception:
             autostart = False
-    retflag, retdata = pvc_vm.modify_vm_metadata(zk_conn, vm, limit, selector, autostart, provisioner_profile)
+    retflag, retdata = pvc_vm.modify_vm_metadata(zk_conn, vm, limit, selector, autostart, provisioner_profile, migration_method)
     pvc_common.stopZKConnection(zk_conn)
 
     if retflag:
@@ -501,6 +535,7 @@ def update_vm_meta(vm, limit, selector, autostart, provisioner_profile):
     }
     return output, retcode
 
+
 def vm_modify(name, restart, xml):
     """
     Modify a VM Libvirt XML in the PVC cluster.
@@ -510,7 +545,7 @@ def vm_modify(name, restart, xml):
         xml_data = etree.fromstring(xml)
         new_cfg = etree.tostring(xml_data, pretty_print=True).decode('utf8')
     except Exception as e:
-        return {'message': 'Error: XML is malformed or incorrect: {}'.format(e)}, 400
+        return {'message': 'XML is malformed or incorrect: {}'.format(e)}, 400
     zk_conn = pvc_common.startZKConnection(config['coordinators'])
     retflag, retdata = pvc_vm.modify_vm(zk_conn, name, restart, new_cfg)
     pvc_common.stopZKConnection(zk_conn)
@@ -525,6 +560,7 @@ def vm_modify(name, restart, xml):
     }
     return output, retcode
 
+
 def vm_undefine(name):
     """
     Undefine a VM from the PVC cluster.
@@ -543,6 +579,7 @@ def vm_undefine(name):
     }
     return output, retcode
 
+
 def vm_remove(name):
     """
     Remove a VM from the PVC cluster.
@@ -561,6 +598,7 @@ def vm_remove(name):
     }
     return output, retcode
 
+
 def vm_start(name):
     """
     Start a VM in the PVC cluster.
@@ -579,12 +617,13 @@ def vm_start(name):
     }
     return output, retcode
 
-def vm_restart(name):
+
+def vm_restart(name, wait):
     """
     Restart a VM in the PVC cluster.
     """
     zk_conn = pvc_common.startZKConnection(config['coordinators'])
-    retflag, retdata = pvc_vm.restart_vm(zk_conn, name)
+    retflag, retdata = pvc_vm.restart_vm(zk_conn, name, wait)
     pvc_common.stopZKConnection(zk_conn)
 
     if retflag:
@@ -597,12 +636,13 @@ def vm_restart(name):
     }
     return output, retcode
 
-def vm_shutdown(name):
+
+def vm_shutdown(name, wait):
     """
     Shutdown a VM in the PVC cluster.
     """
     zk_conn = pvc_common.startZKConnection(config['coordinators'])
-    retflag, retdata = pvc_vm.shutdown_vm(zk_conn, name)
+    retflag, retdata = pvc_vm.shutdown_vm(zk_conn, name, wait)
     pvc_common.stopZKConnection(zk_conn)
 
     if retflag:
@@ -615,6 +655,7 @@ def vm_shutdown(name):
     }
     return output, retcode
 
+
 def vm_stop(name):
     """
     Forcibly stop a VM in the PVC cluster.
@@ -633,6 +674,7 @@ def vm_stop(name):
     }
     return output, retcode
 
+
 def vm_disable(name):
     """
     Disable a (stopped) VM in the PVC cluster.
@@ -651,12 +693,13 @@ def vm_disable(name):
     }
     return output, retcode
 
-def vm_move(name, node):
+
+def vm_move(name, node, wait, force_live):
     """
     Move a VM to another node.
     """
     zk_conn = pvc_common.startZKConnection(config['coordinators'])
-    retflag, retdata = pvc_vm.move_vm(zk_conn, name, node)
+    retflag, retdata = pvc_vm.move_vm(zk_conn, name, node, wait, force_live)
     pvc_common.stopZKConnection(zk_conn)
 
     if retflag:
@@ -669,12 +712,13 @@ def vm_move(name, node):
     }
     return output, retcode
 
-def vm_migrate(name, node, flag_force):
+
+def vm_migrate(name, node, flag_force, wait, force_live):
     """
     Temporarily migrate a VM to another node.
     """
     zk_conn = pvc_common.startZKConnection(config['coordinators'])
-    retflag, retdata = pvc_vm.migrate_vm(zk_conn, name, node, flag_force)
+    retflag, retdata = pvc_vm.migrate_vm(zk_conn, name, node, flag_force, wait, force_live)
     pvc_common.stopZKConnection(zk_conn)
 
     if retflag:
@@ -687,12 +731,13 @@ def vm_migrate(name, node, flag_force):
     }
     return output, retcode
 
-def vm_unmigrate(name):
+
+def vm_unmigrate(name, wait, force_live):
     """
     Unmigrate a migrated VM.
     """
     zk_conn = pvc_common.startZKConnection(config['coordinators'])
-    retflag, retdata = pvc_vm.unmigrate_vm(zk_conn, name)
+    retflag, retdata = pvc_vm.unmigrate_vm(zk_conn, name, wait, force_live)
     pvc_common.stopZKConnection(zk_conn)
 
     if retflag:
@@ -705,6 +750,7 @@ def vm_unmigrate(name):
     }
     return output, retcode
 
+
 def vm_flush_locks(vm):
     """
     Flush locks of a (stopped) VM.
@@ -718,7 +764,7 @@ def vm_flush_locks(vm):
         retdata = retdata[0]
 
     if retdata['state'] not in ['stop', 'disable']:
-        return {"message":"VM must be stopped to flush locks"}, 400
+        return {"message": "VM must be stopped to flush locks"}, 400
 
     zk_conn = pvc_common.startZKConnection(config['coordinators'])
     retflag, retdata = pvc_vm.flush_locks(zk_conn, vm)
@@ -734,6 +780,7 @@ def vm_flush_locks(vm):
     }
     return output, retcode
 
+
 #
 # Network functions
 #
@@ -765,6 +812,7 @@ def net_list(limit=None, is_fuzzy=True):
 
     return retdata, retcode
 
+
 def net_add(vni, description, nettype, domain, name_servers,
             ip4_network, ip4_gateway, ip6_network, ip6_gateway,
             dhcp4_flag, dhcp4_start, dhcp4_end):
@@ -775,8 +823,8 @@ def net_add(vni, description, nettype, domain, name_servers,
         dhcp4_flag = bool(strtobool(dhcp4_flag))
     zk_conn = pvc_common.startZKConnection(config['coordinators'])
     retflag, retdata = pvc_network.add_network(zk_conn, vni, description, nettype, domain, name_servers,
-                                              ip4_network, ip4_gateway, ip6_network, ip6_gateway,
-                                              dhcp4_flag, dhcp4_start, dhcp4_end)
+                                               ip4_network, ip4_gateway, ip6_network, ip6_gateway,
+                                               dhcp4_flag, dhcp4_start, dhcp4_end)
     pvc_common.stopZKConnection(zk_conn)
 
     if retflag:
@@ -789,6 +837,7 @@ def net_add(vni, description, nettype, domain, name_servers,
     }
     return output, retcode
 
+
 def net_modify(vni, description, domain, name_servers,
                ip4_network, ip4_gateway,
                ip6_network, ip6_gateway,
@@ -800,8 +849,8 @@ def net_modify(vni, description, domain, name_servers,
         dhcp4_flag = bool(strtobool(dhcp4_flag))
     zk_conn = pvc_common.startZKConnection(config['coordinators'])
     retflag, retdata = pvc_network.modify_network(zk_conn, vni, description, domain, name_servers,
-                                              ip4_network, ip4_gateway, ip6_network, ip6_gateway,
-                                              dhcp4_flag, dhcp4_start, dhcp4_end)
+                                                  ip4_network, ip4_gateway, ip6_network, ip6_gateway,
+                                                  dhcp4_flag, dhcp4_start, dhcp4_end)
     pvc_common.stopZKConnection(zk_conn)
 
     if retflag:
@@ -814,6 +863,7 @@ def net_modify(vni, description, domain, name_servers,
     }
     return output, retcode
 
+
 def net_remove(network):
     """
     Remove a virtual client network from the PVC cluster.
@@ -832,6 +882,7 @@ def net_remove(network):
     }
     return output, retcode
 
+
 def net_dhcp_list(network, limit=None, static=False):
     """
     Return a list of DHCP leases in network NETWORK with limit LIMIT.
@@ -856,6 +907,7 @@ def net_dhcp_list(network, limit=None, static=False):
 
     return retdata, retcode
 
+
 def net_dhcp_add(network, ipaddress, macaddress, hostname):
     """
     Add a static DHCP lease to a virtual client network.
@@ -874,6 +926,7 @@ def net_dhcp_add(network, ipaddress, macaddress, hostname):
     }
     return output, retcode
 
+
 def net_dhcp_remove(network, macaddress):
     """
     Remove a static DHCP lease from a virtual client network.
@@ -892,6 +945,7 @@ def net_dhcp_remove(network, macaddress):
     }
     return output, retcode
 
+
 def net_acl_list(network, limit=None, direction=None, is_fuzzy=True):
     """
     Return a list of network ACLs in network NETWORK with limit LIMIT.
@@ -920,6 +974,7 @@ def net_acl_list(network, limit=None, direction=None, is_fuzzy=True):
 
     return retdata, retcode
 
+
 def net_acl_add(network, direction, description, rule, order):
     """
     Add an ACL to a virtual client network.
@@ -938,6 +993,7 @@ def net_acl_add(network, direction, description, rule, order):
     }
     return output, retcode
 
+
 def net_acl_remove(network, description):
     """
     Remove an ACL from a virtual client network.
@@ -956,6 +1012,7 @@ def net_acl_remove(network, description):
     }
     return output, retcode
 
+
 #
 # Ceph functions
 #
@@ -974,12 +1031,13 @@ def ceph_status():
 
     return retdata, retcode
 
-def ceph_radosdf():
+
+def ceph_util():
     """
     Get the current Ceph cluster utilization.
     """
     zk_conn = pvc_common.startZKConnection(config['coordinators'])
-    retflag, retdata = pvc_ceph.get_radosdf(zk_conn)
+    retflag, retdata = pvc_ceph.get_util(zk_conn)
     pvc_common.stopZKConnection(zk_conn)
 
     if retflag:
@@ -989,6 +1047,7 @@ def ceph_radosdf():
 
     return retdata, retcode
 
+
 def ceph_osd_list(limit=None):
     """
     Get the list of OSDs in the Ceph storage cluster.
@@ -1013,6 +1072,7 @@ def ceph_osd_list(limit=None):
 
     return retdata, retcode
 
+
 def ceph_osd_state(osd):
     zk_conn = pvc_common.startZKConnection(config['coordinators'])
     retflag, retdata = pvc_ceph.get_list_osd(zk_conn, osd)
@@ -1035,7 +1095,8 @@ def ceph_osd_state(osd):
     in_state = retdata[0]['stats']['in']
     up_state = retdata[0]['stats']['up']
 
-    return { "id": osd, "in": in_state, "up": up_state }, retcode
+    return {"id": osd, "in": in_state, "up": up_state}, retcode
+
 
 def ceph_osd_add(node, device, weight):
     """
@@ -1055,6 +1116,7 @@ def ceph_osd_add(node, device, weight):
     }
     return output, retcode
 
+
 def ceph_osd_remove(osd_id):
     """
     Remove a Ceph OSD from the PVC Ceph storage cluster.
@@ -1073,6 +1135,7 @@ def ceph_osd_remove(osd_id):
     }
     return output, retcode
 
+
 def ceph_osd_in(osd_id):
     """
     Set in a Ceph OSD in the PVC Ceph storage cluster.
@@ -1091,6 +1154,7 @@ def ceph_osd_in(osd_id):
     }
     return output, retcode
 
+
 def ceph_osd_out(osd_id):
     """
     Set out a Ceph OSD in the PVC Ceph storage cluster.
@@ -1109,6 +1173,7 @@ def ceph_osd_out(osd_id):
     }
     return output, retcode
 
+
 def ceph_osd_set(option):
     """
     Set options on a Ceph OSD in the PVC Ceph storage cluster.
@@ -1127,6 +1192,7 @@ def ceph_osd_set(option):
     }
     return output, retcode
 
+
 def ceph_osd_unset(option):
     """
     Unset options on a Ceph OSD in the PVC Ceph storage cluster.
@@ -1145,6 +1211,7 @@ def ceph_osd_unset(option):
     }
     return output, retcode
 
+
 def ceph_pool_list(limit=None, is_fuzzy=True):
     """
     Get the list of RBD pools in the Ceph storage cluster.
@@ -1173,6 +1240,7 @@ def ceph_pool_list(limit=None, is_fuzzy=True):
 
     return retdata, retcode
 
+
 def ceph_pool_add(name, pgs, replcfg):
     """
     Add a Ceph RBD pool to the PVC Ceph storage cluster.
@@ -1191,6 +1259,7 @@ def ceph_pool_add(name, pgs, replcfg):
     }
     return output, retcode
 
+
 def ceph_pool_remove(name):
     """
     Remove a Ceph RBD pool to the PVC Ceph storage cluster.
@@ -1209,6 +1278,7 @@ def ceph_pool_remove(name):
     }
     return output, retcode
 
+
 def ceph_volume_list(pool=None, limit=None, is_fuzzy=True):
     """
     Get the list of RBD volumes in the Ceph storage cluster.
@@ -1237,6 +1307,7 @@ def ceph_volume_list(pool=None, limit=None, is_fuzzy=True):
 
     return retdata, retcode
 
+
 def ceph_volume_add(pool, name, size):
     """
     Add a Ceph RBD volume to the PVC Ceph storage cluster.
@@ -1255,6 +1326,7 @@ def ceph_volume_add(pool, name, size):
     }
     return output, retcode
 
+
 def ceph_volume_clone(pool, name, source_volume):
     """
     Clone a Ceph RBD volume to a new volume on the PVC Ceph storage cluster.
@@ -1273,6 +1345,7 @@ def ceph_volume_clone(pool, name, source_volume):
     }
     return output, retcode
 
+
 def ceph_volume_resize(pool, name, size):
     """
     Resize an existing Ceph RBD volume in the PVC Ceph storage cluster.
@@ -1291,6 +1364,7 @@ def ceph_volume_resize(pool, name, size):
     }
     return output, retcode
 
+
 def ceph_volume_rename(pool, name, new_name):
     """
     Rename a Ceph RBD volume in the PVC Ceph storage cluster.
@@ -1309,6 +1383,7 @@ def ceph_volume_rename(pool, name, new_name):
     }
     return output, retcode
 
+
 def ceph_volume_remove(pool, name):
     """
     Remove a Ceph RBD volume to the PVC Ceph storage cluster.
@@ -1327,6 +1402,158 @@ def ceph_volume_remove(pool, name):
     }
     return output, retcode
 
+
+def ceph_volume_upload(pool, volume, img_type):
+    """
+    Upload a raw file via HTTP post to a PVC Ceph volume
+    """
+    # Determine the image conversion options
+    if img_type not in ['raw', 'vmdk', 'qcow2', 'qed', 'vdi', 'vpc']:
+        output = {
+            "message": "Image type '{}' is not valid.".format(img_type)
+        }
+        retcode = 400
+        return output, retcode
+
+    # Get the size of the target block device
+    zk_conn = pvc_common.startZKConnection(config['coordinators'])
+    retcode, retdata = pvc_ceph.get_list_volume(zk_conn, pool, volume, is_fuzzy=False)
+    pvc_common.stopZKConnection(zk_conn)
+    # If there's no target, return failure
+    if not retcode or len(retdata) < 1:
+        output = {
+            "message": "Target volume '{}' does not exist in pool '{}'.".format(volume, pool)
+        }
+        retcode = 400
+        return output, retcode
+    dev_size = retdata[0]['stats']['size']
+
+    def cleanup_maps_and_volumes():
+        zk_conn = pvc_common.startZKConnection(config['coordinators'])
+        # Unmap the target blockdev
+        retflag, retdata = pvc_ceph.unmap_volume(zk_conn, pool, volume)
+        # Unmap the temporary blockdev
+        retflag, retdata = pvc_ceph.unmap_volume(zk_conn, pool, "{}_tmp".format(volume))
+        # Remove the temporary blockdev
+        retflag, retdata = pvc_ceph.remove_volume(zk_conn, pool, "{}_tmp".format(volume))
+        pvc_common.stopZKConnection(zk_conn)
+
+    # Create a temporary block device to store non-raw images
+    if img_type == 'raw':
+        # Map the target blockdev
+        zk_conn = pvc_common.startZKConnection(config['coordinators'])
+        retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, volume)
+        pvc_common.stopZKConnection(zk_conn)
+        if not retflag:
+            output = {
+                'message': retdata.replace('\"', '\'')
+            }
+            retcode = 400
+            cleanup_maps_and_volumes()
+            return output, retcode
+        dest_blockdev = retdata
+
+        # Save the data to the blockdev directly
+        try:
+            # This sets up a custom stream_factory that writes directly into the ova_blockdev,
+            # rather than the standard stream_factory which writes to a temporary file waiting
+            # on a save() call. This will break if the API ever uploaded multiple files, but
+            # this is an acceptable workaround.
+            def image_stream_factory(total_content_length, filename, content_type, content_length=None):
+                return open(dest_blockdev, 'wb')
+            parse_form_data(flask.request.environ, stream_factory=image_stream_factory)
+        except Exception:
+            output = {
+                'message': "Failed to upload or write image file to temporary volume."
+            }
+            retcode = 400
+            cleanup_maps_and_volumes()
+            return output, retcode
+
+        output = {
+            'message': "Wrote uploaded file to volume '{}' in pool '{}'.".format(volume, pool)
+        }
+        retcode = 200
+        cleanup_maps_and_volumes()
+        return output, retcode
+
+    # Write the image directly to the blockdev
+    else:
+        # Create a temporary blockdev
+        zk_conn = pvc_common.startZKConnection(config['coordinators'])
+        retflag, retdata = pvc_ceph.add_volume(zk_conn, pool, "{}_tmp".format(volume), dev_size)
+        pvc_common.stopZKConnection(zk_conn)
+        if not retflag:
+            output = {
+                'message': retdata.replace('\"', '\'')
+            }
+            retcode = 400
+            cleanup_maps_and_volumes()
+            return output, retcode
+
+        # Map the temporary target blockdev
+        zk_conn = pvc_common.startZKConnection(config['coordinators'])
+        retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, "{}_tmp".format(volume))
+        pvc_common.stopZKConnection(zk_conn)
+        if not retflag:
+            output = {
+                'message': retdata.replace('\"', '\'')
+            }
+            retcode = 400
+            cleanup_maps_and_volumes()
+            return output, retcode
+        temp_blockdev = retdata
+
+        # Map the target blockdev
+        zk_conn = pvc_common.startZKConnection(config['coordinators'])
+        retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, volume)
+        pvc_common.stopZKConnection(zk_conn)
+        if not retflag:
+            output = {
+                'message': retdata.replace('\"', '\'')
+            }
+            retcode = 400
+            cleanup_maps_and_volumes()
+            return output, retcode
+        dest_blockdev = retdata
+
+        # Save the data to the temporary blockdev directly
+        try:
+            # This sets up a custom stream_factory that writes directly into the ova_blockdev,
+            # rather than the standard stream_factory which writes to a temporary file waiting
+            # on a save() call. This will break if the API ever uploaded multiple files, but
+            # this is an acceptable workaround.
+            def image_stream_factory(total_content_length, filename, content_type, content_length=None):
+                return open(temp_blockdev, 'wb')
+            parse_form_data(flask.request.environ, stream_factory=image_stream_factory)
+        except Exception:
+            output = {
+                'message': "Failed to upload or write image file to temporary volume."
+            }
+            retcode = 400
+            cleanup_maps_and_volumes()
+            return output, retcode
+
+        # Convert from the temporary to destination format on the blockdevs
+        retcode, stdout, stderr = pvc_common.run_os_command(
+            'qemu-img convert -C -f {} -O raw {} {}'.format(img_type, temp_blockdev, dest_blockdev)
+        )
+        if retcode:
+            output = {
+                'message': "Failed to convert image format from '{}' to 'raw': {}".format(img_type, stderr)
+            }
+            retcode = 400
+            cleanup_maps_and_volumes()
+            return output, retcode
+
+        output = {
+            'message': "Converted and wrote uploaded file to volume '{}' in pool '{}'.".format(volume, pool)
+        }
+        retcode = 200
+        cleanup_maps_and_volumes()
+        return output, retcode
+
+
 def ceph_volume_snapshot_list(pool=None, volume=None, limit=None, is_fuzzy=True):
     """
     Get the list of RBD volume snapshots in the Ceph storage cluster.
@@ -1355,6 +1582,7 @@ def ceph_volume_snapshot_list(pool=None, volume=None, limit=None, is_fuzzy=True)
 
     return retdata, retcode
 
+
 def ceph_volume_snapshot_add(pool, volume, name):
     """
     Add a Ceph RBD volume snapshot to the PVC Ceph storage cluster.
@@ -1373,6 +1601,7 @@ def ceph_volume_snapshot_add(pool, volume, name):
     }
     return output, retcode
 
+
 def ceph_volume_snapshot_rename(pool, volume, name, new_name):
     """
     Rename a Ceph RBD volume snapshot in the PVC Ceph storage cluster.
@@ -1391,6 +1620,7 @@ def ceph_volume_snapshot_rename(pool, volume, name, new_name):
     }
     return output, retcode
 
+
 def ceph_volume_snapshot_remove(pool, volume, name):
     """
     Remove a Ceph RBD volume snapshot from the PVC Ceph storage cluster.
@@ -1408,4 +1638,3 @@ def ceph_volume_snapshot_remove(pool, volume, name):
         'message': retdata.replace('\"', '\'')
     }
     return output, retcode
-

api-daemon/pvcapid/libvirt_schema.py

@@ -53,6 +53,7 @@ libvirt_header = """<domain type='kvm'>
   <on_reboot>restart</on_reboot>
   <on_crash>restart</on_crash>
   <devices>
+    <console type='pty'/>
 """
 
 # File footer, closing devices and domain elements
@@ -75,7 +76,6 @@ devices_default = """    <emulator>/usr/bin/kvm</emulator>
 devices_serial = """    <serial type='pty'>
       <log file='/var/log/libvirt/{vm_name}.log' append='on'/>
     </serial>
-    <console type='pty'/>
 """
 
 # VNC device
@@ -119,7 +119,7 @@ devices_disk_footer = """      </source>
 
 # vhostmd virtualization passthrough device
 devices_vhostmd = """    <disk type='file' device='disk'>
-      <drive name='qemu' type='raw'/>
+      <driver name='qemu' type='raw'/>
       <source file='/dev/shm/vhostmd0'/>
       <target dev='sdz' bus='usb'/>
       <readonly/>

api-daemon/pvcapid/models.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+
+# models.py - PVC Database models
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from pvcapid.flaskapi import db
+
+
+class DBSystemTemplate(db.Model):
+    __tablename__ = 'system_template'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.Text, nullable=False, unique=True)
+    vcpu_count = db.Column(db.Integer, nullable=False)
+    vram_mb = db.Column(db.Integer, nullable=False)
+    serial = db.Column(db.Boolean, nullable=False)
+    vnc = db.Column(db.Boolean, nullable=False)
+    vnc_bind = db.Column(db.Text)
+    node_limit = db.Column(db.Text)
+    node_selector = db.Column(db.Text)
+    node_autostart = db.Column(db.Boolean, nullable=False)
+    migration_method = db.Column(db.Text)
+    ova = db.Column(db.Integer, db.ForeignKey("ova.id"), nullable=True)
+
+    def __init__(self, name, vcpu_count, vram_mb, serial, vnc, vnc_bind, node_limit, node_selector, node_autostart, migration_method, ova=None):
+        self.name = name
+        self.vcpu_count = vcpu_count
+        self.vram_mb = vram_mb
+        self.serial = serial
+        self.vnc = vnc
+        self.vnc_bind = vnc_bind
+        self.node_limit = node_limit
+        self.node_selector = node_selector
+        self.node_autostart = node_autostart
+        self.migration_method = migration_method
+        self.ova = ova
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+
+class DBNetworkTemplate(db.Model):
+    __tablename__ = 'network_template'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.Text, nullable=False, unique=True)
+    mac_template = db.Column(db.Text)
+    ova = db.Column(db.Integer, db.ForeignKey("ova.id"), nullable=True)
+
+    def __init__(self, name, mac_template, ova=None):
+        self.name = name
+        self.mac_template = mac_template
+        self.ova = ova
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+
+class DBNetworkElement(db.Model):
+    __tablename__ = 'network'
+
+    id = db.Column(db.Integer, primary_key=True)
+    network_template = db.Column(db.Integer, db.ForeignKey("network_template.id"), nullable=False)
+    vni = db.Column(db.Integer, nullable=False)
+
+    def __init__(self, network_template, vni):
+        self.network_template = network_template
+        self.vni = vni
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+
+class DBStorageTemplate(db.Model):
+    __tablename__ = 'storage_template'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.Text, nullable=False, unique=True)
+    ova = db.Column(db.Integer, db.ForeignKey("ova.id"), nullable=True)
+
+    def __init__(self, name, ova=None):
+        self.name = name
+        self.ova = ova
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+
+class DBStorageElement(db.Model):
+    __tablename__ = 'storage'
+
+    id = db.Column(db.Integer, primary_key=True)
+    storage_template = db.Column(db.Integer, db.ForeignKey("storage_template.id"), nullable=False)
+    pool = db.Column(db.Text, nullable=False)
+    disk_id = db.Column(db.Text, nullable=False)
+    source_volume = db.Column(db.Text)
+    disk_size_gb = db.Column(db.Integer)
+    mountpoint = db.Column(db.Text)
+    filesystem = db.Column(db.Text)
+    filesystem_args = db.Column(db.Text)
+
+    def __init__(self, storage_template, pool, disk_id, source_volume, disk_size_gb, mountpoint, filesystem, filesystem_args):
+        self.storage_template = storage_template
+        self.pool = pool
+        self.disk_id = disk_id
+        self.source_volume = source_volume
+        self.disk_size_gb = disk_size_gb
+        self.mountpoint = mountpoint
+        self.filesystem = filesystem
+        self.filesystem_args = filesystem_args
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+
+class DBUserdata(db.Model):
+    __tablename__ = 'userdata'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.Text, nullable=False, unique=True)
+    userdata = db.Column(db.Text, nullable=False)
+
+    def __init__(self, name, userdata):
+        self.name = name
+        self.userdata = userdata
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+
+class DBScript(db.Model):
+    __tablename__ = 'script'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.Text, nullable=False, unique=True)
+    script = db.Column(db.Text, nullable=False)
+
+    def __init__(self, name, script):
+        self.name = name
+        self.script = script
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+
+class DBOva(db.Model):
+    __tablename__ = 'ova'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.Text, nullable=False, unique=True)
+    ovf = db.Column(db.Text, nullable=False)
+
+    def __init__(self, name, ovf):
+        self.name = name
+        self.ovf = ovf
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+
+class DBOvaVolume(db.Model):
+    __tablename__ = 'ova_volume'
+
+    id = db.Column(db.Integer, primary_key=True)
+    ova = db.Column(db.Integer, db.ForeignKey("ova.id"), nullable=False)
+    pool = db.Column(db.Text, nullable=False)
+    volume_name = db.Column(db.Text, nullable=False)
+    volume_format = db.Column(db.Text, nullable=False)
+    disk_id = db.Column(db.Text, nullable=False)
+    disk_size_gb = db.Column(db.Integer, nullable=False)
+
+    def __init__(self, ova, pool, volume_name, volume_format, disk_id, disk_size_gb):
+        self.ova = ova
+        self.pool = pool
+        self.volume_name = volume_name
+        self.volume_format = volume_format
+        self.disk_id = disk_id
+        self.disk_size_gb = disk_size_gb
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+
+class DBProfile(db.Model):
+    __tablename__ = 'profile'
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.Text, nullable=False, unique=True)
+    profile_type = db.Column(db.Text, nullable=False)
+    system_template = db.Column(db.Integer, db.ForeignKey("system_template.id"))
+    network_template = db.Column(db.Integer, db.ForeignKey("network_template.id"))
+    storage_template = db.Column(db.Integer, db.ForeignKey("storage_template.id"))
+    userdata = db.Column(db.Integer, db.ForeignKey("userdata.id"))
+    script = db.Column(db.Integer, db.ForeignKey("script.id"))
+    ova = db.Column(db.Integer, db.ForeignKey("ova.id"))
+    arguments = db.Column(db.Text)
+
+    def __init__(self, name, profile_type, system_template, network_template, storage_template, userdata, script, ova, arguments):
+        self.name = name
+        self.profile_type = profile_type
+        self.system_template = system_template
+        self.network_template = network_template
+        self.storage_template = storage_template
+        self.userdata = userdata
+        self.script = script
+        self.ova = ova
+        self.arguments = arguments
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)
+
+
+class DBStorageBenchmarks(db.Model):
+    __tablename__ = 'storage_benchmarks'
+
+    id = db.Column(db.Integer, primary_key=True)
+    job = db.Column(db.Text, nullable=False)
+    result = db.Column(db.Text, nullable=False)
+
+    def __init__(self, job, result):
+        self.job = job
+        self.result = result
+
+    def __repr__(self):
+        return '<id {}>'.format(self.id)

api-daemon/pvcapid/ova.py

@@ -0,0 +1,552 @@
+#!/usr/bin/env python3
+
+# ova.py - PVC OVA parser library
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import flask
+import psycopg2
+import psycopg2.extras
+import re
+import math
+import tarfile
+
+import lxml.etree
+
+from werkzeug.formparser import parse_form_data
+
+import daemon_lib.common as pvc_common
+import daemon_lib.ceph as pvc_ceph
+
+import pvcapid.provisioner as provisioner
+
+config = None  # Set in this namespace by flaskapi
+
+
+#
+# Common functions
+#
+
+# Database connections
+def open_database(config):
+    conn = psycopg2.connect(
+        host=config['database_host'],
+        port=config['database_port'],
+        dbname=config['database_name'],
+        user=config['database_user'],
+        password=config['database_password']
+    )
+    cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+    return conn, cur
+
+
+def close_database(conn, cur, failed=False):
+    if not failed:
+        conn.commit()
+    cur.close()
+    conn.close()
+
+
+#
+# OVA functions
+#
+def list_ova(limit, is_fuzzy=True):
+    if limit:
+        if is_fuzzy:
+            # Handle fuzzy vs. non-fuzzy limits
+            if not re.match(r'\^.*', limit):
+                limit = '%' + limit
+            else:
+                limit = limit[1:]
+            if not re.match(r'.*\$', limit):
+                limit = limit + '%'
+            else:
+                limit = limit[:-1]
+
+        query = "SELECT id, name FROM {} WHERE name LIKE %s;".format('ova')
+        args = (limit, )
+    else:
+        query = "SELECT id, name FROM {};".format('ova')
+        args = ()
+
+    conn, cur = open_database(config)
+    cur.execute(query, args)
+    data = cur.fetchall()
+    close_database(conn, cur)
+
+    ova_data = list()
+
+    for ova in data:
+        ova_id = ova.get('id')
+        ova_name = ova.get('name')
+
+        query = "SELECT pool, volume_name, volume_format, disk_id, disk_size_gb FROM {} WHERE ova = %s;".format('ova_volume')
+        args = (ova_id,)
+        conn, cur = open_database(config)
+        cur.execute(query, args)
+        volumes = cur.fetchall()
+        close_database(conn, cur)
+
+        ova_data.append({'id': ova_id, 'name': ova_name, 'volumes': volumes})
+
+    if ova_data:
+        return ova_data, 200
+    else:
+        return {'message': 'No OVAs found.'}, 404
+
+
+def delete_ova(name):
+    ova_data, retcode = list_ova(name, is_fuzzy=False)
+    if retcode != 200:
+        retmsg = {'message': 'The OVA "{}" does not exist.'.format(name)}
+        retcode = 400
+        return retmsg, retcode
+
+    conn, cur = open_database(config)
+    ova_id = ova_data[0].get('id')
+    try:
+        # Get the list of volumes for this OVA
+        query = "SELECT pool, volume_name FROM ova_volume WHERE ova = %s;"
+        args = (ova_id,)
+        cur.execute(query, args)
+        volumes = cur.fetchall()
+
+        # Remove each volume for this OVA
+        zk_conn = pvc_common.startZKConnection(config['coordinators'])
+        for volume in volumes:
+            pvc_ceph.remove_volume(zk_conn, volume.get('pool'), volume.get('volume_name'))
+
+        # Delete the volume entries from the database
+        query = "DELETE FROM ova_volume WHERE ova = %s;"
+        args = (ova_id,)
+        cur.execute(query, args)
+
+        # Delete the profile entries from the database
+        query = "DELETE FROM profile WHERE ova = %s;"
+        args = (ova_id,)
+        cur.execute(query, args)
+
+        # Delete the system_template entries from the database
+        query = "DELETE FROM system_template WHERE ova = %s;"
+        args = (ova_id,)
+        cur.execute(query, args)
+
+        # Delete the OVA entry from the database
+        query = "DELETE FROM ova WHERE id = %s;"
+        args = (ova_id,)
+        cur.execute(query, args)
+
+        retmsg = {"message": 'Removed OVA image "{}".'.format(name)}
+        retcode = 200
+    except Exception as e:
+        retmsg = {'message': 'Failed to remove OVA "{}": {}'.format(name, e)}
+        retcode = 400
+    close_database(conn, cur)
+    return retmsg, retcode
+
+
+def upload_ova(pool, name, ova_size):
+    ova_archive = None
+
+    # Cleanup function
+    def cleanup_ova_maps_and_volumes():
+        # Close the OVA archive
+        if ova_archive:
+            ova_archive.close()
+        zk_conn = pvc_common.startZKConnection(config['coordinators'])
+        # Unmap the OVA temporary blockdev
+        retflag, retdata = pvc_ceph.unmap_volume(zk_conn, pool, "ova_{}".format(name))
+        # Remove the OVA temporary blockdev
+        retflag, retdata = pvc_ceph.remove_volume(zk_conn, pool, "ova_{}".format(name))
+        pvc_common.stopZKConnection(zk_conn)
+
+    # Normalize the OVA size to bytes
+    ova_size_bytes = int(pvc_ceph.format_bytes_fromhuman(ova_size)[:-1])
+    ova_size = pvc_ceph.format_bytes_fromhuman(ova_size)
+
+    # Verify that the cluster has enough space to store the OVA volumes (2x OVA size, temporarily, 1x permanently)
+    zk_conn = pvc_common.startZKConnection(config['coordinators'])
+    pool_information = pvc_ceph.getPoolInformation(zk_conn, pool)
+    pvc_common.stopZKConnection(zk_conn)
+    pool_free_space_bytes = int(pool_information['stats']['free_bytes'])
+    if ova_size_bytes * 2 >= pool_free_space_bytes:
+        output = {
+            'message': "The cluster does not have enough free space ({}) to store the OVA volume ({}).".format(
+                pvc_ceph.format_bytes_tohuman(pool_free_space_bytes),
+                pvc_ceph.format_bytes_tohuman(ova_size_bytes)
+            )
+        }
+        retcode = 400
+        cleanup_ova_maps_and_volumes()
+        return output, retcode
+
+    # Create a temporary OVA blockdev
+    zk_conn = pvc_common.startZKConnection(config['coordinators'])
+    retflag, retdata = pvc_ceph.add_volume(zk_conn, pool, "ova_{}".format(name), ova_size)
+    pvc_common.stopZKConnection(zk_conn)
+    if not retflag:
+        output = {
+            'message': retdata.replace('\"', '\'')
+        }
+        retcode = 400
+        cleanup_ova_maps_and_volumes()
+        return output, retcode
+
+    # Map the temporary OVA blockdev
+    zk_conn = pvc_common.startZKConnection(config['coordinators'])
+    retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, "ova_{}".format(name))
+    pvc_common.stopZKConnection(zk_conn)
+    if not retflag:
+        output = {
+            'message': retdata.replace('\"', '\'')
+        }
+        retcode = 400
+        cleanup_ova_maps_and_volumes()
+        return output, retcode
+    ova_blockdev = retdata
+
+    # Save the OVA data to the temporary blockdev directly
+    try:
+        # This sets up a custom stream_factory that writes directly into the ova_blockdev,
+        # rather than the standard stream_factory which writes to a temporary file waiting
+        # on a save() call. This will break if the API ever uploaded multiple files, but
+        # this is an acceptable workaround.
+        def ova_stream_factory(total_content_length, filename, content_type, content_length=None):
+            return open(ova_blockdev, 'wb')
+        parse_form_data(flask.request.environ, stream_factory=ova_stream_factory)
+    except Exception:
+        output = {
+            'message': "Failed to upload or write OVA file to temporary volume."
+        }
+        retcode = 400
+        cleanup_ova_maps_and_volumes()
+        return output, retcode
+
+    try:
+        # Set up the TAR reader for the OVA temporary blockdev
+        ova_archive = tarfile.open(name=ova_blockdev)
+        # Determine the files in the OVA
+        members = ova_archive.getmembers()
+    except tarfile.TarError:
+        output = {
+            'message': "The uploaded OVA file is not readable."
+        }
+        retcode = 400
+        cleanup_ova_maps_and_volumes()
+        return output, retcode
+
+    # Parse through the members list and extract the OVF file
+    for element in set(x for x in members if re.match(r'.*\.ovf$', x.name)):
+        ovf_file = ova_archive.extractfile(element)
+
+    # Parse the OVF file to get our VM details
+    ovf_parser = OVFParser(ovf_file)
+    ovf_xml_raw = ovf_parser.getXML()
+    virtual_system = ovf_parser.getVirtualSystems()[0]
+    virtual_hardware = ovf_parser.getVirtualHardware(virtual_system)
+    disk_map = ovf_parser.getDiskMap(virtual_system)
+
+    # Close the OVF file
+    ovf_file.close()
+
+    # Create and upload each disk volume
+    for idx, disk in enumerate(disk_map):
+        disk_identifier = "sd{}".format(chr(ord('a') + idx))
+        volume = "ova_{}_{}".format(name, disk_identifier)
+        dev_src = disk.get('src')
+        dev_size_raw = ova_archive.getmember(dev_src).size
+        vm_volume_size = disk.get('capacity')
+
+        # Normalize the dev size to bytes
+        dev_size = pvc_ceph.format_bytes_fromhuman(dev_size_raw)
+
+        def cleanup_img_maps():
+            zk_conn = pvc_common.startZKConnection(config['coordinators'])
+            # Unmap the temporary blockdev
+            retflag, retdata = pvc_ceph.unmap_volume(zk_conn, pool, volume)
+            pvc_common.stopZKConnection(zk_conn)
+
+        # Create the blockdev
+        zk_conn = pvc_common.startZKConnection(config['coordinators'])
+        retflag, retdata = pvc_ceph.add_volume(zk_conn, pool, volume, dev_size)
+        pvc_common.stopZKConnection(zk_conn)
+        if not retflag:
+            output = {
+                'message': retdata.replace('\"', '\'')
+            }
+            retcode = 400
+            cleanup_img_maps()
+            cleanup_ova_maps_and_volumes()
+            return output, retcode
+
+        # Map the blockdev
+        zk_conn = pvc_common.startZKConnection(config['coordinators'])
+        retflag, retdata = pvc_ceph.map_volume(zk_conn, pool, volume)
+        pvc_common.stopZKConnection(zk_conn)
+        if not retflag:
+            output = {
+                'message': retdata.replace('\"', '\'')
+            }
+            retcode = 400
+            cleanup_img_maps()
+            cleanup_ova_maps_and_volumes()
+            return output, retcode
+        temp_blockdev = retdata
+
+        try:
+            # Open (extract) the TAR archive file and seek to byte 0
+            vmdk_file = ova_archive.extractfile(disk.get('src'))
+            vmdk_file.seek(0)
+            # Open the temporary blockdev and seek to byte 0
+            blk_file = open(temp_blockdev, 'wb')
+            blk_file.seek(0)
+            # Write the contents of vmdk_file into blk_file
+            blk_file.write(vmdk_file.read())
+            # Close blk_file (and flush the buffers)
+            blk_file.close()
+            # Close vmdk_file
+            vmdk_file.close()
+            # Perform an OS-level sync
+            pvc_common.run_os_command('sync')
+        except Exception:
+            output = {
+                'message': "Failed to write image file '{}' to temporary volume.".format(disk.get('src'))
+            }
+            retcode = 400
+            cleanup_img_maps()
+            cleanup_ova_maps_and_volumes()
+            return output, retcode
+
+        cleanup_img_maps()
+
+    cleanup_ova_maps_and_volumes()
+
+    # Prepare the database entries
+    query = "INSERT INTO ova (name, ovf) VALUES (%s, %s);"
+    args = (name, ovf_xml_raw)
+    conn, cur = open_database(config)
+    try:
+        cur.execute(query, args)
+        close_database(conn, cur)
+    except Exception as e:
+        output = {
+            'message': 'Failed to create OVA entry "{}": {}'.format(name, e)
+        }
+        retcode = 400
+        close_database(conn, cur)
+        return output, retcode
+
+    # Get the OVA database id
+    query = "SELECT id FROM ova WHERE name = %s;"
+    args = (name, )
+    conn, cur = open_database(config)
+    cur.execute(query, args)
+    ova_id = cur.fetchone()['id']
+    close_database(conn, cur)
+
+    # Prepare disk entries in ova_volume
+    for idx, disk in enumerate(disk_map):
+        disk_identifier = "sd{}".format(chr(ord('a') + idx))
+        volume_type = disk.get('src').split('.')[-1]
+        volume = "ova_{}_{}".format(name, disk_identifier)
+        vm_volume_size = disk.get('capacity')
+
+        # The function always return XXXXB, so strip off the B and convert to an integer
+        vm_volume_size_bytes = int(pvc_ceph.format_bytes_fromhuman(vm_volume_size)[:-1])
+        vm_volume_size_gb = math.ceil(vm_volume_size_bytes / 1024 / 1024 / 1024)
+
+        query = "INSERT INTO ova_volume (ova, pool, volume_name, volume_format, disk_id, disk_size_gb) VALUES (%s, %s, %s, %s, %s, %s);"
+        args = (ova_id, pool, volume, volume_type, disk_identifier, vm_volume_size_gb)
+
+        conn, cur = open_database(config)
+        try:
+            cur.execute(query, args)
+            close_database(conn, cur)
+        except Exception as e:
+            output = {
+                'message': 'Failed to create OVA volume entry "{}": {}'.format(volume, e)
+            }
+            retcode = 400
+            close_database(conn, cur)
+            return output, retcode
+
+    # Prepare a system_template for the OVA
+    vcpu_count = virtual_hardware.get('vcpus')
+    vram_mb = virtual_hardware.get('vram')
+    if virtual_hardware.get('graphics-controller') == 1:
+        vnc = True
+        serial = False
+    else:
+        vnc = False
+        serial = True
+    retdata, retcode = provisioner.create_template_system(name, vcpu_count, vram_mb, serial, vnc, vnc_bind=None, ova=ova_id)
+    if retcode != 200:
+        return retdata, retcode
+    system_template, retcode = provisioner.list_template_system(name, is_fuzzy=False)
+    if retcode != 200:
+        return retdata, retcode
+    system_template_name = system_template[0].get('name')
+
+    # Prepare a barebones profile for the OVA
+    retdata, retcode = provisioner.create_profile(name, 'ova', system_template_name, None, None, userdata=None, script=None, ova=name, arguments=None)
+    if retcode != 200:
+        return retdata, retcode
+
+    output = {
+        'message': "Imported OVA image '{}'.".format(name)
+    }
+    retcode = 200
+    return output, retcode
+
+
+#
+# OVF parser
+#
+class OVFParser(object):
+    RASD_TYPE = {
+        "1": "vmci",
+        "3": "vcpus",
+        "4": "vram",
+        "5": "ide-controller",
+        "6": "scsi-controller",
+        "10": "ethernet-adapter",
+        "15": "cdrom",
+        "17": "disk",
+        "20": "other-storage-device",
+        "23": "usb-controller",
+        "24": "graphics-controller",
+        "35": "sound-controller"
+    }
+
+    def _getFilelist(self):
+        path = "{{{schema}}}References/{{{schema}}}File".format(schema=self.OVF_SCHEMA)
+        id_attr = "{{{schema}}}id".format(schema=self.OVF_SCHEMA)
+        href_attr = "{{{schema}}}href".format(schema=self.OVF_SCHEMA)
+        current_list = self.xml.findall(path)
+        results = [(x.get(id_attr), x.get(href_attr)) for x in current_list]
+        return results
+
+    def _getDisklist(self):
+        path = "{{{schema}}}DiskSection/{{{schema}}}Disk".format(schema=self.OVF_SCHEMA)
+        id_attr = "{{{schema}}}diskId".format(schema=self.OVF_SCHEMA)
+        ref_attr = "{{{schema}}}fileRef".format(schema=self.OVF_SCHEMA)
+        cap_attr = "{{{schema}}}capacity".format(schema=self.OVF_SCHEMA)
+        cap_units = "{{{schema}}}capacityAllocationUnits".format(schema=self.OVF_SCHEMA)
+        current_list = self.xml.findall(path)
+        results = [(x.get(id_attr), x.get(ref_attr), x.get(cap_attr), x.get(cap_units)) for x in current_list]
+        return results
+
+    def _getAttributes(self, virtual_system, path, attribute):
+        current_list = virtual_system.findall(path)
+        results = [x.get(attribute) for x in current_list]
+        return results
+
+    def __init__(self, ovf_file):
+        self.xml = lxml.etree.parse(ovf_file)
+
+        # Define our schemas
+        envelope_tag = self.xml.find(".")
+        self.XML_SCHEMA = envelope_tag.nsmap.get('xsi')
+        self.OVF_SCHEMA = envelope_tag.nsmap.get('ovf')
+        self.RASD_SCHEMA = envelope_tag.nsmap.get('rasd')
+        self.SASD_SCHEMA = envelope_tag.nsmap.get('sasd')
+        self.VSSD_SCHEMA = envelope_tag.nsmap.get('vssd')
+
+        self.ovf_version = int(self.OVF_SCHEMA.split('/')[-1])
+
+        # Get the file and disk lists
+        self.filelist = self._getFilelist()
+        self.disklist = self._getDisklist()
+
+    def getVirtualSystems(self):
+        return self.xml.findall("{{{schema}}}VirtualSystem".format(schema=self.OVF_SCHEMA))
+
+    def getXML(self):
+        return lxml.etree.tostring(self.xml, pretty_print=True).decode('utf8')
+
+    def getVirtualHardware(self, virtual_system):
+        hardware_list = virtual_system.findall(
+            "{{{schema}}}VirtualHardwareSection/{{{schema}}}Item".format(schema=self.OVF_SCHEMA)
+        )
+        virtual_hardware = {}
+
+        for item in hardware_list:
+            try:
+                item_type = self.RASD_TYPE[item.find("{{{rasd}}}ResourceType".format(rasd=self.RASD_SCHEMA)).text]
+            except Exception:
+                continue
+            quantity = item.find("{{{rasd}}}VirtualQuantity".format(rasd=self.RASD_SCHEMA))
+            if quantity is None:
+                virtual_hardware[item_type] = 1
+            else:
+                virtual_hardware[item_type] = quantity.text
+
+        return virtual_hardware
+
+    def getDiskMap(self, virtual_system):
+        # OVF v2 uses the StorageItem field, while v1 uses the normal Item field
+        if self.ovf_version < 2:
+            hardware_list = virtual_system.findall(
+                "{{{schema}}}VirtualHardwareSection/{{{schema}}}Item".format(schema=self.OVF_SCHEMA)
+            )
+        else:
+            hardware_list = virtual_system.findall(
+                "{{{schema}}}VirtualHardwareSection/{{{schema}}}StorageItem".format(schema=self.OVF_SCHEMA)
+            )
+        disk_list = []
+
+        for item in hardware_list:
+            item_type = None
+
+            if self.SASD_SCHEMA is not None:
+                item_type = self.RASD_TYPE[item.find("{{{sasd}}}ResourceType".format(sasd=self.SASD_SCHEMA)).text]
+            else:
+                item_type = self.RASD_TYPE[item.find("{{{rasd}}}ResourceType".format(rasd=self.RASD_SCHEMA)).text]
+
+            if item_type != 'disk':
+                continue
+
+            hostref = None
+            if self.SASD_SCHEMA is not None:
+                hostref = item.find("{{{sasd}}}HostResource".format(sasd=self.SASD_SCHEMA))
+            else:
+                hostref = item.find("{{{rasd}}}HostResource".format(rasd=self.RASD_SCHEMA))
+            if hostref is None:
+                continue
+            disk_res = hostref.text
+
+            # Determine which file this disk_res ultimately represents
+            (disk_id, disk_ref, disk_capacity, disk_capacity_unit) = [x for x in self.disklist if x[0] == disk_res.split('/')[-1]][0]
+            (file_id, disk_src) = [x for x in self.filelist if x[0] == disk_ref][0]
+
+            if disk_capacity_unit is not None:
+                # Handle the unit conversion
+                base_unit, action, multiple = disk_capacity_unit.split()
+                multiple_base, multiple_exponent = multiple.split('^')
+                disk_capacity = int(disk_capacity) * (int(multiple_base) ** int(multiple_exponent))
+
+            # Append the disk with all details to the list
+            disk_list.append({
+                "id": disk_id,
+                "ref": disk_ref,
+                "capacity": disk_capacity,
+                "src": disk_src
+            })
+
+        return disk_list

build-and-deploy.sh

@@ -13,26 +13,35 @@ else
 fi
 
 HOSTS=( ${@} )
-echo "${HOSTS[@]}"
+echo "> Deploying to host(s): ${HOSTS[@]}"
 
 # Build the packages
-$SUDO ./build-deb.sh
+echo -n "Building packages... "
+version="$( ./build-unstable-deb.sh 2>/dev/null )"
+echo "done. Package version ${version}."
 
 # Install the client(s) locally
-$SUDO dpkg -i ../pvc-client*.deb
+echo -n "Installing client packages locally... "
+$SUDO dpkg -i ../pvc-client*_${version}*.deb &>/dev/null
+echo "done".
 
 for HOST in ${HOSTS[@]}; do
-    echo "****"
-    echo "Deploying to host ${HOST}"
-    echo "****"
-    ssh $HOST $SUDO rm -rf /tmp/pvc
-    ssh $HOST mkdir /tmp/pvc
-    scp ../*.deb $HOST:/tmp/pvc/
-    ssh $HOST $SUDO dpkg -i /tmp/pvc/*.deb
-    ssh $HOST $SUDO systemctl restart pvcd
-    ssh $HOST rm -rf /tmp/pvc
-    echo "****"
-    echo "Waiting 10s for host ${HOST} to stabilize"
-    echo "****"
-    sleep 10
+    echo "> Deploying packages to host ${HOST}"
+    echo -n "Copying packages... "
+    ssh $HOST $SUDO rm -rf /tmp/pvc &>/dev/null
+    ssh $HOST mkdir /tmp/pvc &>/dev/null
+    scp ../pvc-*_${version}*.deb $HOST:/tmp/pvc/ &>/dev/null
+    echo "done."
+    echo -n "Installing packages... "
+    ssh $HOST $SUDO dpkg -i /tmp/pvc/{pvc-client-cli,pvc-daemon-common,pvc-daemon-api,pvc-daemon-node}*.deb &>/dev/null
+    ssh $HOST rm -rf /tmp/pvc &>/dev/null
+    echo "done."
+    echo -n "Restarting PVC daemons... "
+    ssh $HOST $SUDO systemctl restart pvcapid &>/dev/null
+    ssh $HOST $SUDO systemctl restart pvcapid-worker &>/dev/null
+    ssh $HOST $SUDO systemctl restart pvcnoded &>/dev/null
+    echo "done."
+    echo -n "Waiting 15s for host to stabilize... "
+    sleep 15
+    echo "done."
 done

build-deb.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-ver="0.6"
+ver="$( head -1 debian/changelog | awk -F'[()-]' '{ print $2 }' )"
 git pull
 rm ../pvc_*
 dh_make -p pvc_${ver} --createorig --single --yes

build-unstable-deb.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+set -o xtrace
+exec 3>&1
+exec 1>&2
+# Ensure we're up to date
+git pull --rebase
+# Update the version to a sensible git revision for easy visualization
+base_ver="$( head -1 debian/changelog | awk -F'[()-]' '{ print $2 }' )"
+new_ver="${base_ver}~git-$(git rev-parse --short HEAD)"
+echo ${new_ver} >&3
+# Back up the existing changelog and Daemon.py files
+tmpdir=$( mktemp -d )
+cp -a debian/changelog node-daemon/pvcnoded/Daemon.py ${tmpdir}/
+# Replace the "base" version with the git revision version
+sed -i "s/version = '${base_ver}'/version = '${new_ver}'/" node-daemon/pvcnoded/Daemon.py
+sed -i "s/${base_ver}-0/${new_ver}/" debian/changelog 
+cat <<EOF > debian/changelog
+pvc (${new_ver}) unstable; urgency=medium
+
+  * Unstable revision for commit $(git rev-parse --short HEAD)
+
+ -- Joshua Boniface <joshua@boniface.me>  $( date -R )
+EOF
+# Build source tarball
+dh_make -p pvc_${new_ver} --createorig --single --yes
+# Build packages
+dpkg-buildpackage -us -uc
+# Restore original changelog and Daemon.py files
+cp -a ${tmpdir}/changelog debian/changelog
+cp -a ${tmpdir}/Daemon.py node-daemon/pvcnoded/Daemon.py
+# Clean up
+rm -r ${tmpdir}
+dh_clean

bump-version

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+new_version="${1}"
+if [[ -z ${new_version} ]]; then
+    exit 1
+fi
+
+current_version="$( grep 'version = ' node-daemon/pvcnoded/Daemon.py | awk -F "'" '{ print $2 }' )"
+echo "${current_version} -> ${new_version}"
+
+changelog_file=$( mktemp )
+echo "# Write the changelog below; comments will be ignored" >> ${changelog_file}
+$EDITOR ${changelog_file}
+
+changelog="$( cat ${changelog_file} | grep -v '^#' | sed 's/^*/  */' )"
+
+sed -i "s,version = '${current_version}',version = '${new_version}'," node-daemon/pvcnoded/Daemon.py
+
+readme_tmpdir=$( mktemp -d )
+cp README.md ${readme_tmpdir}/
+cp docs/index.md ${readme_tmpdir}/
+pushd ${readme_tmpdir} &>/dev/null
+
+echo -e "\n#### v${new_version}\n\n${changelog}" >> middle
+
+csplit README.md "/## Changelog/1" &>/dev/null
+cat xx00 middle xx01 > README.md
+rm xx00 xx01
+
+csplit index.md "/## Changelog/1" &>/dev/null
+cat xx00 middle xx01 > index.md
+rm xx00 xx01
+
+popd &>/dev/null
+mv ${readme_tmpdir}/README.md README.md
+mv ${readme_tmpdir}/index.md docs/index.md
+rm -r ${readme_tmpdir}
+
+deb_changelog_orig="$( cat debian/changelog )"
+deb_changelog_new="pvc (${new_version}-0) unstable; urgency=high\n\n${changelog}\n\n -- $( git config --get user.name ) <$( git config --get user.email )>  $( date --rfc-email )\n"
+
+deb_changelog_file=$( mktemp )
+echo -e "${deb_changelog_new}" >> ${deb_changelog_file}
+echo -e "${deb_changelog_orig}" >> ${deb_changelog_file}
+mv ${deb_changelog_file} debian/changelog
+
+git add node-daemon/pvcnoded/Daemon.py README.md docs/index.md debian/changelog
+git commit -v
+
+echo
+echo "GitLab release message:"
+echo
+echo "# Parallel Virtual Cluster version ${new_version}"
+echo
+echo -e "${changelog}" | sed 's/^  \*/*/'
+echo

client-api/client_lib

@@ -1 +0,0 @@
-../client-common

client-api/provisioner/schema.sql

@@ -1,11 +0,0 @@
-CREATE TABLE system_template (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, vcpu_count INT NOT NULL, vram_mb INT NOT NULL, serial BOOL NOT NULL, vnc BOOL NOT NULL, vnc_bind TEXT, node_limit TEXT, node_selector TEXT, node_autostart BOOL NOT NULL);
-CREATE TABLE network_template (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, mac_template TEXT);
-CREATE TABLE network (id SERIAL PRIMARY KEY, network_template INT REFERENCES network_template(id), vni INT NOT NULL);
-CREATE TABLE storage_template (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE);
-CREATE TABLE storage (id SERIAL PRIMARY KEY, storage_template INT REFERENCES storage_template(id), pool TEXT NOT NULL, disk_id TEXT NOT NULL, source_volume TEXT, disk_size_gb INT, mountpoint TEXT, filesystem TEXT, filesystem_args TEXT);
-CREATE TABLE userdata (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, userdata TEXT NOT NULL);
-CREATE TABLE script (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, script TEXT NOT NULL);
-CREATE TABLE profile (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, system_template INT REFERENCES system_template(id), network_template INT REFERENCES network_template(id), storage_template INT REFERENCES storage_template(id), userdata INT REFERENCES userdata(id), script INT REFERENCES script(id), arguments text);
-
-INSERT INTO userdata (name, userdata) VALUES ('empty', '');
-INSERT INTO script (name, script) VALUES ('empty', '');

client-api/pvc-provisioner-worker.service

@@ -1,16 +0,0 @@
-# Parallel Virtual Cluster Provisioner client worker unit file
-
-[Unit]
-Description = Parallel Virtual Cluster Provisioner worker
-After = network-online.target
-
-[Service]
-Type = simple
-WorkingDirectory = /usr/share/pvc
-Environment = PYTHONUNBUFFERED=true
-Environment = PVC_CONFIG_FILE=/etc/pvc/pvc-api.yaml
-ExecStart = /usr/bin/celery worker -A pvc-api.celery --concurrency 1 --loglevel INFO
-Restart = on-failure
-
-[Install]
-WantedBy = multi-user.target

client-cli/cli_lib/ansiprint.py

@@ -22,24 +22,40 @@
 
 import datetime
 
+
 # ANSII colours for output
 def red():
     return '\033[91m'
+
+
 def blue():
     return '\033[94m'
+
+
 def cyan():
     return '\033[96m'
+
+
 def green():
     return '\033[92m'
+
+
 def yellow():
     return '\033[93m'
+
+
 def purple():
     return '\033[95m'
+
+
 def bold():
     return '\033[1m'
+
+
 def end():
     return '\033[0m'
 
+
 # Print function
 def echo(message, prefix, state):
     # Get the date

client-cli/cli_lib/cluster.py

@@ -25,6 +25,7 @@ import json
 import cli_lib.ansiprint as ansiprint
 from cli_lib.common import call_api
 
+
 def initialize(config):
     """
     Initialize the PVC cluster
@@ -33,14 +34,15 @@ def initialize(config):
     API arguments:
     API schema: {json_data_object}
     """
-    response = call_api(config, 'get', '/initialize')
+    response = call_api(config, 'post', '/initialize')
 
     if response.status_code == 200:
         retstatus = True
     else:
         retstatus = False
 
-    return retstatus, response.json()['message']
+    return retstatus, response.json().get('message', '')
+
 
 def maintenance_mode(config, state):
     """
@@ -60,7 +62,8 @@ def maintenance_mode(config, state):
     else:
         retstatus = False
 
-    return retstatus, response.json()['message']
+    return retstatus, response.json().get('message', '')
+
 
 def get_info(config):
     """
@@ -75,7 +78,8 @@ def get_info(config):
     if response.status_code == 200:
         return True, response.json()
     else:
-        return False, response.json()['message']
+        return False, response.json().get('message', '')
+
 
 def format_info(cluster_information, oformat):
     if oformat == 'json':
@@ -92,10 +96,40 @@ def format_info(cluster_information, oformat):
     else:
         health_colour = ansiprint.yellow()
 
+    if cluster_information['storage_health'] == 'Optimal':
+        storage_health_colour = ansiprint.green()
+    elif cluster_information['storage_health'] == 'Maintenance':
+        storage_health_colour = ansiprint.blue()
+    else:
+        storage_health_colour = ansiprint.yellow()
+
     ainformation = []
+
+    if oformat == 'short':
+        ainformation.append('{}PVC cluster status:{}'.format(ansiprint.bold(), ansiprint.end()))
+        ainformation.append('{}Cluster health:{}      {}{}{}'.format(ansiprint.purple(), ansiprint.end(), health_colour, cluster_information['health'], ansiprint.end()))
+        if cluster_information['health_msg']:
+            for line in cluster_information['health_msg']:
+                ainformation.append('                     > {}'.format(line))
+        ainformation.append('{}Storage health:{}      {}{}{}'.format(ansiprint.purple(), ansiprint.end(), storage_health_colour, cluster_information['storage_health'], ansiprint.end()))
+        if cluster_information['storage_health_msg']:
+            for line in cluster_information['storage_health_msg']:
+                ainformation.append('                     > {}'.format(line))
+
+        return '\n'.join(ainformation)
+
     ainformation.append('{}PVC cluster status:{}'.format(ansiprint.bold(), ansiprint.end()))
     ainformation.append('')
     ainformation.append('{}Cluster health:{}      {}{}{}'.format(ansiprint.purple(), ansiprint.end(), health_colour, cluster_information['health'], ansiprint.end()))
+    if cluster_information['health_msg']:
+        for line in cluster_information['health_msg']:
+            ainformation.append('                     > {}'.format(line))
+    ainformation.append('{}Storage health:{}      {}{}{}'.format(ansiprint.purple(), ansiprint.end(), storage_health_colour, cluster_information['storage_health'], ansiprint.end()))
+    if cluster_information['storage_health_msg']:
+        for line in cluster_information['storage_health_msg']:
+            ainformation.append('                     > {}'.format(line))
+
+    ainformation.append('')
     ainformation.append('{}Primary node:{}        {}'.format(ansiprint.purple(), ansiprint.end(), cluster_information['primary_node']))
     ainformation.append('{}Cluster upstream IP:{} {}'.format(ansiprint.purple(), ansiprint.end(), cluster_information['upstream_ip']))
     ainformation.append('')

client-cli/cli_lib/common.py

@@ -20,10 +20,103 @@
 #
 ###############################################################################
 
+import os
+import math
+import time
 import requests
 import click
+from urllib3 import disable_warnings
 
-def call_api(config, operation, request_uri, params=None, data=None):
+
+def format_bytes(size_bytes):
+    byte_unit_matrix = {
+        'B': 1,
+        'K': 1024,
+        'M': 1024 * 1024,
+        'G': 1024 * 1024 * 1024,
+        'T': 1024 * 1024 * 1024 * 1024,
+        'P': 1024 * 1024 * 1024 * 1024 * 1024
+    }
+    human_bytes = '0B'
+    for unit in sorted(byte_unit_matrix, key=byte_unit_matrix.get):
+        formatted_bytes = int(math.ceil(size_bytes / byte_unit_matrix[unit]))
+        if formatted_bytes < 10000:
+            human_bytes = '{}{}'.format(formatted_bytes, unit)
+            break
+    return human_bytes
+
+
+def format_metric(integer):
+    integer_unit_matrix = {
+        '': 1,
+        'K': 1000,
+        'M': 1000 * 1000,
+        'B': 1000 * 1000 * 1000,
+        'T': 1000 * 1000 * 1000 * 1000,
+        'Q': 1000 * 1000 * 1000 * 1000 * 1000
+    }
+    human_integer = '0'
+    for unit in sorted(integer_unit_matrix, key=integer_unit_matrix.get):
+        formatted_integer = int(math.ceil(integer / integer_unit_matrix[unit]))
+        if formatted_integer < 10000:
+            human_integer = '{}{}'.format(formatted_integer, unit)
+            break
+    return human_integer
+
+
+class UploadProgressBar(object):
+    def __init__(self, filename, end_message='', end_nl=True):
+        file_size = os.path.getsize(filename)
+        file_size_human = format_bytes(file_size)
+        click.echo("Uploading file (total size {})...".format(file_size_human))
+
+        self.length = file_size
+        self.time_last = int(round(time.time() * 1000)) - 1000
+        self.bytes_last = 0
+        self.bytes_diff = 0
+        self.is_end = False
+
+        self.end_message = end_message
+        self.end_nl = end_nl
+        if not self.end_nl:
+            self.end_suffix = ' '
+        else:
+            self.end_suffix = ''
+
+        self.bar = click.progressbar(length=self.length, show_eta=True)
+
+    def update(self, monitor):
+        bytes_cur = monitor.bytes_read
+        self.bytes_diff += bytes_cur - self.bytes_last
+        if self.bytes_last == bytes_cur:
+            self.is_end = True
+        self.bytes_last = bytes_cur
+
+        time_cur = int(round(time.time() * 1000))
+        if (time_cur - 1000) > self.time_last:
+            self.time_last = time_cur
+            self.bar.update(self.bytes_diff)
+            self.bytes_diff = 0
+
+        if self.is_end:
+            self.bar.update(self.bytes_diff)
+            self.bytes_diff = 0
+            click.echo()
+            click.echo()
+            if self.end_message:
+                click.echo(self.end_message + self.end_suffix, nl=self.end_nl)
+
+
+class ErrorResponse(requests.Response):
+    def __init__(self, json_data, status_code):
+        self.json_data = json_data
+        self.status_code = status_code
+
+    def json(self):
+        return self.json_data
+
+
+def call_api(config, operation, request_uri, headers={}, params=None, data=None, files=None):
     # Craft the URI
     uri = '{}://{}{}{}'.format(
         config['api_scheme'],
@@ -34,50 +127,56 @@ def call_api(config, operation, request_uri, params=None, data=None):
 
     # Craft the authentication header if required
     if config['api_key']:
-        headers = {'X-Api-Key': config['api_key']}
-    else:
-        headers = None
+        headers['X-Api-Key'] = config['api_key']
 
     # Determine the request type and hit the API
+    disable_warnings()
     try:
         if operation == 'get':
             response = requests.get(
                 uri,
                 headers=headers,
                 params=params,
-                data=data
+                data=data,
+                verify=config['verify_ssl']
             )
         if operation == 'post':
             response = requests.post(
                 uri,
                 headers=headers,
                 params=params,
-                data=data
+                data=data,
+                files=files,
+                verify=config['verify_ssl']
             )
         if operation == 'put':
             response = requests.put(
                 uri,
                 headers=headers,
                 params=params,
-                data=data
+                data=data,
+                files=files,
+                verify=config['verify_ssl']
             )
         if operation == 'patch':
             response = requests.patch(
                 uri,
                 headers=headers,
                 params=params,
-                data=data
+                data=data,
+                verify=config['verify_ssl']
             )
         if operation == 'delete':
             response = requests.delete(
                 uri,
                 headers=headers,
                 params=params,
-                data=data
+                data=data,
+                verify=config['verify_ssl']
             )
     except Exception as e:
-        click.echo('Failed to connect to the API: {}'.format(e))
-        exit(1)
+        message = 'Failed to connect to the API: {}'.format(e)
+        response = ErrorResponse({'message': message}, 500)
 
     # Display debug output
     if config['debug']:
@@ -88,4 +187,3 @@ def call_api(config, operation, request_uri, params=None, data=None):
 
     # Return the response object
     return response
-

client-cli/cli_lib/network.py

@@ -20,38 +20,39 @@
 #
 ###############################################################################
 
-import difflib
-import colorama
-
+import re
 import cli_lib.ansiprint as ansiprint
 from cli_lib.common import call_api
 
+
 def isValidMAC(macaddr):
     allowed = re.compile(r"""
                          (
                             ^([0-9A-F]{2}[:]){5}([0-9A-F]{2})$
                          )
                          """,
-                         re.VERBOSE|re.IGNORECASE)
+                         re.VERBOSE | re.IGNORECASE)
 
     if allowed.match(macaddr):
         return True
     else:
         return False
 
+
 def isValidIP(ipaddr):
     ip4_blocks = str(ipaddr).split(".")
     if len(ip4_blocks) == 4:
         for block in ip4_blocks:
             # Check if number is digit, if not checked before calling this function
             if not block.isdigit():
-                 return False
+                return False
             tmp = int(block)
             if 0 > tmp > 255:
-                 return False
+                return False
         return True
     return False
 
+
 #
 # Primary functions
 #
@@ -68,7 +69,8 @@ def net_info(config, net):
     if response.status_code == 200:
         return True, response.json()
     else:
-        return False, response.json()['message']
+        return False, response.json().get('message', '')
+
 
 def net_list(config, limit):
     """
@@ -87,12 +89,13 @@ def net_list(config, limit):
     if response.status_code == 200:
         return True, response.json()
     else:
-        return False, response.json()['message']
+        return False, response.json().get('message', '')
+
 
 def net_add(config, vni, description, nettype, domain, name_servers, ip4_network, ip4_gateway, ip6_network, ip6_gateway, dhcp4_flag, dhcp4_start, dhcp4_end):
     """
     Add new network
-    
+
     API endpoint: POST /api/v1/network
     API arguments: lots
     API schema: {"message":"{data}"}
@@ -118,12 +121,13 @@ def net_add(config, vni, description, nettype, domain, name_servers, ip4_network
     else:
         retstatus = False
 
-    return retstatus, response.json()['message']
+    return retstatus, response.json().get('message', '')
+
 
 def net_modify(config, net, description, domain, name_servers, ip4_network, ip4_gateway, ip6_network, ip6_gateway, dhcp4_flag, dhcp4_start, dhcp4_end):
     """
     Modify a network
-    
+
     API endpoint: POST /api/v1/network/{net}
     API arguments: lots
     API schema: {"message":"{data}"}
@@ -157,12 +161,13 @@ def net_modify(config, net, description, domain, name_servers, ip4_network, ip4_
     else:
         retstatus = False
 
-    return retstatus, response.json()['message']
+    return retstatus, response.json().get('message', '')
+
 
 def net_remove(config, net):
     """
     Remove a network
-    
+
     API endpoint: DELETE /api/v1/network/{net}
     API arguments:
     API schema: {"message":"{data}"}
@@ -174,7 +179,8 @@ def net_remove(config, net):
     else:
         retstatus = False
 
-    return retstatus, response.json()['message']
+    return retstatus, response.json().get('message', '')
+
 
 #
 # DHCP lease functions
@@ -192,7 +198,8 @@ def net_dhcp_info(config, net, mac):
     if response.status_code == 200:
         return True, response.json()
     else:
-        return False, response.json()['message']
+        return False, response.json().get('message', '')
+
 
 def net_dhcp_list(config, net, limit, only_static=False):
     """
@@ -205,20 +212,24 @@ def net_dhcp_list(config, net, limit, only_static=False):
     params = dict()
     if limit:
         params['limit'] = limit
+
     if only_static:
         params['static'] = True
+    else:
+        params['static'] = False
 
     response = call_api(config, 'get', '/network/{net}/lease'.format(net=net), params=params)
 
     if response.status_code == 200:
         return True, response.json()
     else:
-        return False, response.json()['message']
+        return False, response.json().get('message', '')
+
 
 def net_dhcp_add(config, net, ipaddr, macaddr, hostname):
     """
     Add new network DHCP lease
-    
+
     API endpoint: POST /api/v1/network/{net}/lease
     API arguments: macaddress=macaddr, ipaddress=ipaddr, hostname=hostname
     API schema: {"message":"{data}"}
@@ -235,12 +246,13 @@ def net_dhcp_add(config, net, ipaddr, macaddr, hostname):
     else:
         retstatus = False
 
-    return retstatus, response.json()['message']
+    return retstatus, response.json().get('message', '')
+
 
 def net_dhcp_remove(config, net, mac):
     """
     Remove a network DHCP lease
-    
+
     API endpoint: DELETE /api/v1/network/{vni}/lease/{mac}
     API arguments:
     API schema: {"message":"{data}"}
@@ -252,7 +264,8 @@ def net_dhcp_remove(config, net, mac):
     else:
         retstatus = False
 
-    return retstatus, response.json()['message']
+    return retstatus, response.json().get('message', '')
+
 
 #
 # ACL functions
@@ -270,7 +283,8 @@ def net_acl_info(config, net, description):
     if response.status_code == 200:
         return True, response.json()
     else:
-        return False, response.json()['message']
+        return False, response.json().get('message', '')
+
 
 def net_acl_list(config, net, limit, direction):
     """
@@ -291,12 +305,13 @@ def net_acl_list(config, net, limit, direction):
     if response.status_code == 200:
         return True, response.json()
     else:
-        return False, response.json()['message']
+        return False, response.json().get('message', '')
+
 
 def net_acl_add(config, net, direction, description, rule, order):
     """
     Add new network acl
-    
+
     API endpoint: POST /api/v1/network/{net}/acl
     API arguments: description=description, direction=direction, order=order, rule=rule
     API schema: {"message":"{data}"}
@@ -315,12 +330,14 @@ def net_acl_add(config, net, direction, description, rule, order):
     else:
         retstatus = False
 
-    return retstatus, response.json()['message']
+    return retstatus, response.json().get('message', '')
+
 
 def net_acl_remove(config, net, description):
+
     """
     Remove a network ACL
-    
+
     API endpoint: DELETE /api/v1/network/{vni}/acl/{description}
     API arguments:
     API schema: {"message":"{data}"}
@@ -332,7 +349,7 @@ def net_acl_remove(config, net, description):
     else:
         retstatus = False
 
-    return retstatus, response.json()['message']
+    return retstatus, response.json().get('message', '')
 
 
 #
@@ -359,6 +376,7 @@ def getOutputColours(network_information):
 
     return v6_flag_colour, v4_flag_colour, dhcp6_flag_colour, dhcp4_flag_colour
 
+
 def format_info(config, network_information, long_output):
     if not network_information:
         return "No network found"
@@ -417,13 +435,14 @@ def format_info(config, network_information, long_output):
     # Join it all together
     return '\n'.join(ainformation)
 
+
 def format_list(config, network_list):
     if not network_list:
         return "No network found"
 
     # Handle single-element lists
     if not isinstance(network_list, list):
-        network_list = [ network_list ]
+        network_list = [network_list]
 
     network_list_output = []
 
@@ -461,25 +480,24 @@ def format_list(config, network_list):
 {net_v4_flag: <{net_v4_flag_length}} \
 {net_dhcp4_flag: <{net_dhcp4_flag_length}} \
 {end_bold}'.format(
-            bold=ansiprint.bold(),
-            end_bold=ansiprint.end(),
-            net_vni_length=net_vni_length,
-            net_description_length=net_description_length,
-            net_nettype_length=net_nettype_length,
-            net_domain_length=net_domain_length,
-            net_v6_flag_length=net_v6_flag_length,
-            net_dhcp6_flag_length=net_dhcp6_flag_length,
-            net_v4_flag_length=net_v4_flag_length,
-            net_dhcp4_flag_length=net_dhcp4_flag_length,
-            net_vni='VNI',
-            net_description='Description',
-            net_nettype='Type',
-            net_domain='Domain',
-            net_v6_flag='IPv6',
-            net_dhcp6_flag='DHCPv6',
-            net_v4_flag='IPv4',
-            net_dhcp4_flag='DHCPv4',
-        )
+        bold=ansiprint.bold(),
+        end_bold=ansiprint.end(),
+        net_vni_length=net_vni_length,
+        net_description_length=net_description_length,
+        net_nettype_length=net_nettype_length,
+        net_domain_length=net_domain_length,
+        net_v6_flag_length=net_v6_flag_length,
+        net_dhcp6_flag_length=net_dhcp6_flag_length,
+        net_v4_flag_length=net_v4_flag_length,
+        net_dhcp4_flag_length=net_dhcp4_flag_length,
+        net_vni='VNI',
+        net_description='Description',
+        net_nettype='Type',
+        net_domain='Domain',
+        net_v6_flag='IPv6',
+        net_dhcp6_flag='DHCPv6',
+        net_v4_flag='IPv4',
+        net_dhcp4_flag='DHCPv4')
     )
 
     for network_information in network_list:
@@ -494,13 +512,7 @@ def format_list(config, network_list):
         else:
             v6_flag = 'False'
 
-        if network_information['ip4']['dhcp_flag'] == "True":
-            dhcp4_range = '{} - {}'.format(network_information['ip4']['dhcp_start'], network_information['ip4']['dhcp_end'])
-        else:
-            dhcp4_range = 'N/A'
-
-        network_list_output.append(
-            '{bold}\
+        network_list_output.append('{bold}\
 {net_vni: <{net_vni_length}} \
 {net_description: <{net_description_length}} \
 {net_nettype: <{net_nettype_length}} \
@@ -510,34 +522,34 @@ def format_list(config, network_list):
 {v4_flag_colour}{net_v4_flag: <{net_v4_flag_length}}{colour_off} \
 {dhcp4_flag_colour}{net_dhcp4_flag: <{net_dhcp4_flag_length}}{colour_off} \
 {end_bold}'.format(
-                bold='',
-                end_bold='',
-                net_vni_length=net_vni_length,
-                net_description_length=net_description_length,
-                net_nettype_length=net_nettype_length,
-                net_domain_length=net_domain_length,
-                net_v6_flag_length=net_v6_flag_length,
-                net_dhcp6_flag_length=net_dhcp6_flag_length,
-                net_v4_flag_length=net_v4_flag_length,
-                net_dhcp4_flag_length=net_dhcp4_flag_length,
-                net_vni=network_information['vni'],
-                net_description=network_information['description'],
-                net_nettype=network_information['type'],
-                net_domain=network_information['domain'],
-                net_v6_flag=v6_flag,
-                v6_flag_colour=v6_flag_colour,
-                net_dhcp6_flag=network_information['ip6']['dhcp_flag'],
-                dhcp6_flag_colour=dhcp6_flag_colour,
-                net_v4_flag=v4_flag,
-                v4_flag_colour=v4_flag_colour,
-                net_dhcp4_flag=network_information['ip4']['dhcp_flag'],
-                dhcp4_flag_colour=dhcp4_flag_colour,
-                colour_off=ansiprint.end()
-            )
+            bold='',
+            end_bold='',
+            net_vni_length=net_vni_length,
+            net_description_length=net_description_length,
+            net_nettype_length=net_nettype_length,
+            net_domain_length=net_domain_length,
+            net_v6_flag_length=net_v6_flag_length,
+            net_dhcp6_flag_length=net_dhcp6_flag_length,
+            net_v4_flag_length=net_v4_flag_length,
+            net_dhcp4_flag_length=net_dhcp4_flag_length,
+            net_vni=network_information['vni'],
+            net_description=network_information['description'],
+            net_nettype=network_information['type'],
+            net_domain=network_information['domain'],
+            net_v6_flag=v6_flag,
+            v6_flag_colour=v6_flag_colour,
+            net_dhcp6_flag=network_information['ip6']['dhcp_flag'],
+            dhcp6_flag_colour=dhcp6_flag_colour,
+            net_v4_flag=v4_flag,
+            v4_flag_colour=v4_flag_colour,
+            net_dhcp4_flag=network_information['ip4']['dhcp_flag'],
+            dhcp4_flag_colour=dhcp4_flag_colour,
+            colour_off=ansiprint.end())
         )
 
     return '\n'.join(sorted(network_list_output))
 
+
 def format_list_dhcp(dhcp_lease_list):
     dhcp_lease_list_output = []
 
@@ -567,17 +579,16 @@ def format_list_dhcp(dhcp_lease_list):
 {lease_mac_address: <{lease_mac_address_length}} \
 {lease_timestamp: <{lease_timestamp_length}} \
 {end_bold}'.format(
-            bold=ansiprint.bold(),
-            end_bold=ansiprint.end(),
-            lease_hostname_length=lease_hostname_length,
-            lease_ip4_address_length=lease_ip4_address_length,
-            lease_mac_address_length=lease_mac_address_length,
-            lease_timestamp_length=lease_timestamp_length,
-            lease_hostname='Hostname',
-            lease_ip4_address='IP Address',
-            lease_mac_address='MAC Address',
-            lease_timestamp='Timestamp'
-        )
+        bold=ansiprint.bold(),
+        end_bold=ansiprint.end(),
+        lease_hostname_length=lease_hostname_length,
+        lease_ip4_address_length=lease_ip4_address_length,
+        lease_mac_address_length=lease_mac_address_length,
+        lease_timestamp_length=lease_timestamp_length,
+        lease_hostname='Hostname',
+        lease_ip4_address='IP Address',
+        lease_mac_address='MAC Address',
+        lease_timestamp='Timestamp')
     )
 
     for dhcp_lease_information in dhcp_lease_list:
@@ -587,28 +598,28 @@ def format_list_dhcp(dhcp_lease_list):
 {lease_mac_address: <{lease_mac_address_length}} \
 {lease_timestamp: <{lease_timestamp_length}} \
 {end_bold}'.format(
-                bold='',
-                end_bold='',
-                lease_hostname_length=lease_hostname_length,
-                lease_ip4_address_length=lease_ip4_address_length,
-                lease_mac_address_length=lease_mac_address_length,
-                lease_timestamp_length=12,
-                lease_hostname=str(dhcp_lease_information['hostname']),
-                lease_ip4_address=str(dhcp_lease_information['ip4_address']),
-                lease_mac_address=str(dhcp_lease_information['mac_address']),
-                lease_timestamp=str(dhcp_lease_information['timestamp'])
-            )
+            bold='',
+            end_bold='',
+            lease_hostname_length=lease_hostname_length,
+            lease_ip4_address_length=lease_ip4_address_length,
+            lease_mac_address_length=lease_mac_address_length,
+            lease_timestamp_length=12,
+            lease_hostname=str(dhcp_lease_information['hostname']),
+            lease_ip4_address=str(dhcp_lease_information['ip4_address']),
+            lease_mac_address=str(dhcp_lease_information['mac_address']),
+            lease_timestamp=str(dhcp_lease_information['timestamp']))
         )
 
     return '\n'.join(sorted(dhcp_lease_list_output))
 
+
 def format_list_acl(acl_list):
     # Handle when we get an empty entry
     if not acl_list:
         acl_list = list()
     # Handle when we get a single entry
     if isinstance(acl_list, dict):
-        acl_list = [ acl_list ]
+        acl_list = [acl_list]
 
     acl_list_output = []
 
@@ -638,17 +649,16 @@ def format_list_acl(acl_list):
 {acl_description: <{acl_description_length}} \
 {acl_rule: <{acl_rule_length}} \
 {end_bold}'.format(
-            bold=ansiprint.bold(),
-            end_bold=ansiprint.end(),
-            acl_direction_length=acl_direction_length,
-            acl_order_length=acl_order_length,
-            acl_description_length=acl_description_length,
-            acl_rule_length=acl_rule_length,
-            acl_direction='Direction',
-            acl_order='Order',
-            acl_description='Description',
-            acl_rule='Rule',
-        )
+        bold=ansiprint.bold(),
+        end_bold=ansiprint.end(),
+        acl_direction_length=acl_direction_length,
+        acl_order_length=acl_order_length,
+        acl_description_length=acl_description_length,
+        acl_rule_length=acl_rule_length,
+        acl_direction='Direction',
+        acl_order='Order',
+        acl_description='Description',
+        acl_rule='Rule')
     )
 
     for acl_information in acl_list:
@@ -658,17 +668,16 @@ def format_list_acl(acl_list):
 {acl_description: <{acl_description_length}} \
 {acl_rule: <{acl_rule_length}} \
 {end_bold}'.format(
-                bold='',
-                end_bold='',
-                acl_direction_length=acl_direction_length,
-                acl_order_length=acl_order_length,
-                acl_description_length=acl_description_length,
-                acl_rule_length=acl_rule_length,
-                acl_direction=acl_information['direction'],
-                acl_order=acl_information['order'],
-                acl_description=acl_information['description'],
-                acl_rule=acl_information['rule'],
-            )
+            bold='',
+            end_bold='',
+            acl_direction_length=acl_direction_length,
+            acl_order_length=acl_order_length,
+            acl_description_length=acl_description_length,
+            acl_rule_length=acl_rule_length,
+            acl_direction=acl_information['direction'],
+            acl_order=acl_information['order'],
+            acl_description=acl_information['description'],
+            acl_rule=acl_information['rule'])
         )
 
     return '\n'.join(sorted(acl_list_output))

client-cli/cli_lib/node.py

@@ -23,6 +23,7 @@
 import cli_lib.ansiprint as ansiprint
 from cli_lib.common import call_api
 
+
 #
 # Primary functions
 #
@@ -34,7 +35,7 @@ def node_coordinator_state(config, node, action):
     API arguments: action={action}
     API schema: {"message": "{data}"}
     """
-    params={
+    params = {
         'state': action
     }
     response = call_api(config, 'post', '/node/{node}/coordinator-state'.format(node=node), params=params)
@@ -44,7 +45,8 @@ def node_coordinator_state(config, node, action):
     else:
         retstatus = False
 
-    return retstatus, response.json()['message']
+    return retstatus, response.json().get('message', '')
+
 
 def node_domain_state(config, node, action, wait):
     """
@@ -54,7 +56,7 @@ def node_domain_state(config, node, action, wait):
     API arguments: action={action}, wait={wait}
     API schema: {"message": "{data}"}
     """
-    params={
+    params = {
         'state': action,
         'wait': str(wait).lower()
     }
@@ -65,7 +67,8 @@ def node_domain_state(config, node, action, wait):
     else:
         retstatus = False
 
-    return retstatus, response.json()['message']
+    return retstatus, response.json().get('message', '')
+
 
 def node_info(config, node):
     """
@@ -80,9 +83,10 @@ def node_info(config, node):
     if response.status_code == 200:
         return True, response.json()
     else:
-        return False, response.json()['message']
+        return False, response.json().get('message', '')
 
-def node_list(config, limit):
+
+def node_list(config, limit, target_daemon_state, target_coordinator_state, target_domain_state):
     """
     Get list information about nodes (limited by {limit})
 
@@ -93,13 +97,20 @@ def node_list(config, limit):
     params = dict()
     if limit:
         params['limit'] = limit
+    if target_daemon_state:
+        params['daemon_state'] = target_daemon_state
+    if target_coordinator_state:
+        params['coordinator_state'] = target_coordinator_state
+    if target_domain_state:
+        params['domain_state'] = target_domain_state
 
     response = call_api(config, 'get', '/node', params=params)
 
     if response.status_code == 200:
         return True, response.json()
     else:
-        return False, response.json()['message']
+        return False, response.json().get('message', '')
+
 
 #
 # Output display functions
@@ -130,41 +141,60 @@ def getOutputColours(node_information):
     else:
         domain_state_colour = ansiprint.blue()
 
-    return daemon_state_colour, coordinator_state_colour, domain_state_colour
+    if node_information['memory']['allocated'] > node_information['memory']['total']:
+        mem_allocated_colour = ansiprint.yellow()
+    else:
+        mem_allocated_colour = ''
+
+    if node_information['memory']['provisioned'] > node_information['memory']['total']:
+        mem_provisioned_colour = ansiprint.yellow()
+    else:
+        mem_provisioned_colour = ''
+
+    return daemon_state_colour, coordinator_state_colour, domain_state_colour, mem_allocated_colour, mem_provisioned_colour
+
 
 def format_info(node_information, long_output):
-    daemon_state_colour, coordinator_state_colour, domain_state_colour = getOutputColours(node_information)
+    daemon_state_colour, coordinator_state_colour, domain_state_colour, mem_allocated_colour, mem_provisioned_colour = getOutputColours(node_information)
 
     # Format a nice output; do this line-by-line then concat the elements at the end
     ainformation = []
     # Basic information
-    ainformation.append('{}Name:{}                 {}'.format(ansiprint.purple(), ansiprint.end(), node_information['name']))
-    ainformation.append('{}Daemon State:{}         {}{}{}'.format(ansiprint.purple(), ansiprint.end(), daemon_state_colour, node_information['daemon_state'], ansiprint.end()))
-    ainformation.append('{}Coordinator State:{}    {}{}{}'.format(ansiprint.purple(), ansiprint.end(), coordinator_state_colour, node_information['coordinator_state'], ansiprint.end()))
-    ainformation.append('{}Domain State:{}         {}{}{}'.format(ansiprint.purple(), ansiprint.end(), domain_state_colour, node_information['domain_state'], ansiprint.end()))
-    ainformation.append('{}Active VM Count:{}      {}'.format(ansiprint.purple(), ansiprint.end(), node_information['domains_count']))
+    ainformation.append('{}Name:{}                  {}'.format(ansiprint.purple(), ansiprint.end(), node_information['name']))
+    ainformation.append('{}Daemon State:{}          {}{}{}'.format(ansiprint.purple(), ansiprint.end(), daemon_state_colour, node_information['daemon_state'], ansiprint.end()))
+    ainformation.append('{}Coordinator State:{}     {}{}{}'.format(ansiprint.purple(), ansiprint.end(), coordinator_state_colour, node_information['coordinator_state'], ansiprint.end()))
+    ainformation.append('{}Domain State:{}          {}{}{}'.format(ansiprint.purple(), ansiprint.end(), domain_state_colour, node_information['domain_state'], ansiprint.end()))
+    ainformation.append('{}Active VM Count:{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_information['domains_count']))
     if long_output:
         ainformation.append('')
-        ainformation.append('{}Architecture:{}         {}'.format(ansiprint.purple(), ansiprint.end(), node_information['arch']))
-        ainformation.append('{}Operating System:{}     {}'.format(ansiprint.purple(), ansiprint.end(), node_information['os']))
-        ainformation.append('{}Kernel Version:{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_information['kernel']))
+        ainformation.append('{}Architecture:{}          {}'.format(ansiprint.purple(), ansiprint.end(), node_information['arch']))
+        ainformation.append('{}Operating System:{}      {}'.format(ansiprint.purple(), ansiprint.end(), node_information['os']))
+        ainformation.append('{}Kernel Version:{}        {}'.format(ansiprint.purple(), ansiprint.end(), node_information['kernel']))
     ainformation.append('')
-    ainformation.append('{}Host CPUs:{}            {}'.format(ansiprint.purple(), ansiprint.end(), node_information['vcpu']['total']))
-    ainformation.append('{}vCPUs:{}                {}'.format(ansiprint.purple(), ansiprint.end(), node_information['vcpu']['allocated']))
-    ainformation.append('{}Load:{}                 {}'.format(ansiprint.purple(), ansiprint.end(), node_information['load']))
-    ainformation.append('{}Total RAM (MiB):{}      {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['total']))
-    ainformation.append('{}Used RAM (MiB):{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['used']))
-    ainformation.append('{}Free RAM (MiB):{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['free']))
-    ainformation.append('{}Allocated RAM (MiB):{}  {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['allocated']))
+    ainformation.append('{}Host CPUs:{}             {}'.format(ansiprint.purple(), ansiprint.end(), node_information['vcpu']['total']))
+    ainformation.append('{}vCPUs:{}                 {}'.format(ansiprint.purple(), ansiprint.end(), node_information['vcpu']['allocated']))
+    ainformation.append('{}Load:{}                  {}'.format(ansiprint.purple(), ansiprint.end(), node_information['load']))
+    ainformation.append('{}Total RAM (MiB):{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['total']))
+    ainformation.append('{}Used RAM (MiB):{}        {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['used']))
+    ainformation.append('{}Free RAM (MiB):{}        {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['free']))
+    ainformation.append('{}Allocated RAM (MiB):{}   {}{}{}'.format(ansiprint.purple(), ansiprint.end(), mem_allocated_colour, node_information['memory']['allocated'], ansiprint.end()))
+    ainformation.append('{}Provisioned RAM (MiB):{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), mem_provisioned_colour, node_information['memory']['provisioned'], ansiprint.end()))
 
     # Join it all together
     ainformation.append('')
     return '\n'.join(ainformation)
 
-def format_list(node_list):
+
+def format_list(node_list, raw):
     # Handle single-element lists
     if not isinstance(node_list, list):
-        node_list = [ node_list ]
+        node_list = [node_list]
+
+    if raw:
+        ainformation = list()
+        for node in sorted(item['name'] for item in node_list):
+            ainformation.append(node)
+        return '\n'.join(ainformation)
 
     node_list_output = []
 
@@ -179,7 +209,8 @@ def format_list(node_list):
     mem_total_length = 6
     mem_used_length = 5
     mem_free_length = 5
-    mem_alloc_length = 4
+    mem_alloc_length = 6
+    mem_prov_length = 5
     for node_information in node_list:
         # node_name column
         _node_name_length = len(node_information['name']) + 1
@@ -226,12 +257,17 @@ def format_list(node_list):
         if _mem_alloc_length > mem_alloc_length:
             mem_alloc_length = _mem_alloc_length
 
+        # mem_prov column
+        _mem_prov_length = len(str(node_information['memory']['provisioned'])) + 1
+        if _mem_prov_length > mem_prov_length:
+            mem_prov_length = _mem_prov_length
+
     # Format the string (header)
     node_list_output.append(
         '{bold}{node_name: <{node_name_length}} \
 St: {daemon_state_colour}{node_daemon_state: <{daemon_state_length}}{end_colour} {coordinator_state_colour}{node_coordinator_state: <{coordinator_state_length}}{end_colour} {domain_state_colour}{node_domain_state: <{domain_state_length}}{end_colour} \
 Res: {node_domains_count: <{domains_count_length}} {node_cpu_count: <{cpu_count_length}} {node_load: <{load_length}} \
-Mem (M): {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {node_mem_allocated: <{mem_alloc_length}}{end_bold}'.format(
+Mem (M): {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {node_mem_allocated: <{mem_alloc_length}} {node_mem_provisioned: <{mem_prov_length}}{end_bold}'.format(
             node_name_length=node_name_length,
             daemon_state_length=daemon_state_length,
             coordinator_state_length=coordinator_state_length,
@@ -243,6 +279,7 @@ Mem (M): {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length
             mem_used_length=mem_used_length,
             mem_free_length=mem_free_length,
             mem_alloc_length=mem_alloc_length,
+            mem_prov_length=mem_prov_length,
             bold=ansiprint.bold(),
             end_bold=ansiprint.end(),
             daemon_state_colour='',
@@ -259,18 +296,19 @@ Mem (M): {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length
             node_mem_total='Total',
             node_mem_used='Used',
             node_mem_free='Free',
-            node_mem_allocated='VMs'
+            node_mem_allocated='Alloc',
+            node_mem_provisioned='Prov'
         )
     )
-            
+
     # Format the string (elements)
     for node_information in node_list:
-        daemon_state_colour, coordinator_state_colour, domain_state_colour = getOutputColours(node_information)
+        daemon_state_colour, coordinator_state_colour, domain_state_colour, mem_allocated_colour, mem_provisioned_colour = getOutputColours(node_information)
         node_list_output.append(
             '{bold}{node_name: <{node_name_length}} \
     {daemon_state_colour}{node_daemon_state: <{daemon_state_length}}{end_colour} {coordinator_state_colour}{node_coordinator_state: <{coordinator_state_length}}{end_colour} {domain_state_colour}{node_domain_state: <{domain_state_length}}{end_colour} \
      {node_domains_count: <{domains_count_length}} {node_cpu_count: <{cpu_count_length}} {node_load: <{load_length}} \
-         {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {node_mem_allocated: <{mem_alloc_length}}{end_bold}'.format(
+         {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {mem_allocated_colour}{node_mem_allocated: <{mem_alloc_length}}{end_colour} {mem_provisioned_colour}{node_mem_provisioned: <{mem_prov_length}}{end_colour}{end_bold}'.format(
                 node_name_length=node_name_length,
                 daemon_state_length=daemon_state_length,
                 coordinator_state_length=coordinator_state_length,
@@ -282,11 +320,14 @@ Mem (M): {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length
                 mem_used_length=mem_used_length,
                 mem_free_length=mem_free_length,
                 mem_alloc_length=mem_alloc_length,
+                mem_prov_length=mem_prov_length,
                 bold='',
                 end_bold='',
                 daemon_state_colour=daemon_state_colour,
                 coordinator_state_colour=coordinator_state_colour,
                 domain_state_colour=domain_state_colour,
+                mem_allocated_colour=mem_allocated_colour,
+                mem_provisioned_colour=mem_allocated_colour,
                 end_colour=ansiprint.end(),
                 node_name=node_information['name'],
                 node_daemon_state=node_information['daemon_state'],
@@ -298,7 +339,8 @@ Mem (M): {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length
                 node_mem_total=node_information['memory']['total'],
                 node_mem_used=node_information['memory']['used'],
                 node_mem_free=node_information['memory']['free'],
-                node_mem_allocated=node_information['memory']['allocated']
+                node_mem_allocated=node_information['memory']['allocated'],
+                node_mem_provisioned=node_information['memory']['provisioned']
             )
         )
 

client-cli/cli_lib/zkhandler.py

@@ -20,10 +20,8 @@
 #
 ###############################################################################
 
-import kazoo.client
 import uuid
 
-import client_lib.ansiprint as ansiprint
 
 # Exists function
 def exists(zk_conn, key):
@@ -33,22 +31,25 @@ def exists(zk_conn, key):
     else:
         return False
 
+
 # Child list function
 def listchildren(zk_conn, key):
     children = zk_conn.get_children(key)
     return children
 
+
 # Delete key function
 def deletekey(zk_conn, key, recursive=True):
     zk_conn.delete(key, recursive=recursive)
 
+
 # Data read function
 def readdata(zk_conn, key):
     data_raw = zk_conn.get(key)
     data = data_raw[0].decode('utf8')
-    meta = data_raw[1]
     return data
 
+
 # Data write function
 def writedata(zk_conn, kv):
     # Start up a transaction
@@ -87,12 +88,14 @@ def writedata(zk_conn, kv):
     except Exception:
         return False
 
+
 # Write lock function
 def writelock(zk_conn, key):
     lock_id = str(uuid.uuid1())
     lock = zk_conn.WriteLock('{}'.format(key), lock_id)
     return lock
 
+
 # Read lock function
 def readlock(zk_conn, key):
     lock_id = str(uuid.uuid1())

client-cli/scripts/README

@@ -0,0 +1,32 @@
+# PVC helper scripts
+
+These helper scripts are included with the PVC client to aid administrators in some meta-functions.
+
+The following scripts are provided for use:
+
+## `migrate_vm`
+
+Migrates a VM, with downtime, from one PVC cluster to another.
+
+`migrate_vm <vm> <source_cluster> <destination_cluster>`
+
+### Arguments
+
+ * `vm`: The virtual machine to migrate
+ * `source_cluster`: The source PVC cluster; must be a valid cluster to the local PVC client
+ * `destination_cluster`: The destination PVC cluster; must be a valid cluster to the local PVC client
+
+## `import_vm`
+
+Imports a VM from another platform into a PVC cluster.
+
+## `export_vm`
+
+Exports a (stopped) VM from a PVC cluster to another platform.
+
+`export_vm <vm> <source_cluster>`
+
+### Arguments
+
+ * `vm`: The virtual machine to migrate
+ * `source_cluster`: The source PVC cluster; must be a valid cluster to the local PVC client

client-cli/scripts/export_vm

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+# export_vm - Exports a VM from a PVC cluster to local files
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+set -o errexit
+set -o pipefail
+
+usage() {
+    echo -e "Export a VM from a PVC cluster to local files."
+    echo -e "Usage:"
+    echo -e "  $0 <vm> <source_cluster> [<destination_directory>]"
+    echo -e ""
+    echo -e "Important information:"
+    echo -e " * The local user must have valid SSH access to the primary coordinator in the source_cluster."
+    echo -e " * The user on the cluster primary coordinator must have 'sudo' access."
+    echo -e " * If the VM is not in 'stop' state, it will be shut down."
+    echo -e " * Do not switch the cluster primary coordinator while the script is running."
+    echo -e " * Ensure you have enough space in <destination_directory> to store all VM disk images."
+}
+
+fail() {
+    echo -e "$@"
+    exit 1
+}
+
+# Arguments
+if [[ -z ${1} || -z ${2} ]]; then
+    usage
+    exit 1
+fi
+source_vm="${1}"
+source_cluster="${2}"
+if [[ -n "${3}" ]]; then
+    destination_directory="${3}"
+else
+    destination_directory="."
+fi
+
+# Verify the cluster is reachable
+pvc -c ${source_cluster} status &>/dev/null || fail "Specified source_cluster is not accessible"
+
+# Determine the connection IP
+cluster_address="$( pvc cluster list 2>/dev/null | grep -i "^${source_cluster}" | awk '{ print $2 }' )"
+
+# Attempt to connect to the cluster address
+ssh ${cluster_address} which pvc &>/dev/null || fail "Could not SSH to source_cluster primary coordinator host"
+
+# Verify that the VM exists
+pvc -c ${source_cluster} vm info ${source_vm} &>/dev/null || fail "Specified VM is not present on the cluster"
+
+echo "Verification complete."
+
+# Shut down the VM
+echo -n "Shutting down VM..."
+set +o errexit
+pvc -c ${source_cluster} vm shutdown ${source_vm} &>/dev/null
+shutdown_success=$?
+while ! pvc -c ${source_cluster} vm info ${source_vm} 2>/dev/null | grep '^State' | grep -q -E 'stop|disable'; do
+    sleep 1
+    echo -n "."
+done
+set -o errexit
+echo " done."
+
+# Dump the XML file
+echo -n "Exporting VM configuration file... "
+pvc -c ${source_cluster} vm dump ${source_vm} 1> ${destination_directory}/${source_vm}.xml 2>/dev/null
+echo "done".
+
+# Determine the list of volumes in this VM
+volume_list="$( pvc -c ${source_cluster} vm info --long ${source_vm} 2>/dev/null | grep -w 'rbd' | awk '{ print $3 }' )"
+for volume in ${volume_list}; do
+    volume_pool="$( awk -F '/' '{ print $1 }' <<<"${volume}" )"
+    volume_name="$( awk -F '/' '{ print $2 }' <<<"${volume}" )"
+    volume_size="$( pvc -c ${source_cluster} storage volume list -p ${volume_pool} ${volume_name} 2>/dev/null | grep "^${volume_name}" | awk '{ print $3 }' )"
+    echo -n "Exporting disk ${volume_name} (${volume_size})... "
+    ssh ${cluster_address} sudo rbd map ${volume_pool}/${volume_name} &>/dev/null || fail "Failed to map volume ${volume}"
+    ssh ${cluster_address} sudo dd if="/dev/rbd/${volume_pool}/${volume_name}" bs=1M 2>/dev/null | dd bs=1M of="${destination_directory}/${volume_name}.img" 2>/dev/null
+    ssh ${cluster_address} sudo rbd unmap ${volume_pool}/${volume_name} &>/dev/null || fail "Failed to unmap volume ${volume}"
+    echo "done."
+done

client-cli/scripts/force_single_node

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+
+# force_single_node - Manually promote a single coordinator node from a degraded cluster
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+set -o errexit
+set -o pipefail
+
+usage() {
+    echo -e "Manually promote a single coordinator node from a degraded cluster"
+    echo -e ""
+    echo -e "DANGER: This action will cause a permanent split-brain within the cluster"
+    echo -e "        which will have to be corrected manually upon cluster restoration."
+    echo -e ""
+    echo -e "This script is primarily designed for small clusters in situations where 2"
+    echo -e "of the 3 coordinators have become unreachable or shut down. It will promote"
+    echo -e "the remaining lone_node to act as a standalone coordinator, allowing basic"
+    echo -e "cluster functionality to continue in a heavily degraded state until the"
+    echo -e "situation can be rectified. This should only be done in exceptional cases"
+    echo -e "as a disaster recovery mechanism when the remaining nodes will remain down"
+    echo -e "for a significant amount of time but some VMs are required to run. In general,"
+    echo -e "use of this script is not advisable."
+    echo -e ""
+    echo -e "Usage:"
+    echo -e "  $0 <target_cluster> <lone_node>"
+    echo -e ""
+    echo -e "Important information:"
+    echo -e " * The lone_node must be a fully-qualified name that is directly reachable from"
+    echo -e "   the local system via SSH."
+    echo -e " * The local user must have valid SSH access to the lone_node in the cluster."
+    echo -e " * The user on the cluster node must have 'sudo' access."
+}
+
+fail() {
+    echo -e "$@"
+    exit 1
+}
+
+# Arguments
+if [[ -z ${1} || -z ${2} ]]; then
+    usage
+    exit 1
+fi
+target_cluster="${1}"
+lone_node="${2}"
+lone_node_shortname="${lone_node%%.*}"
+
+# Attempt to connect to the node
+ssh ${lone_node} which pvc &>/dev/null || fail "Could not SSH to the lone_node host"
+
+echo "Verification complete."
+
+echo -n "Allowing Ceph single-node operation... "
+temp_monmap="$( ssh ${lone_node} mktemp )"
+ssh ${lone_node} "sudo systemctl stop ceph-mon@${lone_node_shortname}" &>/dev/null
+ssh ${lone_node} "ceph-mon -i ${lone_node_shortname} --extract-monmap ${temp_monmap}" &>/dev/null
+ssh ${lone_node} "sudo cp ${tmp_monmap} /etc/ceph/monmap.orig" &>/dev/null
+mon_list="$( ssh ${lone_node} strings ${temp_monmap} | sort | uniq )"
+for mon in ${mon_list}; do
+    if [[ ${mon} == ${lone_node_shortname} ]]; then
+        continue
+    fi
+    ssh ${lone_node} "sudo monmaptool ${temp_monmap} --rm ${mon}" &>/dev/null
+done
+ssh ${lone_node} "sudo ceph-mon -i ${lone_node_shortname} --inject-monmap ${temp_monmap}" &>/dev/null
+ssh ${lone_node} "sudo systemctl start ceph-mon@${lone_node_shortname}" &>/dev/null
+sleep 5
+ssh ${lone_node} "sudo ceph osd set noout" &>/dev/null
+echo "done."
+echo -e "Restoration steps:"
+echo -e "  sudo systemctl stop ceph-mon@${lone_node_shortname}"
+echo -e "  sudo ceph-mon -i ${lone_node_shortname} --inject-monmap /etc/ceph/monmap.orig"
+echo -e "  sudo systemctl start ceph-mon@${lone_node_shortname}"
+echo -e "  sudo ceph osd unset noout"
+
+echo -n "Allowing Zookeeper single-node operation... "
+temp_zoocfg="$( ssh ${lone_node} mktemp )"
+ssh ${lone_node} "sudo systemctl stop zookeeper"
+ssh ${lone_node} "sudo awk -v lone_node=${lone_node_shortname} '{
+FS="=|:"
+if ( $1 ~ /^server/ ){
+    if ($2 == lone_node) {
+        print $0
+    } else {
+        print "#" $0
+    }
+} else {
+    print $0
+}
+}' /etc/zookeeper/conf/zoo.cfg > ${temp_zoocfg}"
+ssh ${lone_node} "sudo mv /etc/zookeeper/conf/zoo.cfg /etc/zookeeper/conf/zoo.cfg.orig"
+ssh ${lone_node} "sudo mv ${temp_zoocfg} /etc/zookeeper/conf/zoo.cfg"
+ssh ${lone_node} "sudo systemctl start zookeeper"
+echo "done."
+echo -e "Restoration steps:"
+echo -e "  sudo systemctl stop zookeeper"
+echo -e "  sudo mv /etc/zookeeper/conf/zoo.cfg.orig /etc/zookeeper/conf/zoo.cfg"
+echo -e "  sudo systemctl start zookeeper"
+ssh ${lone_node} "sudo systemctl stop ceph-mon@${lone_node_shortname}"
+
+echo ""
+ssh ${lone_node} "sudo pvc status 2>/dev/null"

client-cli/scripts/import_vm

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+# import_vm - Imports a VM to a PVC cluster from local files
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+set -o errexit
+set -o pipefail
+
+usage() {
+    echo -e "Import a VM to a PVC cluster from local files."
+    echo -e "Usage:"
+    echo -e "  $0 <destination_cluster> <destination_pool> <vm_configuration_file> <vm_disk_file_1> [<vm_disk_file_2>] [...]"
+    echo -e ""
+    echo -e "Important information:"
+    echo -e " * At least one disk must be specified; all disks that are present in vm_configuration_file"
+    echo -e "   should be specified, though this is not strictly requireda."
+    echo -e " * Do not switch the cluster primary coordinator while the script is running."
+    echo -e " * Ensure you have enough space on the destination cluster to store all VM disks."
+}
+
+fail() {
+    echo -e "$@"
+    exit 1
+}
+
+# Arguments
+if [[ -z ${1} || -z ${2} || -z ${3} || -z ${4} ]]; then
+    usage
+    exit 1
+fi
+destination_cluster="${1}"; shift
+destination_pool="${1}"; shift
+vm_config_file="${1}"; shift
+vm_disk_files=( ${@} )
+
+# Verify the cluster is reachable
+pvc -c ${destination_cluster} status &>/dev/null || fail "Specified destination_cluster is not accessible"
+
+# Determine the connection IP
+cluster_address="$( pvc cluster list 2>/dev/null | grep -i "^${destination_cluster}" | awk '{ print $2 }' )"
+
+echo "Verification complete."
+
+# Determine information about the VM from the config file
+parse_xml_field() {
+    field="${1}"
+    line="$( grep -F "<${field}>" ${vm_config_file} )"
+    awk -F '>|<' '{ print $3 }' <<<"${line}"
+}
+vm_name="$( parse_xml_field name )"
+echo "Importing VM ${vm_name}..."
+pvc -c ${destination_cluster} vm define ${vm_config_file} 2>/dev/null
+
+# Create the disks on the cluster
+for disk_file in ${vm_disk_files[@]}; do
+    disk_file_basename="$( basename ${disk_file} )"
+    disk_file_ext="${disk_file_basename##*.}"
+    disk_file_name="$( basename ${disk_file_basename} .${disk_file_ext} )"
+    disk_file_size="$( stat --format="%s" ${disk_file} )"
+
+    echo "Importing disk ${disk_file_name}... "
+    pvc -c ${destination_cluster} storage volume add ${destination_pool} ${disk_file_name} ${disk_file_size}B 2>/dev/null
+    pvc -c ${destination_cluster} storage volume upload ${destination_pool} ${disk_file_name} ${disk_file} 2>/dev/null
+done

client-cli/scripts/migrate_vm

@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+
+# migrate_vm - Exports a VM from a PVC cluster to another PVC cluster
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+set -o errexit
+set -o pipefail
+
+usage() {
+    echo -e "Export a VM from a PVC cluster to another PVC cluster."
+    echo -e "Usage:"
+    echo -e "  $0 <vm> <source_cluster> <destination_cluster> <destination_pool>"
+    echo -e ""
+    echo -e "Important information:"
+    echo -e " * The local user must have valid SSH access to the primary coordinator in the source_cluster."
+    echo -e " * The user on the cluster primary coordinator must have 'sudo' access."
+    echo -e " * If the VM is not in 'stop' state, it will be shut down."
+    echo -e " * Do not switch the cluster primary coordinator on either cluster while the script is running."
+    echo -e " * Ensure you have enough space on the target cluster to store all VM disks."
+}
+
+fail() {
+    echo -e "$@"
+    exit 1
+}
+
+# Arguments
+if [[ -z ${1} || -z ${2} || -z ${3} || -z ${4} ]]; then
+    usage
+    exit 1
+fi
+source_vm="${1}"
+source_cluster="${2}"
+destination_cluster="${3}"
+destination_pool="${4}"
+
+# Verify each cluster is reachable
+pvc -c ${source_cluster} status &>/dev/null || fail "Specified source_cluster is not accessible"
+pvc -c ${destination_cluster} status &>/dev/null || fail "Specified destination_cluster is not accessible"
+
+# Determine the connection IPs
+source_cluster_address="$( pvc cluster list 2>/dev/null | grep -i "^${source_cluster}" | awk '{ print $2 }' )"
+destination_cluster_address="$( pvc cluster list 2>/dev/null | grep -i "^${destination_cluster}" | awk '{ print $2 }' )"
+
+# Attempt to connect to the cluster addresses
+ssh ${source_cluster_address} which pvc &>/dev/null || fail "Could not SSH to source_cluster primary coordinator host"
+ssh ${destination_cluster_address} which pvc &>/dev/null || fail "Could not SSH to destination_cluster primary coordinator host"
+
+# Verify that the VM exists
+pvc -c ${source_cluster} vm info ${source_vm} &>/dev/null || fail "Specified VM is not present on the source cluster"
+
+echo "Verification complete."
+
+# Shut down the VM
+echo -n "Shutting down VM..."
+set +o errexit
+pvc -c ${source_cluster} vm shutdown ${source_vm} &>/dev/null
+shutdown_success=$?
+while ! pvc -c ${source_cluster} vm info ${source_vm} 2>/dev/null | grep '^State' | grep -q -E 'stop|disable'; do
+    sleep 1
+    echo -n "."
+done
+set -o errexit
+echo " done."
+
+tempfile="$( mktemp )"
+
+# Dump the XML file
+echo -n "Exporting VM configuration file from source cluster... "
+pvc -c ${source_cluster} vm dump ${source_vm} 1> ${tempfile} 2>/dev/null
+echo "done."
+
+# Import the XML file
+echo -n "Importing VM configuration file to destination cluster... "
+pvc -c ${destination_cluster} vm define ${tempfile}
+echo "done."
+
+rm -f ${tempfile}
+
+# Determine the list of volumes in this VM
+volume_list="$( pvc -c ${source_cluster} vm info --long ${source_vm} 2>/dev/null | grep -w 'rbd' | awk '{ print $3 }' )"
+
+# Parse and migrate each volume
+for volume in ${volume_list}; do
+    volume_pool="$( awk -F '/' '{ print $1 }' <<<"${volume}" )"
+    volume_name="$( awk -F '/' '{ print $2 }' <<<"${volume}" )"
+    volume_size="$( pvc -c ${source_cluster} storage volume list -p ${volume_pool} ${volume_name} 2>/dev/null | grep "^${volume_name}" | awk '{ print $3 }' )"
+    echo "Transferring disk ${volume_name} (${volume_size})... "
+    pvc -c ${destination_cluster} storage volume add ${destination_pool} ${volume_name} ${volume_size} 2>/dev/null
+    ssh ${source_cluster_address} sudo rbd map ${volume_pool}/${volume_name} &>/dev/null || fail "Failed to map volume ${volume} on source cluster"
+    ssh ${destination_cluster_address} sudo rbd map ${volume_pool}/${volume_name} &>/dev/null || fail "Failed to map volume ${volume} on destination cluster"
+    ssh ${source_cluster_address} sudo dd if="/dev/rbd/${volume_pool}/${volume_name}" bs=1M 2>/dev/null | pv | ssh ${destination_cluster_address} sudo dd bs=1M of="/dev/rbd/${destination_pool}/${volume_name}" 2>/dev/null
+    ssh ${source_cluster_address} sudo rbd unmap ${volume_pool}/${volume_name} &>/dev/null || fail "Failed to unmap volume ${volume} on source cluster"
+    ssh ${destination_cluster_address} sudo rbd unmap ${volume_pool}/${volume_name} &>/dev/null || fail "Failed to unmap volume ${volume} on destination cluster"
+done
+
+if [[ ${shutdown_success} -eq 0 ]]; then
+    pvc -c ${destination_cluster} vm start ${source_vm}
+fi

client-common/ansiprint.py

@@ -1,82 +0,0 @@
-#!/usr/bin/env python3
-
-# ansiprint.py - Printing function for formatted messages
-# Part of the Parallel Virtual Cluster (PVC) system
-#
-#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-###############################################################################
-
-import datetime
-
-# ANSII colours for output
-def red():
-    return '\033[91m'
-def blue():
-    return '\033[94m'
-def cyan():
-    return '\033[96m'
-def green():
-    return '\033[92m'
-def yellow():
-    return '\033[93m'
-def purple():
-    return '\033[95m'
-def bold():
-    return '\033[1m'
-def end():
-    return '\033[0m'
-
-# Print function
-def echo(message, prefix, state):
-    # Get the date
-    date = '{} - '.format(datetime.datetime.now().strftime('%Y/%m/%d %H:%M:%S.%f'))
-    endc = end()
-
-    # Continuation
-    if state == 'c':
-        date = ''
-        colour = ''
-        prompt = '    '
-    # OK
-    elif state == 'o':
-        colour = green()
-        prompt = '>>> '
-    # Error
-    elif state == 'e':
-        colour = red()
-        prompt = '>>> '
-    # Warning
-    elif state == 'w':
-        colour = yellow()
-        prompt = '>>> '
-    # Tick
-    elif state == 't':
-        colour = purple()
-        prompt = '>>> '
-    # Information
-    elif state == 'i':
-        colour = blue()
-        prompt = '>>> '
-    else:
-        colour = bold()
-        prompt = '>>> '
-
-    # Append space to prefix
-    if prefix != '':
-        prefix = prefix + ' '
-
-    print(colour + prompt + endc + date + prefix + message)

client-common/node.py

@@ -1,416 +0,0 @@
-#!/usr/bin/env python3
-
-# node.py - PVC client function library, node management
-# Part of the Parallel Virtual Cluster (PVC) system
-#
-#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-###############################################################################
-
-import os
-import socket
-import time
-import uuid
-import re
-import tempfile
-import subprocess
-import difflib
-import colorama
-import click
-import lxml.objectify
-import configparser
-import kazoo.client
-
-import client_lib.ansiprint as ansiprint
-import client_lib.zkhandler as zkhandler
-import client_lib.common as common
-import client_lib.vm as pvc_vm
-
-def getNodeInformation(zk_conn, node_name):
-    """
-    Gather information about a node from the Zookeeper database and return a dict() containing it.
-    """
-    node_daemon_state = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(node_name))
-    node_coordinator_state = zkhandler.readdata(zk_conn, '/nodes/{}/routerstate'.format(node_name))
-    node_domain_state = zkhandler.readdata(zk_conn, '/nodes/{}/domainstate'.format(node_name))
-    node_static_data = zkhandler.readdata(zk_conn, '/nodes/{}/staticdata'.format(node_name)).split()
-    node_cpu_count = int(node_static_data[0])
-    node_kernel = node_static_data[1]
-    node_os = node_static_data[2]
-    node_arch = node_static_data[3]
-    node_vcpu_allocated = int(zkhandler.readdata(zk_conn, 'nodes/{}/vcpualloc'.format(node_name)))
-    node_mem_total = int(zkhandler.readdata(zk_conn, '/nodes/{}/memtotal'.format(node_name)))
-    node_mem_allocated = int(zkhandler.readdata(zk_conn, '/nodes/{}/memalloc'.format(node_name)))
-    node_mem_used = int(zkhandler.readdata(zk_conn, '/nodes/{}/memused'.format(node_name)))
-    node_mem_free = int(zkhandler.readdata(zk_conn, '/nodes/{}/memfree'.format(node_name)))
-    node_load = float(zkhandler.readdata(zk_conn, '/nodes/{}/cpuload'.format(node_name)))
-    node_domains_count = int(zkhandler.readdata(zk_conn, '/nodes/{}/domainscount'.format(node_name)))
-    node_running_domains = zkhandler.readdata(zk_conn, '/nodes/{}/runningdomains'.format(node_name)).split()
-
-    # Construct a data structure to represent the data
-    node_information = {
-        'name': node_name,
-        'daemon_state': node_daemon_state,
-        'coordinator_state': node_coordinator_state,
-        'domain_state': node_domain_state,
-        'cpu_count': node_cpu_count,
-        'kernel': node_kernel,
-        'os': node_os,
-        'arch': node_arch,
-        'load': node_load,
-        'domains_count': node_domains_count,
-        'running_domains': node_running_domains,
-        'vcpu': {
-            'total': node_cpu_count,
-            'allocated': node_vcpu_allocated
-        },
-        'memory': {
-            'total': node_mem_total,
-            'allocated': node_mem_allocated,
-            'used': node_mem_used,
-            'free': node_mem_free
-        }
-    }
-    return node_information
-
-#
-# Direct Functions
-#
-def secondary_node(zk_conn, node):
-    # Verify node is valid
-    if not common.verifyNode(zk_conn, node):
-        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
-
-    # Ensure node is a coordinator
-    daemon_mode = zkhandler.readdata(zk_conn, '/nodes/{}/daemonmode'.format(node))
-    if daemon_mode == 'hypervisor':
-        return False, 'ERROR: Cannot change router mode on non-coordinator node "{}"'.format(node)
-
-    # Ensure node is in run daemonstate
-    daemon_state = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(node))
-    if daemon_state != 'run':
-        return False, 'ERROR: Node "{}" is not active'.format(node)
-
-    # Get current state
-    current_state = zkhandler.readdata(zk_conn, '/nodes/{}/routerstate'.format(node))
-    if current_state == 'primary':
-        retmsg = 'Setting node {} in secondary router mode.'.format(node)
-        zkhandler.writedata(zk_conn, {
-            '/primary_node': 'none'
-        })
-    else:
-        return False, 'Node "{}" is already in secondary router mode.'.format(node)
-
-    return True, retmsg
-
-def primary_node(zk_conn, node):
-    # Verify node is valid
-    if not common.verifyNode(zk_conn, node):
-        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
-
-    # Ensure node is a coordinator
-    daemon_mode = zkhandler.readdata(zk_conn, '/nodes/{}/daemonmode'.format(node))
-    if daemon_mode == 'hypervisor':
-        return False, 'ERROR: Cannot change router mode on non-coordinator node "{}"'.format(node)
-
-    # Ensure node is in run daemonstate
-    daemon_state = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(node))
-    if daemon_state != 'run':
-        return False, 'ERROR: Node "{}" is not active'.format(node)
-
-    # Get current state
-    current_state = zkhandler.readdata(zk_conn, '/nodes/{}/routerstate'.format(node))
-    if current_state == 'secondary':
-        retmsg = 'Setting node {} in primary router mode.'.format(node)
-        zkhandler.writedata(zk_conn, {
-            '/primary_node': node
-        })
-    else:
-        return False, 'Node "{}" is already in primary router mode.'.format(node)
-
-    return True, retmsg
-
-def flush_node(zk_conn, node, wait):
-    # Verify node is valid
-    if not common.verifyNode(zk_conn, node):
-        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
-
-    retmsg = 'Flushing hypervisor {} of running VMs.'.format(node)
-
-    # Add the new domain to Zookeeper
-    zkhandler.writedata(zk_conn, {
-        '/nodes/{}/domainstate'.format(node): 'flush'
-    })
-
-    # Wait cannot be triggered from the API
-    if wait:
-        while zkhandler.readdata(zk_conn, '/nodes/{}/domainstate'.format(node)) == 'flush':
-            time.sleep(1)
-        retmsg = 'Flushed hypervisor {} of running VMs.'.format(node)
-
-    return True, retmsg
-
-def ready_node(zk_conn, node, wait):
-    # Verify node is valid
-    if not common.verifyNode(zk_conn, node):
-        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
-
-    retmsg = 'Restoring hypervisor {} to active service.'.format(node)
-        
-    # Add the new domain to Zookeeper
-    zkhandler.writedata(zk_conn, {
-        '/nodes/{}/domainstate'.format(node): 'unflush'
-    })
-
-    # Wait cannot be triggered from the API
-    if wait:
-        while zkhandler.readdata(zk_conn, '/nodes/{}/domainstate'.format(node)) == 'unflush':
-            time.sleep(1)
-        retmsg = 'Restored hypervisor {} to active service.'.format(node)
-
-    return True, retmsg
-
-def get_info(zk_conn, node):
-    # Verify node is valid
-    if not common.verifyNode(zk_conn, node):
-        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
-
-    # Get information about node in a pretty format
-    node_information = getNodeInformation(zk_conn, node)
-    if not node_information:
-        return False, 'ERROR: Could not get information about node "{}".'.format(node)
-
-    return True, node_information
-
-def get_list(zk_conn, limit, is_fuzzy=True):
-    node_list = []
-    full_node_list = zkhandler.listchildren(zk_conn, '/nodes')
-
-    for node in full_node_list:
-        if limit:
-            try:
-                if not is_fuzzy:
-                    limit = '^' + limit + '$'
-
-                if re.match(limit, node):
-                    node_list.append(getNodeInformation(zk_conn, node))
-            except Exception as e:
-                return False, 'Regex Error: {}'.format(e)
-        else:
-            node_list.append(getNodeInformation(zk_conn, node))
-
-    return True, node_list
-
-#
-# CLI-specific functions
-#
-def getOutputColours(node_information):
-    if node_information['daemon_state'] == 'run':
-        daemon_state_colour = ansiprint.green()
-    elif node_information['daemon_state'] == 'stop':
-        daemon_state_colour = ansiprint.red()
-    elif node_information['daemon_state'] == 'shutdown':
-        daemon_state_colour = ansiprint.yellow()
-    elif node_information['daemon_state'] == 'init':
-        daemon_state_colour = ansiprint.yellow()
-    elif node_information['daemon_state'] == 'dead':
-        daemon_state_colour = ansiprint.red() + ansiprint.bold()
-    else:
-        daemon_state_colour = ansiprint.blue()
-
-    if node_information['coordinator_state'] == 'primary':
-        coordinator_state_colour = ansiprint.green()
-    elif node_information['coordinator_state'] == 'secondary':
-        coordinator_state_colour = ansiprint.blue()
-    else:
-        coordinator_state_colour = ansiprint.cyan()
-
-    if node_information['domain_state'] == 'ready':
-        domain_state_colour = ansiprint.green()
-    else:
-        domain_state_colour = ansiprint.blue()
-
-    return daemon_state_colour, coordinator_state_colour, domain_state_colour
-
-def format_info(node_information, long_output):
-    daemon_state_colour, coordinator_state_colour, domain_state_colour = getOutputColours(node_information)
-
-    # Format a nice output; do this line-by-line then concat the elements at the end
-    ainformation = []
-    # Basic information
-    ainformation.append('{}Name:{}                 {}'.format(ansiprint.purple(), ansiprint.end(), node_information['name']))
-    ainformation.append('{}Daemon State:{}         {}{}{}'.format(ansiprint.purple(), ansiprint.end(), daemon_state_colour, node_information['daemon_state'], ansiprint.end()))
-    ainformation.append('{}Coordinator State:{}    {}{}{}'.format(ansiprint.purple(), ansiprint.end(), coordinator_state_colour, node_information['coordinator_state'], ansiprint.end()))
-    ainformation.append('{}Domain State:{}         {}{}{}'.format(ansiprint.purple(), ansiprint.end(), domain_state_colour, node_information['domain_state'], ansiprint.end()))
-    ainformation.append('{}Active VM Count:{}      {}'.format(ansiprint.purple(), ansiprint.end(), node_information['domains_count']))
-    if long_output:
-        ainformation.append('')
-        ainformation.append('{}Architecture:{}         {}'.format(ansiprint.purple(), ansiprint.end(), node_information['arch']))
-        ainformation.append('{}Operating System:{}     {}'.format(ansiprint.purple(), ansiprint.end(), node_information['os']))
-        ainformation.append('{}Kernel Version:{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_information['kernel']))
-    ainformation.append('')
-    ainformation.append('{}Host CPUs:{}            {}'.format(ansiprint.purple(), ansiprint.end(), node_information['vcpu']['total']))
-    ainformation.append('{}vCPUs:{}                {}'.format(ansiprint.purple(), ansiprint.end(), node_information['vcpu']['allocated']))
-    ainformation.append('{}Load:{}                 {}'.format(ansiprint.purple(), ansiprint.end(), node_information['load']))
-    ainformation.append('{}Total RAM (MiB):{}      {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['total']))
-    ainformation.append('{}Used RAM (MiB):{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['used']))
-    ainformation.append('{}Free RAM (MiB):{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['free']))
-    ainformation.append('{}Allocated RAM (MiB):{}  {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['allocated']))
-
-    # Join it all together
-    information = '\n'.join(ainformation)
-    click.echo(information)
-
-    click.echo('')
-
-def format_list(node_list):
-    node_list_output = []
-
-    # Determine optimal column widths
-    node_name_length = 5
-    daemon_state_length = 7
-    coordinator_state_length = 12
-    domain_state_length = 8
-    domains_count_length = 4
-    cpu_count_length = 6
-    load_length = 5
-    mem_total_length = 6
-    mem_used_length = 5
-    mem_free_length = 5
-    mem_alloc_length = 4
-    for node_information in node_list:
-        # node_name column
-        _node_name_length = len(node_information['name']) + 1
-        if _node_name_length > node_name_length:
-            node_name_length = _node_name_length
-        # daemon_state column
-        _daemon_state_length = len(node_information['daemon_state']) + 1
-        if _daemon_state_length > daemon_state_length:
-            daemon_state_length = _daemon_state_length
-        # coordinator_state column
-        _coordinator_state_length = len(node_information['coordinator_state']) + 1
-        if _coordinator_state_length > coordinator_state_length:
-            coordinator_state_length = _coordinator_state_length
-        # domain_state column
-        _domain_state_length = len(node_information['domain_state']) + 1
-        if _domain_state_length > domain_state_length:
-            domain_state_length = _domain_state_length
-        # domains_count column
-        _domains_count_length = len(str(node_information['domains_count'])) + 1
-        if _domains_count_length > domains_count_length:
-            domains_count_length = _domains_count_length
-        # cpu_count column
-        _cpu_count_length = len(str(node_information['cpu_count'])) + 1
-        if _cpu_count_length > cpu_count_length:
-            cpu_count_length = _cpu_count_length
-        # load column
-        _load_length = len(str(node_information['load'])) + 1
-        if _load_length > load_length:
-            load_length = _load_length
-        # mem_total column
-        _mem_total_length = len(str(node_information['memory']['total'])) + 1
-        if _mem_total_length > mem_total_length:
-            mem_total_length = _mem_total_length
-        # mem_used column
-        _mem_used_length = len(str(node_information['memory']['used'])) + 1
-        if _mem_used_length > mem_used_length:
-            mem_used_length = _mem_used_length
-        # mem_free column
-        _mem_free_length = len(str(node_information['memory']['free'])) + 1
-        if _mem_free_length > mem_free_length:
-            mem_free_length = _mem_free_length
-        # mem_alloc column
-        _mem_alloc_length = len(str(node_information['memory']['allocated'])) + 1
-        if _mem_alloc_length > mem_alloc_length:
-            mem_alloc_length = _mem_alloc_length
-
-    # Format the string (header)
-    node_list_output.append(
-        '{bold}{node_name: <{node_name_length}} \
-St: {daemon_state_colour}{node_daemon_state: <{daemon_state_length}}{end_colour} {coordinator_state_colour}{node_coordinator_state: <{coordinator_state_length}}{end_colour} {domain_state_colour}{node_domain_state: <{domain_state_length}}{end_colour} \
-Res: {node_domains_count: <{domains_count_length}} {node_cpu_count: <{cpu_count_length}} {node_load: <{load_length}} \
-Mem (M): {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {node_mem_allocated: <{mem_alloc_length}}{end_bold}'.format(
-            node_name_length=node_name_length,
-            daemon_state_length=daemon_state_length,
-            coordinator_state_length=coordinator_state_length,
-            domain_state_length=domain_state_length,
-            domains_count_length=domains_count_length,
-            cpu_count_length=cpu_count_length,
-            load_length=load_length,
-            mem_total_length=mem_total_length,
-            mem_used_length=mem_used_length,
-            mem_free_length=mem_free_length,
-            mem_alloc_length=mem_alloc_length,
-            bold=ansiprint.bold(),
-            end_bold=ansiprint.end(),
-            daemon_state_colour='',
-            coordinator_state_colour='',
-            domain_state_colour='',
-            end_colour='',
-            node_name='Name',
-            node_daemon_state='Daemon',
-            node_coordinator_state='Coordinator',
-            node_domain_state='Domain',
-            node_domains_count='VMs',
-            node_cpu_count='vCPUs',
-            node_load='Load',
-            node_mem_total='Total',
-            node_mem_used='Used',
-            node_mem_free='Free',
-            node_mem_allocated='VMs'
-        )
-    )
-            
-    # Format the string (elements)
-    for node_information in node_list:
-        daemon_state_colour, coordinator_state_colour, domain_state_colour = getOutputColours(node_information)
-        node_list_output.append(
-            '{bold}{node_name: <{node_name_length}} \
-    {daemon_state_colour}{node_daemon_state: <{daemon_state_length}}{end_colour} {coordinator_state_colour}{node_coordinator_state: <{coordinator_state_length}}{end_colour} {domain_state_colour}{node_domain_state: <{domain_state_length}}{end_colour} \
-     {node_domains_count: <{domains_count_length}} {node_cpu_count: <{cpu_count_length}} {node_load: <{load_length}} \
-         {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {node_mem_allocated: <{mem_alloc_length}}{end_bold}'.format(
-                node_name_length=node_name_length,
-                daemon_state_length=daemon_state_length,
-                coordinator_state_length=coordinator_state_length,
-                domain_state_length=domain_state_length,
-                domains_count_length=domains_count_length,
-                cpu_count_length=cpu_count_length,
-                load_length=load_length,
-                mem_total_length=mem_total_length,
-                mem_used_length=mem_used_length,
-                mem_free_length=mem_free_length,
-                mem_alloc_length=mem_alloc_length,
-                bold='',
-                end_bold='',
-                daemon_state_colour=daemon_state_colour,
-                coordinator_state_colour=coordinator_state_colour,
-                domain_state_colour=domain_state_colour,
-                end_colour=ansiprint.end(),
-                node_name=node_information['name'],
-                node_daemon_state=node_information['daemon_state'],
-                node_coordinator_state=node_information['coordinator_state'],
-                node_domain_state=node_information['domain_state'],
-                node_domains_count=node_information['domains_count'],
-                node_cpu_count=node_information['vcpu']['allocated'],
-                node_load=node_information['load'],
-                node_mem_total=node_information['memory']['total'],
-                node_mem_used=node_information['memory']['used'],
-                node_mem_free=node_information['memory']['free'],
-                node_mem_allocated=node_information['memory']['allocated']
-            )
-        )
-
-    click.echo('\n'.join(sorted(node_list_output)))

daemon-common/ceph.py

@@ -0,0 +1,784 @@
+#!/usr/bin/env python3
+
+# ceph.py - PVC client function library, Ceph cluster fuctions
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import os
+import re
+import json
+import time
+import math
+
+import daemon_lib.vm as vm
+import daemon_lib.zkhandler as zkhandler
+import daemon_lib.common as common
+
+
+#
+# Supplemental functions
+#
+
+# Verify OSD is valid in cluster
+def verifyOSD(zk_conn, osd_id):
+    if zkhandler.exists(zk_conn, '/ceph/osds/{}'.format(osd_id)):
+        return True
+    else:
+        return False
+
+
+# Verify Pool is valid in cluster
+def verifyPool(zk_conn, name):
+    if zkhandler.exists(zk_conn, '/ceph/pools/{}'.format(name)):
+        return True
+    else:
+        return False
+
+
+# Verify Volume is valid in cluster
+def verifyVolume(zk_conn, pool, name):
+    if zkhandler.exists(zk_conn, '/ceph/volumes/{}/{}'.format(pool, name)):
+        return True
+    else:
+        return False
+
+
+# Verify Snapshot is valid in cluster
+def verifySnapshot(zk_conn, pool, volume, name):
+    if zkhandler.exists(zk_conn, '/ceph/snapshots/{}/{}/{}'.format(pool, volume, name)):
+        return True
+    else:
+        return False
+
+
+# Verify OSD path is valid in cluster
+def verifyOSDBlock(zk_conn, node, device):
+    for osd in zkhandler.listchildren(zk_conn, '/ceph/osds'):
+        osd_node = zkhandler.readdata(zk_conn, '/ceph/osds/{}/node'.format(osd))
+        osd_device = zkhandler.readdata(zk_conn, '/ceph/osds/{}/device'.format(osd))
+        if node == osd_node and device == osd_device:
+            return osd
+    return None
+
+
+# Matrix of human-to-byte values
+byte_unit_matrix = {
+    'B': 1,
+    'K': 1024,
+    'M': 1024 * 1024,
+    'G': 1024 * 1024 * 1024,
+    'T': 1024 * 1024 * 1024 * 1024,
+    'P': 1024 * 1024 * 1024 * 1024 * 1024
+}
+
+# Matrix of human-to-metric values
+ops_unit_matrix = {
+    '': 1,
+    'K': 1000,
+    'M': 1000 * 1000,
+    'G': 1000 * 1000 * 1000,
+    'T': 1000 * 1000 * 1000 * 1000,
+    'P': 1000 * 1000 * 1000 * 1000 * 1000
+}
+
+
+# Format byte sizes to/from human-readable units
+def format_bytes_tohuman(databytes):
+    datahuman = ''
+    for unit in sorted(byte_unit_matrix, key=byte_unit_matrix.get, reverse=True):
+        new_bytes = int(math.ceil(databytes / byte_unit_matrix[unit]))
+        # Round up if 5 or more digits
+        if new_bytes > 9999:
+            # We can jump down another level
+            continue
+        else:
+            # We're at the end, display with this size
+            datahuman = '{}{}'.format(new_bytes, unit)
+
+    return datahuman
+
+
+def format_bytes_fromhuman(datahuman):
+    # Trim off human-readable character
+    dataunit = str(datahuman)[-1]
+    datasize = int(str(datahuman)[:-1])
+    if not re.match(r'[A-Z]', dataunit):
+        dataunit = 'B'
+        datasize = int(datahuman)
+    databytes = datasize * byte_unit_matrix[dataunit]
+    return '{}B'.format(databytes)
+
+
+# Format ops sizes to/from human-readable units
+def format_ops_tohuman(dataops):
+    datahuman = ''
+    for unit in sorted(ops_unit_matrix, key=ops_unit_matrix.get, reverse=True):
+        new_ops = int(math.ceil(dataops / ops_unit_matrix[unit]))
+        # Round up if 5 or more digits
+        if new_ops > 9999:
+            # We can jump down another level
+            continue
+        else:
+            # We're at the end, display with this size
+            datahuman = '{}{}'.format(new_ops, unit)
+
+    return datahuman
+
+
+def format_ops_fromhuman(datahuman):
+    # Trim off human-readable character
+    dataunit = datahuman[-1]
+    datasize = int(datahuman[:-1])
+    dataops = datasize * ops_unit_matrix[dataunit]
+    return '{}'.format(dataops)
+
+
+def format_pct_tohuman(datapct):
+    datahuman = "{0:.1f}".format(float(datapct * 100.0))
+    return datahuman
+
+
+#
+# Status functions
+#
+def get_status(zk_conn):
+    primary_node = zkhandler.readdata(zk_conn, '/primary_node')
+    ceph_status = zkhandler.readdata(zk_conn, '/ceph').rstrip()
+
+    # Create a data structure for the information
+    status_data = {
+        'type': 'status',
+        'primary_node': primary_node,
+        'ceph_data': ceph_status
+    }
+    return True, status_data
+
+
+def get_util(zk_conn):
+    primary_node = zkhandler.readdata(zk_conn, '/primary_node')
+    ceph_df = zkhandler.readdata(zk_conn, '/ceph/util').rstrip()
+
+    # Create a data structure for the information
+    status_data = {
+        'type': 'utilization',
+        'primary_node': primary_node,
+        'ceph_data': ceph_df
+    }
+    return True, status_data
+
+
+#
+# OSD functions
+#
+def getClusterOSDList(zk_conn):
+    # Get a list of VNIs by listing the children of /networks
+    osd_list = zkhandler.listchildren(zk_conn, '/ceph/osds')
+    return osd_list
+
+
+def getOSDInformation(zk_conn, osd_id):
+    # Parse the stats data
+    osd_stats_raw = zkhandler.readdata(zk_conn, '/ceph/osds/{}/stats'.format(osd_id))
+    osd_stats = dict(json.loads(osd_stats_raw))
+
+    osd_information = {
+        'id': osd_id,
+        'stats': osd_stats
+    }
+    return osd_information
+
+
+# OSD addition and removal uses the /cmd/ceph pipe
+# These actions must occur on the specific node they reference
+def add_osd(zk_conn, node, device, weight):
+    # Verify the target node exists
+    if not common.verifyNode(zk_conn, node):
+        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
+
+    # Verify target block device isn't in use
+    block_osd = verifyOSDBlock(zk_conn, node, device)
+    if block_osd:
+        return False, 'ERROR: Block device "{}" on node "{}" is used by OSD "{}"'.format(device, node, block_osd)
+
+    # Tell the cluster to create a new OSD for the host
+    add_osd_string = 'osd_add {},{},{}'.format(node, device, weight)
+    zkhandler.writedata(zk_conn, {'/cmd/ceph': add_osd_string})
+    # Wait 1/2 second for the cluster to get the message and start working
+    time.sleep(0.5)
+    # Acquire a read lock, so we get the return exclusively
+    lock = zkhandler.readlock(zk_conn, '/cmd/ceph')
+    with lock:
+        try:
+            result = zkhandler.readdata(zk_conn, '/cmd/ceph').split()[0]
+            if result == 'success-osd_add':
+                message = 'Created new OSD with block device "{}" on node "{}".'.format(device, node)
+                success = True
+            else:
+                message = 'ERROR: Failed to create new OSD; check node logs for details.'
+                success = False
+        except Exception:
+            message = 'ERROR: Command ignored by node.'
+            success = False
+
+    # Acquire a write lock to ensure things go smoothly
+    lock = zkhandler.writelock(zk_conn, '/cmd/ceph')
+    with lock:
+        time.sleep(0.5)
+        zkhandler.writedata(zk_conn, {'/cmd/ceph': ''})
+
+    return success, message
+
+
+def remove_osd(zk_conn, osd_id):
+    if not verifyOSD(zk_conn, osd_id):
+        return False, 'ERROR: No OSD with ID "{}" is present in the cluster.'.format(osd_id)
+
+    # Tell the cluster to remove an OSD
+    remove_osd_string = 'osd_remove {}'.format(osd_id)
+    zkhandler.writedata(zk_conn, {'/cmd/ceph': remove_osd_string})
+    # Wait 1/2 second for the cluster to get the message and start working
+    time.sleep(0.5)
+    # Acquire a read lock, so we get the return exclusively
+    lock = zkhandler.readlock(zk_conn, '/cmd/ceph')
+    with lock:
+        try:
+            result = zkhandler.readdata(zk_conn, '/cmd/ceph').split()[0]
+            if result == 'success-osd_remove':
+                message = 'Removed OSD "{}" from the cluster.'.format(osd_id)
+                success = True
+            else:
+                message = 'ERROR: Failed to remove OSD; check node logs for details.'
+                success = False
+        except Exception:
+            success = False
+            message = 'ERROR Command ignored by node.'
+
+    # Acquire a write lock to ensure things go smoothly
+    lock = zkhandler.writelock(zk_conn, '/cmd/ceph')
+    with lock:
+        time.sleep(0.5)
+        zkhandler.writedata(zk_conn, {'/cmd/ceph': ''})
+
+    return success, message
+
+
+def in_osd(zk_conn, osd_id):
+    if not verifyOSD(zk_conn, osd_id):
+        return False, 'ERROR: No OSD with ID "{}" is present in the cluster.'.format(osd_id)
+
+    retcode, stdout, stderr = common.run_os_command('ceph osd in {}'.format(osd_id))
+    if retcode:
+        return False, 'ERROR: Failed to enable OSD {}: {}'.format(osd_id, stderr)
+
+    return True, 'Set OSD {} online.'.format(osd_id)
+
+
+def out_osd(zk_conn, osd_id):
+    if not verifyOSD(zk_conn, osd_id):
+        return False, 'ERROR: No OSD with ID "{}" is present in the cluster.'.format(osd_id)
+
+    retcode, stdout, stderr = common.run_os_command('ceph osd out {}'.format(osd_id))
+    if retcode:
+        return False, 'ERROR: Failed to disable OSD {}: {}'.format(osd_id, stderr)
+
+    return True, 'Set OSD {} offline.'.format(osd_id)
+
+
+def set_osd(zk_conn, option):
+    retcode, stdout, stderr = common.run_os_command('ceph osd set {}'.format(option))
+    if retcode:
+        return False, 'ERROR: Failed to set property "{}": {}'.format(option, stderr)
+
+    return True, 'Set OSD property "{}".'.format(option)
+
+
+def unset_osd(zk_conn, option):
+    retcode, stdout, stderr = common.run_os_command('ceph osd unset {}'.format(option))
+    if retcode:
+        return False, 'ERROR: Failed to unset property "{}": {}'.format(option, stderr)
+
+    return True, 'Unset OSD property "{}".'.format(option)
+
+
+def get_list_osd(zk_conn, limit, is_fuzzy=True):
+    osd_list = []
+    full_osd_list = zkhandler.listchildren(zk_conn, '/ceph/osds')
+
+    if is_fuzzy and limit:
+        # Implicitly assume fuzzy limits
+        if not re.match(r'\^.*', limit):
+            limit = '.*' + limit
+        if not re.match(r'.*\$', limit):
+            limit = limit + '.*'
+
+    for osd in full_osd_list:
+        if limit:
+            try:
+                if re.match(limit, osd):
+                    osd_list.append(getOSDInformation(zk_conn, osd))
+            except Exception as e:
+                return False, 'Regex Error: {}'.format(e)
+        else:
+            osd_list.append(getOSDInformation(zk_conn, osd))
+
+    return True, sorted(osd_list, key=lambda x: int(x['id']))
+
+
+#
+# Pool functions
+#
+def getPoolInformation(zk_conn, pool):
+    # Parse the stats data
+    pool_stats_raw = zkhandler.readdata(zk_conn, '/ceph/pools/{}/stats'.format(pool))
+    pool_stats = dict(json.loads(pool_stats_raw))
+
+    pool_information = {
+        'name': pool,
+        'stats': pool_stats
+    }
+    return pool_information
+
+
+def add_pool(zk_conn, name, pgs, replcfg):
+    # Prepare the copies/mincopies variables
+    try:
+        copies, mincopies = replcfg.split(',')
+        copies = int(copies.replace('copies=', ''))
+        mincopies = int(mincopies.replace('mincopies=', ''))
+    except Exception:
+        copies = None
+        mincopies = None
+    if not copies or not mincopies:
+        return False, 'ERROR: Replication configuration "{}" is not valid.'.format(replcfg)
+
+    # 1. Create the pool
+    retcode, stdout, stderr = common.run_os_command('ceph osd pool create {} {} replicated'.format(name, pgs))
+    if retcode:
+        return False, 'ERROR: Failed to create pool "{}" with {} PGs: {}'.format(name, pgs, stderr)
+
+    # 2. Set the size and minsize
+    retcode, stdout, stderr = common.run_os_command('ceph osd pool set {} size {}'.format(name, copies))
+    if retcode:
+        return False, 'ERROR: Failed to set pool "{}" size of {}: {}'.format(name, copies, stderr)
+
+    retcode, stdout, stderr = common.run_os_command('ceph osd pool set {} min_size {}'.format(name, mincopies))
+    if retcode:
+        return False, 'ERROR: Failed to set pool "{}" minimum size of {}: {}'.format(name, mincopies, stderr)
+
+    # 3. Enable RBD application
+    retcode, stdout, stderr = common.run_os_command('ceph osd pool application enable {} rbd'.format(name))
+    if retcode:
+        return False, 'ERROR: Failed to enable RBD application on pool "{}" : {}'.format(name, stderr)
+
+    # 4. Add the new pool to Zookeeper
+    zkhandler.writedata(zk_conn, {
+        '/ceph/pools/{}'.format(name): '',
+        '/ceph/pools/{}/pgs'.format(name): pgs,
+        '/ceph/pools/{}/stats'.format(name): '{}',
+        '/ceph/volumes/{}'.format(name): '',
+        '/ceph/snapshots/{}'.format(name): '',
+    })
+
+    return True, 'Created RBD pool "{}" with {} PGs'.format(name, pgs)
+
+
+def remove_pool(zk_conn, name):
+    if not verifyPool(zk_conn, name):
+        return False, 'ERROR: No pool with name "{}" is present in the cluster.'.format(name)
+
+    # 1. Remove pool volumes
+    for volume in zkhandler.listchildren(zk_conn, '/ceph/volumes/{}'.format(name)):
+        remove_volume(zk_conn, name, volume)
+
+    # 2. Remove the pool
+    retcode, stdout, stderr = common.run_os_command('ceph osd pool rm {pool} {pool} --yes-i-really-really-mean-it'.format(pool=name))
+    if retcode:
+        return False, 'ERROR: Failed to remove pool "{}": {}'.format(name, stderr)
+
+    # 3. Delete pool from Zookeeper
+    zkhandler.deletekey(zk_conn, '/ceph/pools/{}'.format(name))
+    zkhandler.deletekey(zk_conn, '/ceph/volumes/{}'.format(name))
+    zkhandler.deletekey(zk_conn, '/ceph/snapshots/{}'.format(name))
+
+    return True, 'Removed RBD pool "{}" and all volumes.'.format(name)
+
+
+def get_list_pool(zk_conn, limit, is_fuzzy=True):
+    pool_list = []
+    full_pool_list = zkhandler.listchildren(zk_conn, '/ceph/pools')
+
+    if limit:
+        if not is_fuzzy:
+            limit = '^' + limit + '$'
+
+    for pool in full_pool_list:
+        if limit:
+            try:
+                if re.match(limit, pool):
+                    pool_list.append(getPoolInformation(zk_conn, pool))
+            except Exception as e:
+                return False, 'Regex Error: {}'.format(e)
+        else:
+            pool_list.append(getPoolInformation(zk_conn, pool))
+
+    return True, sorted(pool_list, key=lambda x: int(x['stats']['id']))
+
+
+#
+# Volume functions
+#
+def getCephVolumes(zk_conn, pool):
+    volume_list = list()
+    if not pool:
+        pool_list = zkhandler.listchildren(zk_conn, '/ceph/pools')
+    else:
+        pool_list = [pool]
+
+    for pool_name in pool_list:
+        for volume_name in zkhandler.listchildren(zk_conn, '/ceph/volumes/{}'.format(pool_name)):
+            volume_list.append('{}/{}'.format(pool_name, volume_name))
+
+    return volume_list
+
+
+def getVolumeInformation(zk_conn, pool, volume):
+    # Parse the stats data
+    volume_stats_raw = zkhandler.readdata(zk_conn, '/ceph/volumes/{}/{}/stats'.format(pool, volume))
+    volume_stats = dict(json.loads(volume_stats_raw))
+    # Format the size to something nicer
+    volume_stats['size'] = format_bytes_tohuman(volume_stats['size'])
+
+    volume_information = {
+        'name': volume,
+        'pool': pool,
+        'stats': volume_stats
+    }
+    return volume_information
+
+
+def add_volume(zk_conn, pool, name, size):
+    # 1. Create the volume
+    retcode, stdout, stderr = common.run_os_command('rbd create --size {} --image-feature layering,exclusive-lock {}/{}'.format(size, pool, name))
+    if retcode:
+        return False, 'ERROR: Failed to create RBD volume "{}": {}'.format(name, stderr)
+
+    # 2. Get volume stats
+    retcode, stdout, stderr = common.run_os_command('rbd info --format json {}/{}'.format(pool, name))
+    volstats = stdout
+
+    # 3. Add the new volume to Zookeeper
+    zkhandler.writedata(zk_conn, {
+        '/ceph/volumes/{}/{}'.format(pool, name): '',
+        '/ceph/volumes/{}/{}/stats'.format(pool, name): volstats,
+        '/ceph/snapshots/{}/{}'.format(pool, name): '',
+    })
+
+    return True, 'Created RBD volume "{}/{}" ({}).'.format(pool, name, size)
+
+
+def clone_volume(zk_conn, pool, name_src, name_new):
+    if not verifyVolume(zk_conn, pool, name_src):
+        return False, 'ERROR: No volume with name "{}" is present in pool "{}".'.format(name_src, pool)
+
+    # 1. Clone the volume
+    retcode, stdout, stderr = common.run_os_command('rbd copy {}/{} {}/{}'.format(pool, name_src, pool, name_new))
+    if retcode:
+        return False, 'ERROR: Failed to clone RBD volume "{}" to "{}" in pool "{}": {}'.format(name_src, name_new, pool, stderr)
+
+    # 2. Get volume stats
+    retcode, stdout, stderr = common.run_os_command('rbd info --format json {}/{}'.format(pool, name_new))
+    volstats = stdout
+
+    # 3. Add the new volume to Zookeeper
+    zkhandler.writedata(zk_conn, {
+        '/ceph/volumes/{}/{}'.format(pool, name_new): '',
+        '/ceph/volumes/{}/{}/stats'.format(pool, name_new): volstats,
+        '/ceph/snapshots/{}/{}'.format(pool, name_new): '',
+    })
+
+    return True, 'Cloned RBD volume "{}" to "{}" in pool "{}"'.format(name_src, name_new, pool)
+
+
+def resize_volume(zk_conn, pool, name, size):
+    if not verifyVolume(zk_conn, pool, name):
+        return False, 'ERROR: No volume with name "{}" is present in pool "{}".'.format(name, pool)
+
+    # 1. Resize the volume
+    retcode, stdout, stderr = common.run_os_command('rbd resize --size {} {}/{}'.format(size, pool, name))
+    if retcode:
+        return False, 'ERROR: Failed to resize RBD volume "{}" to size "{}" in pool "{}": {}'.format(name, size, pool, stderr)
+
+    # 2a. Determine the node running this VM if applicable
+    active_node = None
+    volume_vm_name = name.split('_')[0]
+    retcode, vm_info = vm.get_info(zk_conn, volume_vm_name)
+    if retcode:
+        for disk in vm_info['disks']:
+            # This block device is present in this VM so we can continue
+            if disk['name'] == '{}/{}'.format(pool, name):
+                active_node = vm_info['node']
+                volume_id = disk['dev']
+    # 2b. Perform a live resize in libvirt if the VM is running
+    if active_node is not None and vm_info.get('state', '') == 'start':
+        import libvirt
+        # Run the libvirt command against the target host
+        try:
+            dest_lv = 'qemu+tcp://{}/system'.format(active_node)
+            target_lv_conn = libvirt.open(dest_lv)
+            target_vm_conn = target_lv_conn.lookupByName(vm_info['name'])
+            if target_vm_conn:
+                target_vm_conn.blockResize(volume_id, int(format_bytes_fromhuman(size)[:-1]), libvirt.VIR_DOMAIN_BLOCK_RESIZE_BYTES)
+            target_lv_conn.close()
+        except Exception:
+            pass
+
+    # 2. Get volume stats
+    retcode, stdout, stderr = common.run_os_command('rbd info --format json {}/{}'.format(pool, name))
+    volstats = stdout
+
+    # 3. Add the new volume to Zookeeper
+    zkhandler.writedata(zk_conn, {
+        '/ceph/volumes/{}/{}'.format(pool, name): '',
+        '/ceph/volumes/{}/{}/stats'.format(pool, name): volstats,
+        '/ceph/snapshots/{}/{}'.format(pool, name): '',
+    })
+
+    return True, 'Resized RBD volume "{}" to size "{}" in pool "{}".'.format(name, size, pool)
+
+
+def rename_volume(zk_conn, pool, name, new_name):
+    if not verifyVolume(zk_conn, pool, name):
+        return False, 'ERROR: No volume with name "{}" is present in pool "{}".'.format(name, pool)
+
+    # 1. Rename the volume
+    retcode, stdout, stderr = common.run_os_command('rbd rename {}/{} {}'.format(pool, name, new_name))
+    if retcode:
+        return False, 'ERROR: Failed to rename volume "{}" to "{}" in pool "{}": {}'.format(name, new_name, pool, stderr)
+
+    # 2. Rename the volume in Zookeeper
+    zkhandler.renamekey(zk_conn, {
+        '/ceph/volumes/{}/{}'.format(pool, name): '/ceph/volumes/{}/{}'.format(pool, new_name),
+        '/ceph/snapshots/{}/{}'.format(pool, name): '/ceph/snapshots/{}/{}'.format(pool, new_name)
+    })
+
+    # 3. Get volume stats
+    retcode, stdout, stderr = common.run_os_command('rbd info --format json {}/{}'.format(pool, new_name))
+    volstats = stdout
+
+    # 4. Update the volume stats in Zookeeper
+    zkhandler.writedata(zk_conn, {
+        '/ceph/volumes/{}/{}/stats'.format(pool, new_name): volstats,
+    })
+
+    return True, 'Renamed RBD volume "{}" to "{}" in pool "{}".'.format(name, new_name, pool)
+
+
+def remove_volume(zk_conn, pool, name):
+    if not verifyVolume(zk_conn, pool, name):
+        return False, 'ERROR: No volume with name "{}" is present in pool "{}".'.format(name, pool)
+
+    # 1. Remove volume snapshots
+    for snapshot in zkhandler.listchildren(zk_conn, '/ceph/snapshots/{}/{}'.format(pool, name)):
+        remove_snapshot(zk_conn, pool, name, snapshot)
+
+    # 2. Remove the volume
+    retcode, stdout, stderr = common.run_os_command('rbd rm {}/{}'.format(pool, name))
+    if retcode:
+        return False, 'ERROR: Failed to remove RBD volume "{}" in pool "{}": {}'.format(name, pool, stderr)
+
+    # 3. Delete volume from Zookeeper
+    zkhandler.deletekey(zk_conn, '/ceph/volumes/{}/{}'.format(pool, name))
+    zkhandler.deletekey(zk_conn, '/ceph/snapshots/{}/{}'.format(pool, name))
+
+    return True, 'Removed RBD volume "{}" in pool "{}".'.format(name, pool)
+
+
+def map_volume(zk_conn, pool, name):
+    if not verifyVolume(zk_conn, pool, name):
+        return False, 'ERROR: No volume with name "{}" is present in pool "{}".'.format(name, pool)
+
+    # 1. Map the volume onto the local system
+    retcode, stdout, stderr = common.run_os_command('rbd map {}/{}'.format(pool, name))
+    if retcode:
+        return False, 'ERROR: Failed to map RBD volume "{}" in pool "{}": {}'.format(name, pool, stderr)
+
+    # 2. Calculate the absolute path to the mapped volume
+    mapped_volume = '/dev/rbd/{}/{}'.format(pool, name)
+
+    # 3. Ensure the volume exists
+    if not os.path.exists(mapped_volume):
+        return False, 'ERROR: Mapped volume not found at expected location "{}".'.format(mapped_volume)
+
+    return True, mapped_volume
+
+
+def unmap_volume(zk_conn, pool, name):
+    if not verifyVolume(zk_conn, pool, name):
+        return False, 'ERROR: No volume with name "{}" is present in pool "{}".'.format(name, pool)
+
+    mapped_volume = '/dev/rbd/{}/{}'.format(pool, name)
+
+    # 1. Ensure the volume exists
+    if not os.path.exists(mapped_volume):
+        return False, 'ERROR: Mapped volume not found at expected location "{}".'.format(mapped_volume)
+
+    # 2. Unap the volume
+    retcode, stdout, stderr = common.run_os_command('rbd unmap {}'.format(mapped_volume))
+    if retcode:
+        return False, 'ERROR: Failed to unmap RBD volume at "{}": {}'.format(mapped_volume, stderr)
+
+    return True, 'Unmapped RBD volume at "{}".'.format(mapped_volume)
+
+
+def get_list_volume(zk_conn, pool, limit, is_fuzzy=True):
+    volume_list = []
+    if pool and not verifyPool(zk_conn, pool):
+        return False, 'ERROR: No pool with name "{}" is present in the cluster.'.format(pool)
+
+    full_volume_list = getCephVolumes(zk_conn, pool)
+
+    if limit:
+        if not is_fuzzy:
+            limit = '^' + limit + '$'
+        else:
+            # Implicitly assume fuzzy limits
+            if not re.match(r'\^.*', limit):
+                limit = '.*' + limit
+            if not re.match(r'.*\$', limit):
+                limit = limit + '.*'
+
+    for volume in full_volume_list:
+        pool_name, volume_name = volume.split('/')
+        if limit:
+            try:
+                if re.match(limit, volume_name):
+                    volume_list.append(getVolumeInformation(zk_conn, pool_name, volume_name))
+            except Exception as e:
+                return False, 'Regex Error: {}'.format(e)
+        else:
+            volume_list.append(getVolumeInformation(zk_conn, pool_name, volume_name))
+
+    return True, sorted(volume_list, key=lambda x: str(x['name']))
+
+
+#
+# Snapshot functions
+#
+def getCephSnapshots(zk_conn, pool, volume):
+    snapshot_list = list()
+    volume_list = list()
+
+    volume_list = getCephVolumes(zk_conn, pool)
+    if volume:
+        for volume_entry in volume_list:
+            volume_pool, volume_name = volume_entry.split('/')
+            if volume_name == volume:
+                volume_list = ['{}/{}'.format(volume_pool, volume_name)]
+
+    for volume_entry in volume_list:
+        for snapshot_name in zkhandler.listchildren(zk_conn, '/ceph/snapshots/{}'.format(volume_entry)):
+            snapshot_list.append('{}@{}'.format(volume_entry, snapshot_name))
+
+    return snapshot_list
+
+
+def add_snapshot(zk_conn, pool, volume, name):
+    if not verifyVolume(zk_conn, pool, volume):
+        return False, 'ERROR: No volume with name "{}" is present in pool "{}".'.format(volume, pool)
+
+    # 1. Create the snapshot
+    retcode, stdout, stderr = common.run_os_command('rbd snap create {}/{}@{}'.format(pool, volume, name))
+    if retcode:
+        return False, 'ERROR: Failed to create RBD snapshot "{}" of volume "{}" in pool "{}": {}'.format(name, volume, pool, stderr)
+
+    # 2. Add the snapshot to Zookeeper
+    zkhandler.writedata(zk_conn, {
+        '/ceph/snapshots/{}/{}/{}'.format(pool, volume, name): '',
+        '/ceph/snapshots/{}/{}/{}/stats'.format(pool, volume, name): '{}'
+    })
+
+    return True, 'Created RBD snapshot "{}" of volume "{}" in pool "{}".'.format(name, volume, pool)
+
+
+def rename_snapshot(zk_conn, pool, volume, name, new_name):
+    if not verifyVolume(zk_conn, pool, volume):
+        return False, 'ERROR: No volume with name "{}" is present in pool "{}".'.format(volume, pool)
+    if not verifySnapshot(zk_conn, pool, volume, name):
+        return False, 'ERROR: No snapshot with name "{}" is present for volume "{}" in pool "{}".'.format(name, volume, pool)
+
+    # 1. Rename the snapshot
+    retcode, stdout, stderr = common.run_os_command('rbd snap rename {}/{}@{} {}'.format(pool, volume, name, new_name))
+    if retcode:
+        return False, 'ERROR: Failed to rename RBD snapshot "{}" to "{}" for volume "{}" in pool "{}": {}'.format(name, new_name, volume, pool, stderr)
+
+    # 2. Rename the snapshot in ZK
+    zkhandler.renamekey(zk_conn, {
+        '/ceph/snapshots/{}/{}/{}'.format(pool, volume, name): '/ceph/snapshots/{}/{}/{}'.format(pool, volume, new_name)
+    })
+
+    return True, 'Renamed RBD snapshot "{}" to "{}" for volume "{}" in pool "{}".'.format(name, new_name, volume, pool)
+
+
+def remove_snapshot(zk_conn, pool, volume, name):
+    if not verifyVolume(zk_conn, pool, volume):
+        return False, 'ERROR: No volume with name "{}" is present in pool "{}".'.format(volume, pool)
+    if not verifySnapshot(zk_conn, pool, volume, name):
+        return False, 'ERROR: No snapshot with name "{}" is present of volume {} in pool {}.'.format(name, volume, pool)
+
+    # 1. Remove the snapshot
+    retcode, stdout, stderr = common.run_os_command('rbd snap rm {}/{}@{}'.format(pool, volume, name))
+    if retcode:
+        return False, 'Failed to remove RBD snapshot "{}" of volume "{}" in pool "{}": {}'.format(name, volume, pool, stderr)
+
+    # 2. Delete snapshot from Zookeeper
+    zkhandler.deletekey(zk_conn, '/ceph/snapshots/{}/{}/{}'.format(pool, volume, name))
+
+    return True, 'Removed RBD snapshot "{}" of volume "{}" in pool "{}".'.format(name, volume, pool)
+
+
+def get_list_snapshot(zk_conn, pool, volume, limit, is_fuzzy=True):
+    snapshot_list = []
+    if pool and not verifyPool(zk_conn, pool):
+        return False, 'ERROR: No pool with name "{}" is present in the cluster.'.format(pool)
+
+    if volume and not verifyPool(zk_conn, volume):
+        return False, 'ERROR: No volume with name "{}" is present in the cluster.'.format(volume)
+
+    full_snapshot_list = getCephSnapshots(zk_conn, pool, volume)
+
+    if is_fuzzy and limit:
+        # Implicitly assume fuzzy limits
+        if not re.match(r'\^.*', limit):
+            limit = '.*' + limit
+        if not re.match(r'.*\$', limit):
+            limit = limit + '.*'
+
+    for snapshot in full_snapshot_list:
+        volume, snapshot_name = snapshot.split('@')
+        pool_name, volume_name = volume.split('/')
+        if limit:
+            try:
+                if re.match(limit, snapshot_name):
+                    snapshot_list.append({'pool': pool_name, 'volume': volume_name, 'snapshot': snapshot_name})
+            except Exception as e:
+                return False, 'Regex Error: {}'.format(e)
+        else:
+            snapshot_list.append({'pool': pool_name, 'volume': volume_name, 'snapshot': snapshot_name})
+
+    return True, sorted(snapshot_list, key=lambda x: int(x['id']))

daemon-common/cluster.py

@@ -20,17 +20,15 @@
 #
 ###############################################################################
 
-import json
+import re
 
-from distutils.util import strtobool
+import daemon_lib.zkhandler as zkhandler
+import daemon_lib.common as common
+import daemon_lib.vm as pvc_vm
+import daemon_lib.node as pvc_node
+import daemon_lib.network as pvc_network
+import daemon_lib.ceph as pvc_ceph
 
-import client_lib.ansiprint as ansiprint
-import client_lib.zkhandler as zkhandler
-import client_lib.common as common
-import client_lib.vm as pvc_vm
-import client_lib.node as pvc_node
-import client_lib.network as pvc_network
-import client_lib.ceph as pvc_ceph
 
 def set_maintenance(zk_conn, maint_state):
     try:
@@ -40,16 +38,21 @@ def set_maintenance(zk_conn, maint_state):
         else:
             zkhandler.writedata(zk_conn, {'/maintenance': 'false'})
             return True, 'Successfully set cluster in normal mode'
-    except:
+    except Exception:
         return False, 'Failed to set cluster maintenance state'
 
+
 def getClusterInformation(zk_conn):
     # Get cluster maintenance state
     try:
         maint_state = zkhandler.readdata(zk_conn, '/maintenance')
-    except:
+    except Exception:
         maint_state = 'false'
 
+    # List of messages to display to the clients
+    cluster_health_msg = []
+    storage_health_msg = []
+
     # Get node information object list
     retcode, node_list = pvc_node.get_list(zk_conn, None)
 
@@ -74,6 +77,36 @@ def getClusterInformation(zk_conn):
     ceph_volume_count = len(ceph_volume_list)
     ceph_snapshot_count = len(ceph_snapshot_list)
 
+    # Determinations for general cluster health
+    cluster_healthy_status = True
+    # Check for (n-1) overprovisioning
+    #   Assume X nodes. If the total VM memory allocation (counting only running VMss) is greater than
+    #   the total memory of the (n-1) smallest nodes, trigger this warning.
+    n_minus_1_total = 0
+    alloc_total = 0
+
+    node_largest_index = None
+    node_largest_count = 0
+    for index, node in enumerate(node_list):
+        node_mem_total = node['memory']['total']
+        node_mem_alloc = node['memory']['allocated']
+        alloc_total += node_mem_alloc
+
+        # Determine if this node is the largest seen so far
+        if node_mem_total > node_largest_count:
+            node_largest_index = index
+            node_largest_count = node_mem_total
+    n_minus_1_node_list = list()
+    for index, node in enumerate(node_list):
+        if index == node_largest_index:
+            continue
+        n_minus_1_node_list.append(node)
+    for index, node in enumerate(n_minus_1_node_list):
+        n_minus_1_total += node['memory']['total']
+    if alloc_total > n_minus_1_total:
+        cluster_healthy_status = False
+        cluster_health_msg.append("Total VM memory ({}) is overprovisioned (max {}) for (n-1) failure scenarios".format(alloc_total, n_minus_1_total))
+
     # Determinations for node health
     node_healthy_status = list(range(0, node_count))
     node_report_status = list(range(0, node_count))
@@ -82,9 +115,10 @@ def getClusterInformation(zk_conn):
         domain_state = node['domain_state']
         if daemon_state != 'run' and domain_state != 'ready':
             node_healthy_status[index] = False
+            cluster_health_msg.append("Node '{}' in {},{} state".format(node['name'], daemon_state, domain_state))
         else:
             node_healthy_status[index] = True
-        node_report_status[index] = daemon_state + ',' +  domain_state
+        node_report_status[index] = daemon_state + ',' + domain_state
 
     # Determinations for VM health
     vm_healthy_status = list(range(0, vm_count))
@@ -93,6 +127,7 @@ def getClusterInformation(zk_conn):
         vm_state = vm['state']
         if vm_state not in ['start', 'disable', 'migrate', 'unmigrate', 'provision']:
             vm_healthy_status[index] = False
+            cluster_health_msg.append("VM '{}' in {} state".format(vm['name'], vm_state))
         else:
             vm_healthy_status[index] = True
         vm_report_status[index] = vm_state
@@ -111,27 +146,51 @@ def getClusterInformation(zk_conn):
         except KeyError:
             ceph_osd_in = 0
 
+        up_texts = {1: 'up', 0: 'down'}
+        in_texts = {1: 'in', 0: 'out'}
+
         if not ceph_osd_up or not ceph_osd_in:
             ceph_osd_healthy_status[index] = False
+            cluster_health_msg.append('OSD {} in {},{} state'.format(ceph_osd['id'], up_texts[ceph_osd_up], in_texts[ceph_osd_in]))
         else:
             ceph_osd_healthy_status[index] = True
-        up_texts = { 1: 'up', 0: 'down' }
-        in_texts = { 1: 'in', 0: 'out' }
         ceph_osd_report_status[index] = up_texts[ceph_osd_up] + ',' + in_texts[ceph_osd_in]
 
     # Find out the overall cluster health; if any element of a healthy_status is false, it's unhealthy
     if maint_state == 'true':
         cluster_health = 'Maintenance'
-    elif False in node_healthy_status or False in vm_healthy_status or False in ceph_osd_healthy_status:
+    elif cluster_healthy_status is False or False in node_healthy_status or False in vm_healthy_status or False in ceph_osd_healthy_status:
         cluster_health = 'Degraded'
     else:
         cluster_health = 'Optimal'
 
+    # Find out our storage health from Ceph
+    ceph_status = zkhandler.readdata(zk_conn, '/ceph').split('\n')
+    ceph_health = ceph_status[2].split()[-1]
+
+    # Parse the status output to get the health indicators
+    line_record = False
+    for index, line in enumerate(ceph_status):
+        if re.search('services:', line):
+            line_record = False
+        if line_record and len(line.strip()) > 0:
+            storage_health_msg.append(line.strip())
+        if re.search('health:', line):
+            line_record = True
+
+    if maint_state == 'true':
+        storage_health = 'Maintenance'
+    elif ceph_health != 'HEALTH_OK':
+        storage_health = 'Degraded'
+    else:
+        storage_health = 'Optimal'
+
     # State lists
     node_state_combinations = [
         'run,ready', 'run,flush', 'run,flushed', 'run,unflush',
         'init,ready', 'init,flush', 'init,flushed', 'init,unflush',
-        'stop,ready', 'stop,flush', 'stop,flushed', 'stop,unflush'
+        'stop,ready', 'stop,flush', 'stop,flushed', 'stop,unflush',
+        'dead,ready', 'dead,flush', 'dead,flushed', 'dead,unflush'
     ]
     vm_state_combinations = [
         'start', 'restart', 'shutdown', 'stop', 'disable', 'fail', 'migrate', 'unmigrate', 'provision'
@@ -173,6 +232,9 @@ def getClusterInformation(zk_conn):
     # Format the status data
     cluster_information = {
         'health': cluster_health,
+        'health_msg': cluster_health_msg,
+        'storage_health': storage_health,
+        'storage_health_msg': storage_health_msg,
         'primary_node': common.getPrimaryNode(zk_conn),
         'upstream_ip': zkhandler.readdata(zk_conn, '/upstream_ip'),
         'nodes': formatted_node_states,
@@ -186,6 +248,7 @@ def getClusterInformation(zk_conn):
 
     return cluster_information
 
+
 def get_info(zk_conn):
     # This is a thin wrapper function for naming purposes
     cluster_information = getClusterInformation(zk_conn)

daemon-common/common.py

@@ -20,19 +20,52 @@
 #
 ###############################################################################
 
+import time
 import uuid
 import lxml
-import math
+import shlex
+import subprocess
 import kazoo.client
+from json import loads
 
 from distutils.util import strtobool
 
-import client_lib.zkhandler as zkhandler
+import daemon_lib.zkhandler as zkhandler
 
 ###############################################################################
 # Supplemental functions
 ###############################################################################
 
+
+#
+# Run a local OS command via shell
+#
+def run_os_command(command_string, background=False, environment=None, timeout=None, shell=False):
+    command = shlex.split(command_string)
+    try:
+        command_output = subprocess.run(
+            command,
+            shell=shell,
+            env=environment,
+            timeout=timeout,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        retcode = command_output.returncode
+    except subprocess.TimeoutExpired:
+        retcode = 128
+
+    try:
+        stdout = command_output.stdout.decode('ascii')
+    except Exception:
+        stdout = ''
+    try:
+        stderr = command_output.stderr.decode('ascii')
+    except Exception:
+        stderr = ''
+    return retcode, stdout, stderr
+
+
 #
 # Validate a UUID
 #
@@ -40,9 +73,10 @@ def validateUUID(dom_uuid):
     try:
         uuid.UUID(dom_uuid)
         return True
-    except:
+    except Exception:
         return False
 
+
 #
 # Connect and disconnect from Zookeeper
 #
@@ -58,23 +92,27 @@ def startZKConnection(zk_host):
         exit(1)
     return zk_conn
 
+
 def stopZKConnection(zk_conn):
     zk_conn.stop()
     zk_conn.close()
     return 0
+
+
 #
 # Parse a Domain XML object
 #
 def getDomainXML(zk_conn, dom_uuid):
     try:
         xml = zkhandler.readdata(zk_conn, '/domains/{}/xml'.format(dom_uuid))
-    except:
+    except Exception:
         return None
-    
+
     # Parse XML using lxml.objectify
     parsed_xml = lxml.objectify.fromstring(xml)
     return parsed_xml
 
+
 #
 # Get the main details for a VM object from XML
 #
@@ -87,19 +125,20 @@ def getDomainMainDetails(parsed_xml):
         ddescription = "N/A"
     dname = str(parsed_xml.name)
     dmemory = str(parsed_xml.memory)
-    dmemory_unit = str(parsed_xml.memory.attrib['unit'])
+    dmemory_unit = str(parsed_xml.memory.attrib.get('unit'))
     if dmemory_unit == 'KiB':
         dmemory = int(int(dmemory) / 1024)
     elif dmemory_unit == 'GiB':
         dmemory = int(int(dmemory) * 1024)
     dvcpu = str(parsed_xml.vcpu)
     try:
-        dvcputopo = '{}/{}/{}'.format(parsed_xml.cpu.topology.attrib['sockets'], parsed_xml.cpu.topology.attrib['cores'], parsed_xml.cpu.topology.attrib['threads'])
-    except:
+        dvcputopo = '{}/{}/{}'.format(parsed_xml.cpu.topology.attrib.get('sockets'), parsed_xml.cpu.topology.attrib.get('cores'), parsed_xml.cpu.topology.attrib.get('threads'))
+    except Exception:
         dvcputopo = 'N/A'
 
     return duuid, dname, ddescription, dmemory, dvcpu, dvcputopo
 
+
 #
 # Get long-format details
 #
@@ -112,36 +151,66 @@ def getDomainExtraDetails(parsed_xml):
 
     return dtype, darch, dmachine, dconsole, demulator
 
+
 #
 # Get CPU features
 #
 def getDomainCPUFeatures(parsed_xml):
     dfeatures = []
-    for feature in parsed_xml.features.getchildren():
-        dfeatures.append(feature.tag)
+    try:
+        for feature in parsed_xml.features.getchildren():
+            dfeatures.append(feature.tag)
+    except Exception:
+        pass
 
     return dfeatures
 
+
 #
 # Get disk devices
 #
-def getDomainDisks(parsed_xml):
+def getDomainDisks(parsed_xml, stats_data):
     ddisks = []
     for device in parsed_xml.devices.getchildren():
         if device.tag == 'disk':
             disk_attrib = device.source.attrib
             disk_target = device.target.attrib
-            disk_type = device.attrib['type']
+            disk_type = device.attrib.get('type')
+            disk_stats_list = [x for x in stats_data.get('disk_stats', []) if x.get('name') == disk_attrib.get('name')]
+            try:
+                disk_stats = disk_stats_list[0]
+            except Exception:
+                disk_stats = {}
+
             if disk_type == 'network':
-                disk_obj = { 'type': disk_attrib.get('protocol'), 'name': disk_attrib.get('name'), 'dev': disk_target.get('dev'), 'bus': disk_target.get('bus') }
+                disk_obj = {
+                    'type': disk_attrib.get('protocol'),
+                    'name': disk_attrib.get('name'),
+                    'dev': disk_target.get('dev'),
+                    'bus': disk_target.get('bus'),
+                    'rd_req': disk_stats.get('rd_req', 0),
+                    'rd_bytes': disk_stats.get('rd_bytes', 0),
+                    'wr_req': disk_stats.get('wr_req', 0),
+                    'wr_bytes': disk_stats.get('wr_bytes', 0)
+                }
             elif disk_type == 'file':
-                disk_obj = { 'type': 'file', 'name': disk_attrib.get('file'), 'dev': disk_target.get('dev'), 'bus': disk_target.get('bus') }
+                disk_obj = {
+                    'type': 'file',
+                    'name': disk_attrib.get('file'),
+                    'dev': disk_target.get('dev'),
+                    'bus': disk_target.get('bus'),
+                    'rd_req': disk_stats.get('rd_req', 0),
+                    'rd_bytes': disk_stats.get('rd_bytes', 0),
+                    'wr_req': disk_stats.get('wr_req', 0),
+                    'wr_bytes': disk_stats.get('wr_bytes', 0)
+                }
             else:
                 disk_obj = {}
             ddisks.append(disk_obj)
 
     return ddisks
 
+
 #
 # Get a list of disk devices
 #
@@ -150,9 +219,10 @@ def getDomainDiskList(zk_conn, dom_uuid):
     disk_list = []
     for disk in domain_information['disks']:
         disk_list.append(disk['name'])
-       
+
     return disk_list
 
+
 #
 # Get domain information from XML
 #
@@ -168,16 +238,20 @@ def getInformationFromXML(zk_conn, uuid):
 
     try:
         domain_node_limit = zkhandler.readdata(zk_conn, '/domains/{}/node_limit'.format(uuid))
-    except:
+    except Exception:
         domain_node_limit = None
     try:
         domain_node_selector = zkhandler.readdata(zk_conn, '/domains/{}/node_selector'.format(uuid))
-    except:
+    except Exception:
         domain_node_selector = None
     try:
         domain_node_autostart = zkhandler.readdata(zk_conn, '/domains/{}/node_autostart'.format(uuid))
-    except:
+    except Exception:
         domain_node_autostart = None
+    try:
+        domain_migration_method = zkhandler.readdata(zk_conn, '/domains/{}/migration_method'.format(uuid))
+    except Exception:
+        domain_migration_method = None
 
     if not domain_node_limit:
         domain_node_limit = None
@@ -189,20 +263,25 @@ def getInformationFromXML(zk_conn, uuid):
 
     try:
         domain_profile = zkhandler.readdata(zk_conn, '/domains/{}/profile'.format(uuid))
-    except:
+    except Exception:
         domain_profile = None
 
     parsed_xml = getDomainXML(zk_conn, uuid)
 
+    try:
+        stats_data = loads(zkhandler.readdata(zk_conn, '/domains/{}/stats'.format(uuid)))
+    except Exception:
+        stats_data = {}
+
     domain_uuid, domain_name, domain_description, domain_memory, domain_vcpu, domain_vcputopo = getDomainMainDetails(parsed_xml)
-    domain_networks = getDomainNetworks(parsed_xml)
+    domain_networks = getDomainNetworks(parsed_xml, stats_data)
 
     domain_type, domain_arch, domain_machine, domain_console, domain_emulator = getDomainExtraDetails(parsed_xml)
 
     domain_features = getDomainCPUFeatures(parsed_xml)
-    domain_disks = getDomainDisks(parsed_xml)
+    domain_disks = getDomainDisks(parsed_xml, stats_data)
     domain_controllers = getDomainControllers(parsed_xml)
-    
+
     if domain_lastnode:
         domain_migrated = 'from {}'.format(domain_lastnode)
     else:
@@ -219,11 +298,14 @@ def getInformationFromXML(zk_conn, uuid):
         'node_limit': domain_node_limit,
         'node_selector': domain_node_selector,
         'node_autostart': bool(strtobool(domain_node_autostart)),
+        'migration_method': domain_migration_method,
         'description': domain_description,
         'profile': domain_profile,
         'memory': int(domain_memory),
+        'memory_stats': stats_data.get('mem_stats', {}),
         'vcpu': int(domain_vcpu),
         'vcpu_topology': domain_vcputopo,
+        'vcpu_stats': stats_data.get('cpu_stats', {}),
         'networks': domain_networks,
         'type': domain_type,
         'arch': domain_arch,
@@ -238,22 +320,62 @@ def getInformationFromXML(zk_conn, uuid):
 
     return domain_information
 
+
 #
 # Get network devices
 #
-def getDomainNetworks(parsed_xml):
+def getDomainNetworks(parsed_xml, stats_data):
     dnets = []
     for device in parsed_xml.devices.getchildren():
         if device.tag == 'interface':
-            net_type = device.attrib['type']
-            net_mac = device.mac.attrib['address']
-            net_bridge = device.source.attrib[net_type]
-            net_model = device.model.attrib['type']
-            net_obj = { 'type': net_type, 'mac': net_mac, 'source': net_bridge, 'model': net_model }
+            try:
+                net_type = device.attrib.get('type')
+            except Exception:
+                net_type = None
+            try:
+                net_mac = device.mac.attrib.get('address')
+            except Exception:
+                net_mac = None
+            try:
+                net_bridge = device.source.attrib.get(net_type)
+            except Exception:
+                net_bridge = None
+            try:
+                net_model = device.model.attrib.get('type')
+            except Exception:
+                net_model = None
+            try:
+                net_stats_list = [x for x in stats_data.get('net_stats', []) if x.get('bridge') == net_bridge]
+                net_stats = net_stats_list[0]
+            except Exception:
+                net_stats = {}
+            net_rd_bytes = net_stats.get('rd_bytes', 0)
+            net_rd_packets = net_stats.get('rd_packets', 0)
+            net_rd_errors = net_stats.get('rd_errors', 0)
+            net_rd_drops = net_stats.get('rd_drops', 0)
+            net_wr_bytes = net_stats.get('wr_bytes', 0)
+            net_wr_packets = net_stats.get('wr_packets', 0)
+            net_wr_errors = net_stats.get('wr_errors', 0)
+            net_wr_drops = net_stats.get('wr_drops', 0)
+            net_obj = {
+                'type': net_type,
+                'mac': net_mac,
+                'source': net_bridge,
+                'model': net_model,
+                'rd_bytes': net_rd_bytes,
+                'rd_packets': net_rd_packets,
+                'rd_errors': net_rd_errors,
+                'rd_drops': net_rd_drops,
+                'wr_bytes': net_wr_bytes,
+                'wr_packets': net_wr_packets,
+                'wr_errors': net_wr_errors,
+                'wr_drops': net_wr_drops
+            }
             dnets.append(net_obj)
 
     return dnets
 
+
 #
 # Get controller devices
 #
@@ -261,16 +383,17 @@ def getDomainControllers(parsed_xml):
     dcontrollers = []
     for device in parsed_xml.devices.getchildren():
         if device.tag == 'controller':
-            controller_type = device.attrib['type']
+            controller_type = device.attrib.get('type')
             try:
-                controller_model = device.attrib['model']
+                controller_model = device.attrib.get('model')
             except KeyError:
                 controller_model = 'none'
-            controller_obj = { 'type': controller_type, 'model': controller_model }
+            controller_obj = {'type': controller_type, 'model': controller_model}
             dcontrollers.append(controller_obj)
 
     return dcontrollers
 
+
 #
 # Verify node is valid in cluster
 #
@@ -280,6 +403,7 @@ def verifyNode(zk_conn, node):
     else:
         return False
 
+
 #
 # Get the primary coordinator node
 #
@@ -288,7 +412,7 @@ def getPrimaryNode(zk_conn):
     while True:
         try:
             primary_node = zkhandler.readdata(zk_conn, '/primary_node')
-        except:
+        except Exception:
             primary_node == 'none'
 
         if primary_node == 'none':
@@ -304,6 +428,7 @@ def getPrimaryNode(zk_conn):
 
     return primary_node
 
+
 #
 # Find a migration target
 #
@@ -313,13 +438,13 @@ def findTargetNode(zk_conn, dom_uuid):
         node_limit = zkhandler.readdata(zk_conn, '/domains/{}/node_limit'.format(dom_uuid)).split(',')
         if not any(node_limit):
             node_limit = None
-    except:
+    except Exception:
         node_limit = None
 
     # Determine VM search field or use default; set config value if read fails
     try:
         search_field = zkhandler.readdata(zk_conn, '/domains/{}/node_selector'.format(dom_uuid))
-    except:
+    except Exception:
         search_field = 'mem'
 
     # Execute the search
@@ -335,6 +460,7 @@ def findTargetNode(zk_conn, dom_uuid):
     # Nothing was found
     return None
 
+
 # Get the list of valid target nodes
 def getNodes(zk_conn, node_limit, dom_uuid):
     valid_node_list = []
@@ -361,25 +487,27 @@ def getNodes(zk_conn, node_limit, dom_uuid):
 
     return valid_node_list
 
+
 # via free memory (relative to allocated memory)
 def findTargetNodeMem(zk_conn, node_limit, dom_uuid):
-    most_allocfree = 0
+    most_provfree = 0
     target_node = None
 
     node_list = getNodes(zk_conn, node_limit, dom_uuid)
     for node in node_list:
-        memalloc = int(zkhandler.readdata(zk_conn, '/nodes/{}/memalloc'.format(node)))
+        memprov = int(zkhandler.readdata(zk_conn, '/nodes/{}/memprov'.format(node)))
         memused = int(zkhandler.readdata(zk_conn, '/nodes/{}/memused'.format(node)))
         memfree = int(zkhandler.readdata(zk_conn, '/nodes/{}/memfree'.format(node)))
         memtotal = memused + memfree
-        allocfree = memtotal - memalloc
+        provfree = memtotal - memprov
 
-        if allocfree > most_allocfree:
-            most_allocfree = allocfree
+        if provfree > most_provfree:
+            most_provfree = provfree
             target_node = node
 
     return target_node
 
+
 # via load average
 def findTargetNodeLoad(zk_conn, node_limit, dom_uuid):
     least_load = 9999.0
@@ -395,6 +523,7 @@ def findTargetNodeLoad(zk_conn, node_limit, dom_uuid):
 
     return target_node
 
+
 # via total vCPUs
 def findTargetNodeVCPUs(zk_conn, node_limit, dom_uuid):
     least_vcpus = 9999
@@ -410,6 +539,7 @@ def findTargetNodeVCPUs(zk_conn, node_limit, dom_uuid):
 
     return target_node
 
+
 # via total VMs
 def findTargetNodeVMs(zk_conn, node_limit, dom_uuid):
     least_vms = 9999
@@ -425,6 +555,7 @@ def findTargetNodeVMs(zk_conn, node_limit, dom_uuid):
 
     return target_node
 
+
 # Connect to the primary host and run a command
 def runRemoteCommand(node, command, become=False):
     import paramiko
@@ -452,7 +583,6 @@ def runRemoteCommand(node, command, become=False):
     ssh_client = paramiko.client.SSHClient()
     ssh_client.load_system_host_keys()
     ssh_client.set_missing_host_key_policy(DnssecPolicy())
-    #ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
     ssh_client.connect(node)
     stdin, stdout, stderr = ssh_client.exec_command(command)
     return stdout.read().decode('ascii').rstrip(), stderr.read().decode('ascii').rstrip()

daemon-common/network.py

@@ -20,23 +20,12 @@
 #
 ###############################################################################
 
-import os
-import socket
-import time
-import uuid
 import re
-import tempfile
-import subprocess
-import difflib
-import colorama
-import click
-import lxml.objectify
-import configparser
-import kazoo.client
 
-import client_lib.ansiprint as ansiprint
-import client_lib.zkhandler as zkhandler
-import client_lib.common as common
+from kazoo.exceptions import NoNodeError
+
+import daemon_lib.zkhandler as zkhandler
+
 
 #
 # Cluster search functions
@@ -50,6 +39,7 @@ def getClusterNetworkList(zk_conn):
         description_list.append(zkhandler.readdata(zk_conn, '/networks/{}'.format(vni)))
     return vni_list, description_list
 
+
 def searchClusterByVNI(zk_conn, vni):
     try:
         # Get the lists
@@ -64,6 +54,7 @@ def searchClusterByVNI(zk_conn, vni):
 
     return description
 
+
 def searchClusterByDescription(zk_conn, description):
     try:
         # Get the lists
@@ -78,6 +69,7 @@ def searchClusterByDescription(zk_conn, description):
 
     return vni
 
+
 def getNetworkVNI(zk_conn, network):
     # Validate and obtain alternate passed value
     if network.isdigit():
@@ -89,6 +81,7 @@ def getNetworkVNI(zk_conn, network):
 
     return net_vni
 
+
 def getNetworkDescription(zk_conn, network):
     # Validate and obtain alternate passed value
     if network.isdigit():
@@ -100,16 +93,19 @@ def getNetworkDescription(zk_conn, network):
 
     return net_description
 
+
 def getNetworkDHCPLeases(zk_conn, vni):
     # Get a list of DHCP leases by listing the children of /networks/<vni>/dhcp4_leases
     dhcp4_leases = zkhandler.listchildren(zk_conn, '/networks/{}/dhcp4_leases'.format(vni))
     return sorted(dhcp4_leases)
 
+
 def getNetworkDHCPReservations(zk_conn, vni):
     # Get a list of DHCP reservations by listing the children of /networks/<vni>/dhcp4_reservations
     dhcp4_reservations = zkhandler.listchildren(zk_conn, '/networks/{}/dhcp4_reservations'.format(vni))
     return sorted(dhcp4_reservations)
 
+
 def getNetworkACLs(zk_conn, vni, _direction):
     # Get the (sorted) list of active ACLs
     if _direction == 'both':
@@ -131,6 +127,7 @@ def getNetworkACLs(zk_conn, vni, _direction):
 
     return full_acl_list
 
+
 def getNetworkInformation(zk_conn, vni):
     description = zkhandler.readdata(zk_conn, '/networks/{}'.format(vni))
     nettype = zkhandler.readdata(zk_conn, '/networks/{}/nettype'.format(vni))
@@ -156,23 +153,32 @@ def getNetworkInformation(zk_conn, vni):
             'network': ip6_network,
             'gateway': ip6_gateway,
             'dhcp_flag': dhcp6_flag,
-         },
-         'ip4': {
+        },
+        'ip4': {
             'network': ip4_network,
             'gateway': ip4_gateway,
             'dhcp_flag': dhcp4_flag,
             'dhcp_start': dhcp4_start,
             'dhcp_end': dhcp4_end
-         }
+        }
     }
     return network_information
 
+
 def getDHCPLeaseInformation(zk_conn, vni, mac_address):
-    hostname = zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_leases/{}/hostname'.format(vni, mac_address))
-    ip4_address = zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_leases/{}/ipaddr'.format(vni, mac_address))
+    # Check whether this is a dynamic or static lease
     try:
-        timestamp = zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_leases/{}/expiry'.format(vni, mac_address))
-    except:
+        zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_leases/{}'.format(vni, mac_address))
+        type_key = 'dhcp4_leases'
+    except NoNodeError:
+        zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_reservations/{}'.format(vni, mac_address))
+        type_key = 'dhcp4_reservations'
+
+    hostname = zkhandler.readdata(zk_conn, '/networks/{}/{}/{}/hostname'.format(vni, type_key, mac_address))
+    ip4_address = zkhandler.readdata(zk_conn, '/networks/{}/{}/{}/ipaddr'.format(vni, type_key, mac_address))
+    if type_key == 'dhcp4_leases':
+        timestamp = zkhandler.readdata(zk_conn, '/networks/{}/{}/{}/expiry'.format(vni, type_key, mac_address))
+    else:
         timestamp = 'static'
 
     # Construct a data structure to represent the data
@@ -184,6 +190,7 @@ def getDHCPLeaseInformation(zk_conn, vni, mac_address):
     }
     return lease_information
 
+
 def getACLInformation(zk_conn, vni, direction, description):
     order = zkhandler.readdata(zk_conn, '/networks/{}/firewall_rules/{}/{}/order'.format(vni, direction, description))
     rule = zkhandler.readdata(zk_conn, '/networks/{}/firewall_rules/{}/{}/rule'.format(vni, direction, description))
@@ -197,32 +204,35 @@ def getACLInformation(zk_conn, vni, direction, description):
     }
     return acl_information
 
+
 def isValidMAC(macaddr):
     allowed = re.compile(r"""
                          (
                             ^([0-9A-F]{2}[:]){5}([0-9A-F]{2})$
                          )
                          """,
-                         re.VERBOSE|re.IGNORECASE)
+                         re.VERBOSE | re.IGNORECASE)
 
     if allowed.match(macaddr):
         return True
     else:
         return False
 
+
 def isValidIP(ipaddr):
     ip4_blocks = str(ipaddr).split(".")
     if len(ip4_blocks) == 4:
         for block in ip4_blocks:
             # Check if number is digit, if not checked before calling this function
             if not block.isdigit():
-                 return False
+                return False
             tmp = int(block)
             if 0 > tmp > 255:
-                 return False
+                return False
         return True
     return False
 
+
 #
 # Direct functions
 #
@@ -230,7 +240,7 @@ def add_network(zk_conn, vni, description, nettype,
                 domain, name_servers, ip4_network, ip4_gateway, ip6_network, ip6_gateway,
                 dhcp4_flag, dhcp4_start, dhcp4_end):
     # Ensure start and end DHCP ranges are set if the flag is set
-    if dhcp4_flag and ( not dhcp4_start or not dhcp4_end ):
+    if dhcp4_flag and (not dhcp4_start or not dhcp4_end):
         return False, 'ERROR: DHCPv4 start and end addresses are required for a DHCPv4-enabled network.'
 
     # Check if a network with this VNI or description already exists
@@ -276,6 +286,7 @@ def add_network(zk_conn, vni, description, nettype,
 
     return True, 'Network "{}" added successfully!'.format(description)
 
+
 def modify_network(zk_conn, vni, description=None, domain=None, name_servers=None,
                    ip4_network=None, ip4_gateway=None, ip6_network=None, ip6_gateway=None,
                    dhcp4_flag=None, dhcp4_start=None, dhcp4_end=None):
@@ -317,6 +328,7 @@ def modify_network(zk_conn, vni, description=None, domain=None, name_servers=Non
 
     return True, 'Network "{}" modified successfully!'.format(vni)
 
+
 def remove_network(zk_conn, network):
     # Validate and obtain alternate passed value
     vni = getNetworkVNI(zk_conn, network)
@@ -360,6 +372,7 @@ def add_dhcp_reservation(zk_conn, network, ipaddress, macaddress, hostname):
 
     return True, 'DHCP reservation "{}" added successfully!'.format(macaddress)
 
+
 def remove_dhcp_reservation(zk_conn, network, reservation):
     # Validate and obtain standard passed value
     net_vni = getNetworkVNI(zk_conn, network)
@@ -394,11 +407,12 @@ def remove_dhcp_reservation(zk_conn, network, reservation):
     # Remove the entry from zookeeper
     try:
         zkhandler.deletekey(zk_conn, '/networks/{}/dhcp4_{}/{}'.format(net_vni, lease_type_zk, match_description))
-    except:
+    except Exception:
         return False, 'ERROR: Failed to write to Zookeeper!'
 
     return True, 'DHCP {} "{}" removed successfully!'.format(lease_type_human, match_description)
 
+
 def add_acl(zk_conn, network, direction, description, rule, order):
     # Validate and obtain standard passed value
     net_vni = getNetworkVNI(zk_conn, network)
@@ -462,6 +476,7 @@ def add_acl(zk_conn, network, direction, description, rule, order):
 
     return True, 'Firewall rule "{}" added successfully!'.format(description)
 
+
 def remove_acl(zk_conn, network, description):
     # Validate and obtain standard passed value
     net_vni = getNetworkVNI(zk_conn, network)
@@ -502,6 +517,7 @@ def remove_acl(zk_conn, network, description):
 
     return True, 'Firewall rule "{}" removed successfully!'.format(match_description)
 
+
 def get_info(zk_conn, network):
     # Validate and obtain alternate passed value
     net_vni = getNetworkVNI(zk_conn, network)
@@ -514,6 +530,7 @@ def get_info(zk_conn, network):
 
     return True, network_information
 
+
 def get_list(zk_conn, limit, is_fuzzy=True):
     net_list = []
     full_net_list = zkhandler.listchildren(zk_conn, '/networks')
@@ -534,9 +551,9 @@ def get_list(zk_conn, limit, is_fuzzy=True):
         else:
             net_list.append(getNetworkInformation(zk_conn, net))
 
-    #output_string = formatNetworkList(zk_conn, net_list)
     return True, net_list
 
+
 def get_list_dhcp(zk_conn, network, limit, only_static=False, is_fuzzy=True):
     # Validate and obtain alternate passed value
     net_vni = getNetworkVNI(zk_conn, network)
@@ -547,10 +564,9 @@ def get_list_dhcp(zk_conn, network, limit, only_static=False, is_fuzzy=True):
 
     if only_static:
         full_dhcp_list = getNetworkDHCPReservations(zk_conn, net_vni)
-        reservations = True
     else:
-        full_dhcp_list = getNetworkDHCPLeases(zk_conn, net_vni)
-        reservations = False
+        full_dhcp_list = getNetworkDHCPReservations(zk_conn, net_vni)
+        full_dhcp_list += getNetworkDHCPLeases(zk_conn, net_vni)
 
     if limit:
         try:
@@ -558,14 +574,13 @@ def get_list_dhcp(zk_conn, network, limit, only_static=False, is_fuzzy=True):
                 limit = '^' + limit + '$'
 
             # Implcitly assume fuzzy limits
-            if not re.match('\^.*', limit):
+            if not re.match(r'\^.*', limit):
                 limit = '.*' + limit
-            if not re.match('.*\$', limit):
+            if not re.match(r'.*\$', limit):
                 limit = limit + '.*'
         except Exception as e:
             return False, 'Regex Error: {}'.format(e)
 
-
     for lease in full_dhcp_list:
         valid_lease = False
         if limit:
@@ -579,9 +594,9 @@ def get_list_dhcp(zk_conn, network, limit, only_static=False, is_fuzzy=True):
         if valid_lease:
             dhcp_list.append(getDHCPLeaseInformation(zk_conn, net_vni, lease))
 
-    #output_string = formatDHCPLeaseList(zk_conn, net_vni, dhcp_list, reservations=reservations)
     return True, dhcp_list
 
+
 def get_list_acl(zk_conn, network, limit, direction, is_fuzzy=True):
     # Validate and obtain alternate passed value
     net_vni = getNetworkVNI(zk_conn, network)
@@ -605,9 +620,9 @@ def get_list_acl(zk_conn, network, limit, direction, is_fuzzy=True):
                 limit = '^' + limit + '$'
 
             # Implcitly assume fuzzy limits
-            if not re.match('\^.*', limit):
+            if not re.match(r'\^.*', limit):
                 limit = '.*' + limit
-            if not re.match('.*\$', limit):
+            if not re.match(r'.*\$', limit):
                 limit = limit + '.*'
         except Exception as e:
             return False, 'Regex Error: {}'.format(e)
@@ -623,326 +638,4 @@ def get_list_acl(zk_conn, network, limit, direction, is_fuzzy=True):
         if valid_acl:
             acl_list.append(acl)
 
-    #output_string = formatACLList(zk_conn, net_vni, direction, acl_list)
     return True, acl_list
-
-# CLI-only functions
-def getOutputColours(network_information):
-    if network_information['ip6']['network'] != "None":
-        v6_flag_colour = ansiprint.green()
-    else:
-        v6_flag_colour = ansiprint.blue()
-    if network_information['ip4']['network'] != "None":
-        v4_flag_colour = ansiprint.green()
-    else:
-        v4_flag_colour = ansiprint.blue()
-
-    if network_information['ip6']['dhcp_flag'] == "True":
-        dhcp6_flag_colour = ansiprint.green()
-    else:
-        dhcp6_flag_colour = ansiprint.blue()
-    if network_information['ip4']['dhcp_flag'] == "True":
-        dhcp4_flag_colour = ansiprint.green()
-    else:
-        dhcp4_flag_colour = ansiprint.blue()
-
-    return v6_flag_colour, v4_flag_colour, dhcp6_flag_colour, dhcp4_flag_colour
-
-def format_info(network_information, long_output):
-    if not network_information:
-        click.echo("No network found")
-        return
-
-    v6_flag_colour, v4_flag_colour, dhcp6_flag_colour, dhcp4_flag_colour = getOutputColours(network_information)
-
-    # Format a nice output: do this line-by-line then concat the elements at the end
-    ainformation = []
-    ainformation.append('{}Virtual network information:{}'.format(ansiprint.bold(), ansiprint.end()))
-    ainformation.append('')
-    # Basic information
-    ainformation.append('{}VNI:{}            {}'.format(ansiprint.purple(), ansiprint.end(), network_information['vni']))
-    ainformation.append('{}Type:{}           {}'.format(ansiprint.purple(), ansiprint.end(), network_information['type']))
-    ainformation.append('{}Description:{}    {}'.format(ansiprint.purple(), ansiprint.end(), network_information['description']))
-    if network_information['type'] == 'managed':
-        ainformation.append('{}Domain:{}         {}'.format(ansiprint.purple(), ansiprint.end(), network_information['domain']))
-        ainformation.append('{}DNS Servers:{}    {}'.format(ansiprint.purple(), ansiprint.end(), ', '.join(network_information['name_servers'])))
-        if network_information['ip6']['network'] != "None":
-            ainformation.append('')
-            ainformation.append('{}IPv6 network:{}   {}'.format(ansiprint.purple(), ansiprint.end(), network_information['ip6']['network']))
-            ainformation.append('{}IPv6 gateway:{}   {}'.format(ansiprint.purple(), ansiprint.end(), network_information['ip6']['gateway']))
-            ainformation.append('{}DHCPv6 enabled:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), dhcp6_flag_colour, network_information['ip6']['dhcp_flag'], ansiprint.end()))
-        if network_information['ip4']['network'] != "None":
-            ainformation.append('')
-            ainformation.append('{}IPv4 network:{}   {}'.format(ansiprint.purple(), ansiprint.end(), network_information['ip4']['network']))
-            ainformation.append('{}IPv4 gateway:{}   {}'.format(ansiprint.purple(), ansiprint.end(), network_information['ip4']['gateway']))
-            ainformation.append('{}DHCPv4 enabled:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), dhcp4_flag_colour, network_information['ip4']['dhcp_flag'], ansiprint.end()))
-            if network_information['ip4']['dhcp_flag'] == "True":
-                ainformation.append('{}DHCPv4 range:{}   {} - {}'.format(ansiprint.purple(), ansiprint.end(), network_information['ip4']['dhcp_start'], network_information['ip4']['dhcp_end']))
-
-        if long_output:
-            dhcp4_reservations_list = getNetworkDHCPReservations(zk_conn, vni)
-            if dhcp4_reservations_list:
-                ainformation.append('')
-                ainformation.append('{}Client DHCPv4 reservations:{}'.format(ansiprint.bold(), ansiprint.end()))
-                ainformation.append('')
-                # Only show static reservations in the detailed information
-                dhcp4_reservations_string = formatDHCPLeaseList(zk_conn, vni, dhcp4_reservations_list, reservations=True)
-                for line in dhcp4_reservations_string.split('\n'):
-                    ainformation.append(line)
-
-            firewall_rules = zkhandler.listchildren(zk_conn, '/networks/{}/firewall_rules'.format(vni))
-            if firewall_rules:
-                ainformation.append('')
-                ainformation.append('{}Network firewall rules:{}'.format(ansiprint.bold(), ansiprint.end()))
-                ainformation.append('')
-                formatted_firewall_rules = get_list_firewall_rules(zk_conn, vni)
-
-    # Join it all together
-    click.echo('\n'.join(ainformation))
-
-def format_list(network_list):
-    if not network_list:
-        click.echo("No network found")
-        return
-
-    network_list_output = []
-
-    # Determine optimal column widths
-    net_vni_length = 5
-    net_description_length = 12
-    net_nettype_length = 8
-    net_domain_length = 6
-    net_v6_flag_length = 6
-    net_dhcp6_flag_length = 7
-    net_v4_flag_length = 6
-    net_dhcp4_flag_length = 7
-    for network_information in network_list:
-        # vni column
-        _net_vni_length = len(str(network_information['vni'])) + 1
-        if _net_vni_length > net_vni_length:
-            net_vni_length = _net_vni_length
-        # description column
-        _net_description_length = len(network_information['description']) + 1
-        if _net_description_length > net_description_length:
-            net_description_length = _net_description_length
-        # domain column
-        _net_domain_length = len(network_information['domain']) + 1
-        if _net_domain_length > net_domain_length:
-            net_domain_length = _net_domain_length
-
-    # Format the string (header)
-    network_list_output.append('{bold}\
-{net_vni: <{net_vni_length}} \
-{net_description: <{net_description_length}} \
-{net_nettype: <{net_nettype_length}} \
-{net_domain: <{net_domain_length}}  \
-{net_v6_flag: <{net_v6_flag_length}} \
-{net_dhcp6_flag: <{net_dhcp6_flag_length}} \
-{net_v4_flag: <{net_v4_flag_length}} \
-{net_dhcp4_flag: <{net_dhcp4_flag_length}} \
-{end_bold}'.format(
-            bold=ansiprint.bold(),
-            end_bold=ansiprint.end(),
-            net_vni_length=net_vni_length,
-            net_description_length=net_description_length,
-            net_nettype_length=net_nettype_length,
-            net_domain_length=net_domain_length,
-            net_v6_flag_length=net_v6_flag_length,
-            net_dhcp6_flag_length=net_dhcp6_flag_length,
-            net_v4_flag_length=net_v4_flag_length,
-            net_dhcp4_flag_length=net_dhcp4_flag_length,
-            net_vni='VNI',
-            net_description='Description',
-            net_nettype='Type',
-            net_domain='Domain',
-            net_v6_flag='IPv6',
-            net_dhcp6_flag='DHCPv6',
-            net_v4_flag='IPv4',
-            net_dhcp4_flag='DHCPv4',
-        )
-    )
-
-    for network_information in network_list:
-        v6_flag_colour, v4_flag_colour, dhcp6_flag_colour, dhcp4_flag_colour = getOutputColours(network_information)
-        if network_information['ip4']['network'] != "None":
-            v4_flag = 'True'
-        else:
-            v4_flag = 'False'
-
-        if network_information['ip6']['network'] != "None":
-            v6_flag = 'True'
-        else:
-            v6_flag = 'False'
-
-        if network_information['ip4']['dhcp_flag'] == "True":
-            dhcp4_range = '{} - {}'.format(network_information['ip4']['dhcp_start'], network_information['ip4']['dhcp_end'])
-        else:
-            dhcp4_range = 'N/A'
-
-        network_list_output.append(
-            '{bold}\
-{net_vni: <{net_vni_length}} \
-{net_description: <{net_description_length}} \
-{net_nettype: <{net_nettype_length}} \
-{net_domain: <{net_domain_length}}  \
-{v6_flag_colour}{net_v6_flag: <{net_v6_flag_length}}{colour_off} \
-{dhcp6_flag_colour}{net_dhcp6_flag: <{net_dhcp6_flag_length}}{colour_off} \
-{v4_flag_colour}{net_v4_flag: <{net_v4_flag_length}}{colour_off} \
-{dhcp4_flag_colour}{net_dhcp4_flag: <{net_dhcp4_flag_length}}{colour_off} \
-{end_bold}'.format(
-                bold='',
-                end_bold='',
-                net_vni_length=net_vni_length,
-                net_description_length=net_description_length,
-                net_nettype_length=net_nettype_length,
-                net_domain_length=net_domain_length,
-                net_v6_flag_length=net_v6_flag_length,
-                net_dhcp6_flag_length=net_dhcp6_flag_length,
-                net_v4_flag_length=net_v4_flag_length,
-                net_dhcp4_flag_length=net_dhcp4_flag_length,
-                net_vni=network_information['vni'],
-                net_description=network_information['description'],
-                net_nettype=network_information['type'],
-                net_domain=network_information['domain'],
-                net_v6_flag=v6_flag,
-                v6_flag_colour=v6_flag_colour,
-                net_dhcp6_flag=network_information['ip6']['dhcp_flag'],
-                dhcp6_flag_colour=dhcp6_flag_colour,
-                net_v4_flag=v4_flag,
-                v4_flag_colour=v4_flag_colour,
-                net_dhcp4_flag=network_information['ip4']['dhcp_flag'],
-                dhcp4_flag_colour=dhcp4_flag_colour,
-                colour_off=ansiprint.end()
-            )
-        )
-
-    click.echo('\n'.join(sorted(network_list_output)))
-
-def format_list_dhcp(dhcp_lease_list):
-    dhcp_lease_list_output = []
-
-    # Determine optimal column widths
-    lease_hostname_length = 9
-    lease_ip4_address_length = 11
-    lease_mac_address_length = 13
-    lease_timestamp_length = 13
-    for dhcp_lease_information in dhcp_lease_list:
-        # hostname column
-        _lease_hostname_length = len(dhcp_lease_information['hostname']) + 1
-        if _lease_hostname_length > lease_hostname_length:
-            lease_hostname_length = _lease_hostname_length
-        # ip4_address column
-        _lease_ip4_address_length = len(dhcp_lease_information['ip4_address']) + 1
-        if _lease_ip4_address_length > lease_ip4_address_length:
-            lease_ip4_address_length = _lease_ip4_address_length
-        # mac_address column
-        _lease_mac_address_length = len(dhcp_lease_information['mac_address']) + 1
-        if _lease_mac_address_length > lease_mac_address_length:
-            lease_mac_address_length = _lease_mac_address_length
-
-    # Format the string (header)
-    dhcp_lease_list_output.append('{bold}\
-{lease_hostname: <{lease_hostname_length}} \
-{lease_ip4_address: <{lease_ip4_address_length}} \
-{lease_mac_address: <{lease_mac_address_length}} \
-{lease_timestamp: <{lease_timestamp_length}} \
-{end_bold}'.format(
-            bold=ansiprint.bold(),
-            end_bold=ansiprint.end(),
-            lease_hostname_length=lease_hostname_length,
-            lease_ip4_address_length=lease_ip4_address_length,
-            lease_mac_address_length=lease_mac_address_length,
-            lease_timestamp_length=lease_timestamp_length,
-            lease_hostname='Hostname',
-            lease_ip4_address='IP Address',
-            lease_mac_address='MAC Address',
-            lease_timestamp='Timestamp'
-        )
-    )
-
-    for dhcp_lease_information in dhcp_lease_list:
-        dhcp_lease_list_output.append('{bold}\
-{lease_hostname: <{lease_hostname_length}} \
-{lease_ip4_address: <{lease_ip4_address_length}} \
-{lease_mac_address: <{lease_mac_address_length}} \
-{lease_timestamp: <{lease_timestamp_length}} \
-{end_bold}'.format(
-                bold='',
-                end_bold='',
-                lease_hostname_length=lease_hostname_length,
-                lease_ip4_address_length=lease_ip4_address_length,
-                lease_mac_address_length=lease_mac_address_length,
-                lease_timestamp_length=12,
-                lease_hostname=dhcp_lease_information['hostname'],
-                lease_ip4_address=dhcp_lease_information['ip4_address'],
-                lease_mac_address=dhcp_lease_information['mac_address'],
-                lease_timestamp=dhcp_lease_information['timestamp']
-            )
-        )
-
-    click.echo('\n'.join(sorted(dhcp_lease_list_output)))
-
-def format_list_acl(acl_list):
-    acl_list_output = []
-
-    # Determine optimal column widths
-    acl_direction_length = 10
-    acl_order_length = 6
-    acl_description_length = 12
-    acl_rule_length = 5
-    for acl_information in acl_list:
-        # order column
-        _acl_order_length = len(str(acl_information['order'])) + 1
-        if _acl_order_length > acl_order_length:
-            acl_order_length = _acl_order_length
-        # description column
-        _acl_description_length = len(acl_information['description']) + 1
-        if _acl_description_length > acl_description_length:
-            acl_description_length = _acl_description_length
-        # rule column
-        _acl_rule_length = len(acl_information['rule']) + 1
-        if _acl_rule_length > acl_rule_length:
-            acl_rule_length = _acl_rule_length
-
-    # Format the string (header)
-    acl_list_output.append('{bold}\
-{acl_direction: <{acl_direction_length}} \
-{acl_order: <{acl_order_length}} \
-{acl_description: <{acl_description_length}} \
-{acl_rule: <{acl_rule_length}} \
-{end_bold}'.format(
-            bold=ansiprint.bold(),
-            end_bold=ansiprint.end(),
-            acl_direction_length=acl_direction_length,
-            acl_order_length=acl_order_length,
-            acl_description_length=acl_description_length,
-            acl_rule_length=acl_rule_length,
-            acl_direction='Direction',
-            acl_order='Order',
-            acl_description='Description',
-            acl_rule='Rule',
-        )
-    )
-
-    for acl_information in acl_list:
-        acl_list_output.append('{bold}\
-{acl_direction: <{acl_direction_length}} \
-{acl_order: <{acl_order_length}} \
-{acl_description: <{acl_description_length}} \
-{acl_rule: <{acl_rule_length}} \
-{end_bold}'.format(
-                bold='',
-                end_bold='',
-                acl_direction_length=acl_direction_length,
-                acl_order_length=acl_order_length,
-                acl_description_length=acl_description_length,
-                acl_rule_length=acl_rule_length,
-                acl_direction=acl_information['direction'],
-                acl_order=acl_information['order'],
-                acl_description=acl_information['description'],
-                acl_rule=acl_information['rule'],
-            )
-        )
-
-    click.echo('\n'.join(sorted(acl_list_output)))
-

daemon-common/node.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+
+# node.py - PVC client function library, node management
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import time
+import re
+
+import daemon_lib.zkhandler as zkhandler
+import daemon_lib.common as common
+
+
+def getNodeInformation(zk_conn, node_name):
+    """
+    Gather information about a node from the Zookeeper database and return a dict() containing it.
+    """
+    node_daemon_state = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(node_name))
+    node_coordinator_state = zkhandler.readdata(zk_conn, '/nodes/{}/routerstate'.format(node_name))
+    node_domain_state = zkhandler.readdata(zk_conn, '/nodes/{}/domainstate'.format(node_name))
+    node_static_data = zkhandler.readdata(zk_conn, '/nodes/{}/staticdata'.format(node_name)).split()
+    node_cpu_count = int(node_static_data[0])
+    node_kernel = node_static_data[1]
+    node_os = node_static_data[2]
+    node_arch = node_static_data[3]
+    node_vcpu_allocated = int(zkhandler.readdata(zk_conn, 'nodes/{}/vcpualloc'.format(node_name)))
+    node_mem_total = int(zkhandler.readdata(zk_conn, '/nodes/{}/memtotal'.format(node_name)))
+    node_mem_allocated = int(zkhandler.readdata(zk_conn, '/nodes/{}/memalloc'.format(node_name)))
+    node_mem_provisioned = int(zkhandler.readdata(zk_conn, '/nodes/{}/memprov'.format(node_name)))
+    node_mem_used = int(zkhandler.readdata(zk_conn, '/nodes/{}/memused'.format(node_name)))
+    node_mem_free = int(zkhandler.readdata(zk_conn, '/nodes/{}/memfree'.format(node_name)))
+    node_load = float(zkhandler.readdata(zk_conn, '/nodes/{}/cpuload'.format(node_name)))
+    node_domains_count = int(zkhandler.readdata(zk_conn, '/nodes/{}/domainscount'.format(node_name)))
+    node_running_domains = zkhandler.readdata(zk_conn, '/nodes/{}/runningdomains'.format(node_name)).split()
+
+    # Construct a data structure to represent the data
+    node_information = {
+        'name': node_name,
+        'daemon_state': node_daemon_state,
+        'coordinator_state': node_coordinator_state,
+        'domain_state': node_domain_state,
+        'cpu_count': node_cpu_count,
+        'kernel': node_kernel,
+        'os': node_os,
+        'arch': node_arch,
+        'load': node_load,
+        'domains_count': node_domains_count,
+        'running_domains': node_running_domains,
+        'vcpu': {
+            'total': node_cpu_count,
+            'allocated': node_vcpu_allocated
+        },
+        'memory': {
+            'total': node_mem_total,
+            'allocated': node_mem_allocated,
+            'provisioned': node_mem_provisioned,
+            'used': node_mem_used,
+            'free': node_mem_free
+        }
+    }
+    return node_information
+
+
+#
+# Direct Functions
+#
+def secondary_node(zk_conn, node):
+    # Verify node is valid
+    if not common.verifyNode(zk_conn, node):
+        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
+
+    # Ensure node is a coordinator
+    daemon_mode = zkhandler.readdata(zk_conn, '/nodes/{}/daemonmode'.format(node))
+    if daemon_mode == 'hypervisor':
+        return False, 'ERROR: Cannot change router mode on non-coordinator node "{}"'.format(node)
+
+    # Ensure node is in run daemonstate
+    daemon_state = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(node))
+    if daemon_state != 'run':
+        return False, 'ERROR: Node "{}" is not active'.format(node)
+
+    # Get current state
+    current_state = zkhandler.readdata(zk_conn, '/nodes/{}/routerstate'.format(node))
+    if current_state == 'primary':
+        retmsg = 'Setting node {} in secondary router mode.'.format(node)
+        zkhandler.writedata(zk_conn, {
+            '/primary_node': 'none'
+        })
+    else:
+        return False, 'Node "{}" is already in secondary router mode.'.format(node)
+
+    return True, retmsg
+
+
+def primary_node(zk_conn, node):
+    # Verify node is valid
+    if not common.verifyNode(zk_conn, node):
+        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
+
+    # Ensure node is a coordinator
+    daemon_mode = zkhandler.readdata(zk_conn, '/nodes/{}/daemonmode'.format(node))
+    if daemon_mode == 'hypervisor':
+        return False, 'ERROR: Cannot change router mode on non-coordinator node "{}"'.format(node)
+
+    # Ensure node is in run daemonstate
+    daemon_state = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(node))
+    if daemon_state != 'run':
+        return False, 'ERROR: Node "{}" is not active'.format(node)
+
+    # Get current state
+    current_state = zkhandler.readdata(zk_conn, '/nodes/{}/routerstate'.format(node))
+    if current_state == 'secondary':
+        retmsg = 'Setting node {} in primary router mode.'.format(node)
+        zkhandler.writedata(zk_conn, {
+            '/primary_node': node
+        })
+    else:
+        return False, 'Node "{}" is already in primary router mode.'.format(node)
+
+    return True, retmsg
+
+
+def flush_node(zk_conn, node, wait=False):
+    # Verify node is valid
+    if not common.verifyNode(zk_conn, node):
+        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
+
+    retmsg = 'Flushing hypervisor {} of running VMs.'.format(node)
+
+    # Add the new domain to Zookeeper
+    zkhandler.writedata(zk_conn, {
+        '/nodes/{}/domainstate'.format(node): 'flush'
+    })
+
+    if wait:
+        while zkhandler.readdata(zk_conn, '/nodes/{}/domainstate'.format(node)) == 'flush':
+            time.sleep(1)
+        retmsg = 'Flushed hypervisor {} of running VMs.'.format(node)
+
+    return True, retmsg
+
+
+def ready_node(zk_conn, node, wait=False):
+    # Verify node is valid
+    if not common.verifyNode(zk_conn, node):
+        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
+
+    retmsg = 'Restoring hypervisor {} to active service.'.format(node)
+
+    # Add the new domain to Zookeeper
+    zkhandler.writedata(zk_conn, {
+        '/nodes/{}/domainstate'.format(node): 'unflush'
+    })
+
+    if wait:
+        while zkhandler.readdata(zk_conn, '/nodes/{}/domainstate'.format(node)) == 'unflush':
+            time.sleep(1)
+        retmsg = 'Restored hypervisor {} to active service.'.format(node)
+
+    return True, retmsg
+
+
+def get_info(zk_conn, node):
+    # Verify node is valid
+    if not common.verifyNode(zk_conn, node):
+        return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
+
+    # Get information about node in a pretty format
+    node_information = getNodeInformation(zk_conn, node)
+    if not node_information:
+        return False, 'ERROR: Could not get information about node "{}".'.format(node)
+
+    return True, node_information
+
+
+def get_list(zk_conn, limit, daemon_state=None, coordinator_state=None, domain_state=None, is_fuzzy=True):
+    node_list = []
+    full_node_list = zkhandler.listchildren(zk_conn, '/nodes')
+
+    for node in full_node_list:
+        if limit:
+            try:
+                if not is_fuzzy:
+                    limit = '^' + limit + '$'
+
+                if re.match(limit, node):
+                    node_list.append(getNodeInformation(zk_conn, node))
+            except Exception as e:
+                return False, 'Regex Error: {}'.format(e)
+        else:
+            node_list.append(getNodeInformation(zk_conn, node))
+
+    if daemon_state or coordinator_state or domain_state:
+        limited_node_list = []
+        for node in node_list:
+            add_node = False
+            if daemon_state and node['daemon_state'] == daemon_state:
+                add_node = True
+            if coordinator_state and node['coordinator_state'] == coordinator_state:
+                add_node = True
+            if domain_state and node['domain_state'] == domain_state:
+                add_node = True
+            if add_node:
+                limited_node_list.append(node)
+        node_list = limited_node_list
+
+    return True, node_list

daemon-common/vm.py

@@ -20,26 +20,15 @@
 #
 ###############################################################################
 
-import os
-import socket
 import time
-import uuid
 import re
-import subprocess
-import difflib
-import colorama
-import click
 import lxml.objectify
-import configparser
-import kazoo.client
 
-from collections import deque
+import daemon_lib.zkhandler as zkhandler
+import daemon_lib.common as common
 
-import client_lib.ansiprint as ansiprint
-import client_lib.zkhandler as zkhandler
-import client_lib.common as common
+import daemon_lib.ceph as ceph
 
-import client_lib.ceph as ceph
 
 #
 # Cluster search functions
@@ -53,6 +42,7 @@ def getClusterDomainList(zk_conn):
         name_list.append(zkhandler.readdata(zk_conn, '/domains/%s' % uuid))
     return uuid_list, name_list
 
+
 def searchClusterByUUID(zk_conn, uuid):
     try:
         # Get the lists
@@ -67,6 +57,7 @@ def searchClusterByUUID(zk_conn, uuid):
 
     return name
 
+
 def searchClusterByName(zk_conn, name):
     try:
         # Get the lists
@@ -81,6 +72,7 @@ def searchClusterByName(zk_conn, name):
 
     return uuid
 
+
 def getDomainUUID(zk_conn, domain):
     # Validate that VM exists in cluster
     if common.validateUUID(domain):
@@ -92,6 +84,7 @@ def getDomainUUID(zk_conn, domain):
 
     return dom_uuid
 
+
 def getDomainName(zk_conn, domain):
     # Validate that VM exists in cluster
     if common.validateUUID(domain):
@@ -103,6 +96,7 @@ def getDomainName(zk_conn, domain):
 
     return dom_name
 
+
 #
 # Direct functions
 #
@@ -118,6 +112,7 @@ def is_migrated(zk_conn, domain):
     else:
         return False
 
+
 def flush_locks(zk_conn, domain):
     # Validate that VM exists in cluster
     dom_uuid = getDomainUUID(zk_conn, domain)
@@ -145,7 +140,7 @@ def flush_locks(zk_conn, domain):
             else:
                 message = 'ERROR: Failed to flush locks on VM "{}"; check node logs for details.'.format(domain)
                 success = False
-        except:
+        except Exception:
             message = 'ERROR: Command ignored by node.'
             success = False
 
@@ -157,11 +152,12 @@ def flush_locks(zk_conn, domain):
 
     return success, message
 
-def define_vm(zk_conn, config_data, target_node, node_limit, node_selector, node_autostart, profile=None, initial_state='stop'):
+
+def define_vm(zk_conn, config_data, target_node, node_limit, node_selector, node_autostart, migration_method=None, profile=None, initial_state='stop'):
     # Parse the XML data
     try:
         parsed_xml = lxml.objectify.fromstring(config_data)
-    except:
+    except Exception:
         return False, 'ERROR: Failed to parse XML data.'
     dom_uuid = parsed_xml.uuid.text
     dom_name = parsed_xml.name.text
@@ -179,7 +175,7 @@ def define_vm(zk_conn, config_data, target_node, node_limit, node_selector, node
             return False, 'ERROR: Specified node "{}" is invalid.'.format(target_node)
 
     # Obtain the RBD disk list using the common functions
-    ddisks = common.getDomainDisks(parsed_xml)
+    ddisks = common.getDomainDisks(parsed_xml, {})
     rbd_list = []
     for disk in ddisks:
         if disk['type'] == 'rbd':
@@ -206,6 +202,7 @@ def define_vm(zk_conn, config_data, target_node, node_limit, node_selector, node
         '/domains/{}/node_limit'.format(dom_uuid): formatted_node_limit,
         '/domains/{}/node_selector'.format(dom_uuid): node_selector,
         '/domains/{}/node_autostart'.format(dom_uuid): node_autostart,
+        '/domains/{}/migration_method'.format(dom_uuid): migration_method,
         '/domains/{}/failedreason'.format(dom_uuid): '',
         '/domains/{}/consolelog'.format(dom_uuid): '',
         '/domains/{}/rbdlist'.format(dom_uuid): formatted_rbd_list,
@@ -215,7 +212,8 @@ def define_vm(zk_conn, config_data, target_node, node_limit, node_selector, node
 
     return True, 'Added new VM with Name "{}" and UUID "{}" to database.'.format(dom_name, dom_uuid)
 
-def modify_vm_metadata(zk_conn, domain, node_limit, node_selector, node_autostart, provisioner_profile):
+
+def modify_vm_metadata(zk_conn, domain, node_limit, node_selector, node_autostart, provisioner_profile, migration_method):
     dom_uuid = getDomainUUID(zk_conn, domain)
     if not dom_uuid:
         return False, 'ERROR: Could not find VM "{}" in the cluster!'.format(domain)
@@ -240,25 +238,55 @@ def modify_vm_metadata(zk_conn, domain, node_limit, node_selector, node_autostar
             '/domains/{}/profile'.format(dom_uuid): provisioner_profile
         })
 
+    if migration_method is not None:
+        zkhandler.writedata(zk_conn, {
+            '/domains/{}/migration_method'.format(dom_uuid): migration_method
+        })
+
     return True, 'Successfully modified PVC metadata of VM "{}".'.format(domain)
 
+
 def modify_vm(zk_conn, domain, restart, new_vm_config):
     dom_uuid = getDomainUUID(zk_conn, domain)
     if not dom_uuid:
         return False, 'ERROR: Could not find VM "{}" in the cluster!'.format(domain)
     dom_name = getDomainName(zk_conn, domain)
 
+    # Parse and valiate the XML
+    try:
+        parsed_xml = lxml.objectify.fromstring(new_vm_config)
+    except Exception:
+        return False, 'ERROR: Failed to parse XML data.'
+
+    # Obtain the RBD disk list using the common functions
+    ddisks = common.getDomainDisks(parsed_xml, {})
+    rbd_list = []
+    for disk in ddisks:
+        if disk['type'] == 'rbd':
+            rbd_list.append(disk['name'])
+
+    # Join the RBD list
+    if isinstance(rbd_list, list) and rbd_list:
+        formatted_rbd_list = ','.join(rbd_list)
+    else:
+        formatted_rbd_list = ''
+
     # Add the modified config to Zookeeper
     zk_data = {
         '/domains/{}'.format(dom_uuid): dom_name,
+        '/domains/{}/rbdlist'.format(dom_uuid): formatted_rbd_list,
         '/domains/{}/xml'.format(dom_uuid): new_vm_config
     }
     zkhandler.writedata(zk_conn, zk_data)
 
     if restart:
+        lock = zkhandler.exclusivelock(zk_conn, '/domains/{}/state'.format(dom_uuid))
+        lock.acquire()
         zkhandler.writedata(zk_conn, {'/domains/{}/state'.format(dom_uuid): 'restart'})
+        lock.release()
+
+    return True, 'Successfully modified configuration of VM "{}".'.format(domain)
 
-    return True, ''
 
 def dump_vm(zk_conn, domain):
     dom_uuid = getDomainUUID(zk_conn, domain)
@@ -270,13 +298,8 @@ def dump_vm(zk_conn, domain):
 
     return True, vm_xml
 
-def purge_vm(zk_conn, domain, is_cli=False):
-    """
-    Helper function for both undefine and remove VM to perform the shutdown, termination,
-    and configuration deletion.
-    """
 
-def undefine_vm(zk_conn, domain, is_cli=False):
+def undefine_vm(zk_conn, domain):
     # Validate that VM exists in cluster
     dom_uuid = getDomainUUID(zk_conn, domain)
     if not dom_uuid:
@@ -285,30 +308,26 @@ def undefine_vm(zk_conn, domain, is_cli=False):
     # Shut down the VM
     current_vm_state = zkhandler.readdata(zk_conn, '/domains/{}/state'.format(dom_uuid))
     if current_vm_state != 'stop':
-        if is_cli:
-            click.echo('Forcibly stopping VM "{}".'.format(domain))
         # Set the domain into stop mode
+        lock = zkhandler.exclusivelock(zk_conn, '/domains/{}/state'.format(dom_uuid))
+        lock.acquire()
         zkhandler.writedata(zk_conn, {'/domains/{}/state'.format(dom_uuid): 'stop'})
+        lock.release()
 
-        # Wait for 1 second to allow state to flow to all nodes
-        if is_cli:
-            click.echo('Waiting for cluster to update.')
+        # Wait for 2 seconds to allow state to flow to all nodes
         time.sleep(2)
 
     # Gracefully terminate the class instances
-    if is_cli:
-        click.echo('Deleting VM "{}" from nodes.'.format(domain))
     zkhandler.writedata(zk_conn, {'/domains/{}/state'.format(dom_uuid): 'delete'})
     time.sleep(2)
 
     # Delete the configurations
-    if is_cli:
-        click.echo('Undefining VM "{}".'.format(domain))
     zkhandler.deletekey(zk_conn, '/domains/{}'.format(dom_uuid))
 
     return True, 'Undefined VM "{}" from the cluster.'.format(domain)
 
-def remove_vm(zk_conn, domain, is_cli=False):
+
+def remove_vm(zk_conn, domain):
     # Validate that VM exists in cluster
     dom_uuid = getDomainUUID(zk_conn, domain)
     if not dom_uuid:
@@ -319,25 +338,20 @@ def remove_vm(zk_conn, domain, is_cli=False):
     # Shut down the VM
     current_vm_state = zkhandler.readdata(zk_conn, '/domains/{}/state'.format(dom_uuid))
     if current_vm_state != 'stop':
-        if is_cli:
-            click.echo('Forcibly stopping VM "{}".'.format(domain))
         # Set the domain into stop mode
+        lock = zkhandler.exclusivelock(zk_conn, '/domains/{}/state'.format(dom_uuid))
+        lock.acquire()
         zkhandler.writedata(zk_conn, {'/domains/{}/state'.format(dom_uuid): 'stop'})
+        lock.release()
 
-        # Wait for 1 second to allow state to flow to all nodes
-        if is_cli:
-            click.echo('Waiting for cluster to update.')
+        # Wait for 2 seconds to allow state to flow to all nodes
         time.sleep(2)
 
     # Gracefully terminate the class instances
-    if is_cli:
-        click.echo('Deleting VM "{}" from nodes.'.format(domain))
     zkhandler.writedata(zk_conn, {'/domains/{}/state'.format(dom_uuid): 'delete'})
     time.sleep(2)
 
     # Delete the configurations
-    if is_cli:
-        click.echo('Undefining VM "{}".'.format(domain))
     zkhandler.deletekey(zk_conn, '/domains/{}'.format(dom_uuid))
     time.sleep(2)
 
@@ -347,13 +361,12 @@ def remove_vm(zk_conn, domain, is_cli=False):
         try:
             disk_pool, disk_name = disk.split('/')
             retcode, message = ceph.remove_volume(zk_conn, disk_pool, disk_name)
-            if is_cli and message:
-                click.echo('{}'.format(message))
         except ValueError:
             continue
 
     return True, 'Removed VM "{}" and disks from the cluster.'.format(domain)
 
+
 def start_vm(zk_conn, domain):
     # Validate that VM exists in cluster
     dom_uuid = getDomainUUID(zk_conn, domain)
@@ -361,11 +374,15 @@ def start_vm(zk_conn, domain):
         return False, 'ERROR: Could not find VM "{}" in the cluster!'.format(domain)
 
     # Set the VM to start
+    lock = zkhandler.exclusivelock(zk_conn, '/domains/{}/state'.format(dom_uuid))
+    lock.acquire()
     zkhandler.writedata(zk_conn, {'/domains/{}/state'.format(dom_uuid): 'start'})
+    lock.release()
 
     return True, 'Starting VM "{}".'.format(domain)
 
-def restart_vm(zk_conn, domain):
+
+def restart_vm(zk_conn, domain, wait=False):
     # Validate that VM exists in cluster
     dom_uuid = getDomainUUID(zk_conn, domain)
     if not dom_uuid:
@@ -376,12 +393,23 @@ def restart_vm(zk_conn, domain):
     if current_state != 'start':
         return False, 'ERROR: VM "{}" is not in "start" state!'.format(domain)
 
-    # Set the VM to start
+    retmsg = 'Restarting VM "{}".'.format(domain)
+
+    # Set the VM to restart
+    lock = zkhandler.exclusivelock(zk_conn, '/domains/{}/state'.format(dom_uuid))
+    lock.acquire()
     zkhandler.writedata(zk_conn, {'/domains/{}/state'.format(dom_uuid): 'restart'})
+    lock.release()
 
-    return True, 'Restarting VM "{}".'.format(domain)
+    if wait:
+        while zkhandler.readdata(zk_conn, '/domains/{}/state'.format(dom_uuid)) == 'restart':
+            time.sleep(1)
+        retmsg = 'Restarted VM "{}"'.format(domain)
 
-def shutdown_vm(zk_conn, domain):
+    return True, retmsg
+
+
+def shutdown_vm(zk_conn, domain, wait=False):
     # Validate that VM exists in cluster
     dom_uuid = getDomainUUID(zk_conn, domain)
     if not dom_uuid:
@@ -392,10 +420,21 @@ def shutdown_vm(zk_conn, domain):
     if current_state != 'start':
         return False, 'ERROR: VM "{}" is not in "start" state!'.format(domain)
 
+    retmsg = 'Shutting down VM "{}"'.format(domain)
+
     # Set the VM to shutdown
+    lock = zkhandler.exclusivelock(zk_conn, '/domains/{}/state'.format(dom_uuid))
+    lock.acquire()
     zkhandler.writedata(zk_conn, {'/domains/{}/state'.format(dom_uuid): 'shutdown'})
+    lock.release()
+
+    if wait:
+        while zkhandler.readdata(zk_conn, '/domains/{}/state'.format(dom_uuid)) == 'shutdown':
+            time.sleep(1)
+        retmsg = 'Shut down VM "{}"'.format(domain)
+
+    return True, retmsg
 
-    return True, 'Shutting down VM "{}".'.format(domain)
 
 def stop_vm(zk_conn, domain):
     # Validate that VM exists in cluster
@@ -403,14 +442,15 @@ def stop_vm(zk_conn, domain):
     if not dom_uuid:
         return False, 'ERROR: Could not find VM "{}" in the cluster!'.format(domain)
 
-    # Get state and verify we're OK to proceed
-    current_state = zkhandler.readdata(zk_conn, '/domains/{}/state'.format(dom_uuid))
-
     # Set the VM to start
+    lock = zkhandler.exclusivelock(zk_conn, '/domains/{}/state'.format(dom_uuid))
+    lock.acquire()
     zkhandler.writedata(zk_conn, {'/domains/{}/state'.format(dom_uuid): 'stop'})
+    lock.release()
 
     return True, 'Forcibly stopping VM "{}".'.format(domain)
 
+
 def disable_vm(zk_conn, domain):
     # Validate that VM exists in cluster
     dom_uuid = getDomainUUID(zk_conn, domain)
@@ -423,16 +463,31 @@ def disable_vm(zk_conn, domain):
         return False, 'ERROR: VM "{}" must be stopped before disabling!'.format(domain)
 
     # Set the VM to start
+    lock = zkhandler.exclusivelock(zk_conn, '/domains/{}/state'.format(dom_uuid))
+    lock.acquire()
     zkhandler.writedata(zk_conn, {'/domains/{}/state'.format(dom_uuid): 'disable'})
+    lock.release()
 
     return True, 'Marked VM "{}" as disable.'.format(domain)
 
-def move_vm(zk_conn, domain, target_node):
+
+def move_vm(zk_conn, domain, target_node, wait=False, force_live=False):
     # Validate that VM exists in cluster
     dom_uuid = getDomainUUID(zk_conn, domain)
     if not dom_uuid:
         return False, 'ERROR: Could not find VM "{}" in the cluster!'.format(domain)
 
+    # Get state and verify we're OK to proceed
+    current_state = zkhandler.readdata(zk_conn, '/domains/{}/state'.format(dom_uuid))
+    if current_state != 'start':
+        # If the current state isn't start, preserve it; we're not doing live migration
+        target_state = current_state
+    else:
+        if force_live:
+            target_state = 'migrate-live'
+        else:
+            target_state = 'migrate'
+
     current_node = zkhandler.readdata(zk_conn, '/domains/{}/node'.format(dom_uuid))
 
     if not target_node:
@@ -450,27 +505,36 @@ def move_vm(zk_conn, domain, target_node):
 
         # Verify if node is current node
         if target_node == current_node:
+            last_node = zkhandler.readdata(zk_conn, '/domains/{}/lastnode'.format(dom_uuid))
+            if last_node:
+                zkhandler.writedata(zk_conn, {'/domains/{}/lastnode'.format(dom_uuid): ''})
+                return True, 'Making temporary migration permanent for VM "{}".'.format(domain)
+
             return False, 'ERROR: VM "{}" is already running on node "{}".'.format(domain, current_node)
 
     if not target_node:
         return False, 'ERROR: Could not find a valid migration target for VM "{}".'.format(domain)
 
-    current_vm_state = zkhandler.readdata(zk_conn, '/domains/{}/state'.format(dom_uuid))
-    if current_vm_state == 'start':
-        zkhandler.writedata(zk_conn, {
-            '/domains/{}/state'.format(dom_uuid): 'migrate',
-            '/domains/{}/node'.format(dom_uuid): target_node,
-            '/domains/{}/lastnode'.format(dom_uuid): ''
-        })
-    else:
-        zkhandler.writedata(zk_conn, {
-            '/domains/{}/node'.format(dom_uuid): target_node,
-            '/domains/{}/lastnode'.format(dom_uuid): ''
-        })
+    retmsg = 'Permanently migrating VM "{}" to node "{}".'.format(domain, target_node)
 
-    return True, 'Permanently migrating VM "{}" to node "{}".'.format(domain, target_node)
+    lock = zkhandler.exclusivelock(zk_conn, '/domains/{}/state'.format(dom_uuid))
+    lock.acquire()
+    zkhandler.writedata(zk_conn, {
+        '/domains/{}/state'.format(dom_uuid): target_state,
+        '/domains/{}/node'.format(dom_uuid): target_node,
+        '/domains/{}/lastnode'.format(dom_uuid): ''
+    })
+    lock.release()
 
-def migrate_vm(zk_conn, domain, target_node, force_migrate, is_cli=False):
+    if wait:
+        while zkhandler.readdata(zk_conn, '/domains/{}/state'.format(dom_uuid)) == target_state:
+            time.sleep(1)
+        retmsg = 'Permanently migrated VM "{}" to node "{}"'.format(domain, target_node)
+
+    return True, retmsg
+
+
+def migrate_vm(zk_conn, domain, target_node, force_migrate, wait=False, force_live=False):
     # Validate that VM exists in cluster
     dom_uuid = getDomainUUID(zk_conn, domain)
     if not dom_uuid:
@@ -479,22 +543,19 @@ def migrate_vm(zk_conn, domain, target_node, force_migrate, is_cli=False):
     # Get state and verify we're OK to proceed
     current_state = zkhandler.readdata(zk_conn, '/domains/{}/state'.format(dom_uuid))
     if current_state != 'start':
-        target_state = 'start'
+        # If the current state isn't start, preserve it; we're not doing live migration
+        target_state = current_state
     else:
-        target_state = 'migrate'
+        if force_live:
+            target_state = 'migrate-live'
+        else:
+            target_state = 'migrate'
 
     current_node = zkhandler.readdata(zk_conn, '/domains/{}/node'.format(dom_uuid))
     last_node = zkhandler.readdata(zk_conn, '/domains/{}/lastnode'.format(dom_uuid))
 
     if last_node and not force_migrate:
-        if is_cli:
-            click.echo('ERROR: VM "{}" has been previously migrated.'.format(domain))
-            click.echo('> Last node: {}'.format(last_node))
-            click.echo('> Current node: {}'.format(current_node))
-            click.echo('Run `vm unmigrate` to restore the VM to its previous node, or use `--force` to override this check.')
-            return False, ''
-        else:
-            return False, 'ERROR: VM "{}" has been previously migrated.'.format(domain)
+        return False, 'ERROR: VM "{}" has been previously migrated.'.format(domain)
 
     if not target_node:
         target_node = common.findTargetNode(zk_conn, dom_uuid)
@@ -520,15 +581,26 @@ def migrate_vm(zk_conn, domain, target_node, force_migrate, is_cli=False):
     if last_node and force_migrate:
         current_node = last_node
 
+    retmsg = 'Migrating VM "{}" to node "{}".'.format(domain, target_node)
+
+    lock = zkhandler.exclusivelock(zk_conn, '/domains/{}/state'.format(dom_uuid))
+    lock.acquire()
     zkhandler.writedata(zk_conn, {
-        '/domains/{}/state'.format(dom_uuid): 'migrate',
+        '/domains/{}/state'.format(dom_uuid): target_state,
         '/domains/{}/node'.format(dom_uuid): target_node,
         '/domains/{}/lastnode'.format(dom_uuid): current_node
     })
+    lock.release()
 
-    return True, 'Migrating VM "{}" to node "{}".'.format(domain, target_node)
+    if wait:
+        while zkhandler.readdata(zk_conn, '/domains/{}/state'.format(dom_uuid)) == target_state:
+            time.sleep(1)
+        retmsg = 'Migrated VM "{}" to node "{}"'.format(domain, target_node)
 
-def unmigrate_vm(zk_conn, domain):
+    return True, retmsg
+
+
+def unmigrate_vm(zk_conn, domain, wait=False, force_live=False):
     # Validate that VM exists in cluster
     dom_uuid = getDomainUUID(zk_conn, domain)
     if not dom_uuid:
@@ -540,20 +612,34 @@ def unmigrate_vm(zk_conn, domain):
         # If the current state isn't start, preserve it; we're not doing live migration
         target_state = current_state
     else:
-        target_state = 'migrate'
+        if force_live:
+            target_state = 'migrate-live'
+        else:
+            target_state = 'migrate'
 
     target_node = zkhandler.readdata(zk_conn, '/domains/{}/lastnode'.format(dom_uuid))
 
     if target_node == '':
         return False, 'ERROR: VM "{}" has not been previously migrated.'.format(domain)
 
+    retmsg = 'Unmigrating VM "{}" back to node "{}".'.format(domain, target_node)
+
+    lock = zkhandler.exclusivelock(zk_conn, '/domains/{}/state'.format(dom_uuid))
+    lock.acquire()
     zkhandler.writedata(zk_conn, {
         '/domains/{}/state'.format(dom_uuid): target_state,
         '/domains/{}/node'.format(dom_uuid): target_node,
         '/domains/{}/lastnode'.format(dom_uuid): ''
     })
+    lock.release()
+
+    if wait:
+        while zkhandler.readdata(zk_conn, '/domains/{}/state'.format(dom_uuid)) == target_state:
+            time.sleep(1)
+        retmsg = 'Unmigrated VM "{}" back to node "{}"'.format(domain, target_node)
+
+    return True, retmsg
 
-    return True, 'Unmigrating VM "{}" back to node "{}".'.format(domain, target_node)
 
 def get_console_log(zk_conn, domain, lines=1000):
     # Validate that VM exists in cluster
@@ -570,53 +656,6 @@ def get_console_log(zk_conn, domain, lines=1000):
 
     return True, loglines
 
-def follow_console_log(zk_conn, domain, lines=10):
-    # Validate that VM exists in cluster
-    dom_uuid = getDomainUUID(zk_conn, domain)
-    if not dom_uuid:
-        return False, 'ERROR: Could not find VM "{}" in the cluster!'.format(domain)
-
-    # Get the initial data from ZK
-    console_log = zkhandler.readdata(zk_conn, '/domains/{}/consolelog'.format(dom_uuid))
-
-    # Shrink the log buffer to length lines
-    shrunk_log = console_log.split('\n')[-lines:]
-    loglines = '\n'.join(shrunk_log)
-
-    # Print the initial data and begin following
-    print(loglines, end='')
-
-    try:
-        while True:
-            # Grab the next line set
-            new_console_log = zkhandler.readdata(zk_conn, '/domains/{}/consolelog'.format(dom_uuid))
-            # Split the new and old log strings into constitutent lines
-            old_console_loglines = console_log.split('\n')
-            new_console_loglines = new_console_log.split('\n')
-            # Set the console log to the new log value for the next iteration
-            console_log = new_console_log
-            # Remove the lines from the old log until we hit the first line of the new log; this
-            # ensures that the old log is a string that we can remove from the new log entirely
-            for index, line in enumerate(old_console_loglines, start=0):
-                if line == new_console_loglines[0]:
-                    del old_console_loglines[0:index]
-                    break
-            # Rejoin the log lines into strings
-            old_console_log = '\n'.join(old_console_loglines)
-            new_console_log = '\n'.join(new_console_loglines)
-            # Remove the old lines from the new log
-            diff_console_log = new_console_log.replace(old_console_log, "")
-            # If there's a difference, print it out
-            if diff_console_log:
-                print(diff_console_log, end='')
-            # Wait a second
-            time.sleep(1)
-    except kazoo.exceptions.NoNodeError:
-        return False, 'ERROR: VM has gone away.'
-    except:
-        return False, 'ERROR: Lost connection to Zookeeper node.'
-
-    return True, ''
 
 def get_info(zk_conn, domain):
     # Validate that VM exists in cluster
@@ -631,6 +670,7 @@ def get_info(zk_conn, domain):
 
     return True, domain_information
 
+
 def get_list(zk_conn, node, state, limit, is_fuzzy=True):
     if node:
         # Verify node is valid
@@ -638,8 +678,8 @@ def get_list(zk_conn, node, state, limit, is_fuzzy=True):
             return False, 'Specified node "{}" is invalid.'.format(node)
 
     if state:
-        valid_states = [ 'start', 'restart', 'shutdown', 'stop', 'disable', 'fail', 'migrate', 'unmigrate', 'provision' ]
-        if not state in valid_states:
+        valid_states = ['start', 'restart', 'shutdown', 'stop', 'disable', 'fail', 'migrate', 'unmigrate', 'provision']
+        if state not in valid_states:
             return False, 'VM state "{}" is not valid.'.format(state)
 
     full_vm_list = zkhandler.listchildren(zk_conn, '/domains')
@@ -649,9 +689,9 @@ def get_list(zk_conn, node, state, limit, is_fuzzy=True):
     if limit and is_fuzzy:
         try:
             # Implcitly assume fuzzy limits
-            if not re.match('\^.*', limit):
+            if not re.match(r'\^.*', limit):
                 limit = '.*' + limit
-            if not re.match('.*\$', limit):
+            if not re.match(r'.*\$', limit):
                 limit = limit + '.*'
         except Exception as e:
             return False, 'Regex Error: {}'.format(e)
@@ -691,263 +731,3 @@ def get_list(zk_conn, node, state, limit, is_fuzzy=True):
                     vm_list.append(common.getInformationFromXML(zk_conn, vm))
 
     return True, vm_list
-
-#
-# CLI-specific functions
-#
-def format_info(zk_conn, domain_information, long_output):
-    # Format a nice output; do this line-by-line then concat the elements at the end
-    ainformation = []
-    ainformation.append('{}Virtual machine information:{}'.format(ansiprint.bold(), ansiprint.end()))
-    ainformation.append('')
-    # Basic information
-    ainformation.append('{}UUID:{}               {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['uuid']))
-    ainformation.append('{}Name:{}               {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['name']))
-    ainformation.append('{}Description:{}        {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['description']))
-    ainformation.append('{}Profile:{}            {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['profile']))
-    ainformation.append('{}Memory (M):{}         {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['memory']))
-    ainformation.append('{}vCPUs:{}              {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['vcpu']))
-    ainformation.append('{}Topology (S/C/T):{}   {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['vcpu_topology']))
-
-    if long_output == True:
-        # Virtualization information
-        ainformation.append('')
-        ainformation.append('{}Emulator:{}           {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['emulator']))
-        ainformation.append('{}Type:{}               {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['type']))
-        ainformation.append('{}Arch:{}               {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['arch']))
-        ainformation.append('{}Machine:{}            {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['machine']))
-        ainformation.append('{}Features:{}           {}'.format(ansiprint.purple(), ansiprint.end(), ' '.join(domain_information['features'])))
-
-    # PVC cluster information
-    ainformation.append('')
-    dstate_colour = {
-        'start': ansiprint.green(),
-        'restart': ansiprint.yellow(),
-        'shutdown': ansiprint.yellow(),
-        'stop': ansiprint.red(),
-        'disable': ansiprint.blue(),
-        'fail': ansiprint.red(),
-        'migrate': ansiprint.blue(),
-        'unmigrate': ansiprint.blue()
-    }
-    ainformation.append('{}State:{}              {}{}{}'.format(ansiprint.purple(), ansiprint.end(), dstate_colour[domain_information['state']], domain_information['state'], ansiprint.end()))
-    ainformation.append('{}Current Node:{}       {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['node']))
-    if not domain_information['last_node']:
-        domain_information['last_node'] = "N/A"
-    ainformation.append('{}Previous Node:{}      {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['last_node']))
-
-    # Get a failure reason if applicable
-    if domain_information['failed_reason']:
-        ainformation.append('')
-        ainformation.append('{}Failure reason:{}     {}'.format(ansiprint.purple(), ansiprint.end(), domain_information['failed_reason']))
-
-    if not domain_information['node_selector']:
-        formatted_node_selector = "False"
-    else:
-        formatted_node_selector = domain_information['node_selector']
-
-    if not domain_information['node_limit']:
-        formatted_node_limit = "False"
-    else:
-        formatted_node_limit = ', '.join(domain_information['node_limit'])
-
-    if not domain_information['node_autostart']:
-        formatted_node_autostart = "False"
-    else:
-        formatted_node_autostart = domain_information['node_autostart']
-
-    ainformation.append('{}Migration selector:{} {}'.format(ansiprint.purple(), ansiprint.end(), formatted_node_selector))
-    ainformation.append('{}Node limit:{}         {}'.format(ansiprint.purple(), ansiprint.end(), formatted_node_limit))
-    ainformation.append('{}Autostart:{}          {}'.format(ansiprint.purple(), ansiprint.end(), formatted_node_autostart))
-
-    # Network list
-    net_list = []
-    for net in domain_information['networks']:
-        # Split out just the numerical (VNI) part of the brXXXX name
-        net_vnis = re.findall(r'\d+', net['source'])
-        if net_vnis:
-            net_vni = net_vnis[0]
-        else:
-            net_vni = re.sub('br', '', net['source'])
-        net_exists = zkhandler.exists(zk_conn, '/networks/{}'.format(net_vni))
-        if not net_exists and net_vni != 'cluster':
-            net_list.append(ansiprint.red() + net_vni + ansiprint.end() + ' [invalid]')
-        else:
-            net_list.append(net_vni)
-    ainformation.append('')
-    ainformation.append('{}Networks:{}           {}'.format(ansiprint.purple(), ansiprint.end(), ', '.join(net_list)))
-
-    if long_output == True:
-        # Disk list
-        ainformation.append('')
-        name_length = 0
-        for disk in domain_information['disks']:
-            _name_length = len(disk['name']) + 1
-            if _name_length > name_length:
-                name_length = _name_length
-        ainformation.append('{0}Disks:{1}        {2}ID  Type  {3: <{width}} Dev  Bus{4}'.format(ansiprint.purple(), ansiprint.end(), ansiprint.bold(), 'Name', ansiprint.end(), width=name_length))
-        for disk in domain_information['disks']:
-            ainformation.append('              {0: <3} {1: <5} {2: <{width}} {3: <4} {4: <5}'.format(domain_information['disks'].index(disk), disk['type'], disk['name'], disk['dev'], disk['bus'], width=name_length))
-        ainformation.append('')
-        ainformation.append('{}Interfaces:{}   {}ID  Type     Source     Model    MAC{}'.format(ansiprint.purple(), ansiprint.end(), ansiprint.bold(), ansiprint.end()))
-        for net in domain_information['networks']:
-            ainformation.append('              {0: <3} {1: <8} {2: <10} {3: <8} {4}'.format(domain_information['networks'].index(net), net['type'], net['source'], net['model'], net['mac']))
-        # Controller list
-        ainformation.append('')
-        ainformation.append('{}Controllers:{}  {}ID  Type           Model{}'.format(ansiprint.purple(), ansiprint.end(), ansiprint.bold(), ansiprint.end()))
-        for controller in domain_information['controllers']:
-            ainformation.append('              {0: <3} {1: <14} {2: <8}'.format(domain_information['controllers'].index(controller), controller['type'], controller['model']))
-
-    # Join it all together
-    information = '\n'.join(ainformation)
-    click.echo(information)
-
-    click.echo('')
-
-def format_list(zk_conn, vm_list, raw):
-    # Function to strip the "br" off of nets and return a nicer list
-    def getNiceNetID(domain_information):
-        # Network list
-        net_list = []
-        for net in domain_information['networks']:
-            # Split out just the numerical (VNI) part of the brXXXX name
-            net_vnis = re.findall(r'\d+', net['source'])
-            if net_vnis:
-                net_vni = net_vnis[0]
-            else:
-                net_vni = re.sub('br', '', net['source'])
-            net_list.append(net_vni)
-        return net_list
-
-    # Handle raw mode since it just lists the names
-    if raw:
-        for vm in sorted(item['name'] for item in vm_list):
-            click.echo(vm)
-        return True, ''
-
-    vm_list_output = []
-
-    # Determine optimal column widths
-    # Dynamic columns: node_name, node, migrated
-    vm_name_length = 5
-    vm_uuid_length = 37
-    vm_state_length = 6
-    vm_nets_length = 9
-    vm_ram_length = 8
-    vm_vcpu_length = 6
-    vm_node_length = 8
-    vm_migrated_length = 10
-    for domain_information in vm_list:
-        net_list = getNiceNetID(domain_information)
-        # vm_name column
-        _vm_name_length = len(domain_information['name']) + 1
-        if _vm_name_length > vm_name_length:
-            vm_name_length = _vm_name_length
-        # vm_state column
-        _vm_state_length = len(domain_information['state']) + 1
-        if _vm_state_length > vm_state_length:
-            vm_state_length = _vm_state_length
-        # vm_nets column
-        _vm_nets_length = len(','.join(net_list)) + 1
-        if _vm_nets_length > vm_nets_length:
-            vm_nets_length = _vm_nets_length
-        # vm_node column
-        _vm_node_length = len(domain_information['node']) + 1
-        if _vm_node_length > vm_node_length:
-            vm_node_length = _vm_node_length
-        # vm_migrated column
-        _vm_migrated_length = len(domain_information['migrated']) + 1
-        if _vm_migrated_length > vm_migrated_length:
-            vm_migrated_length = _vm_migrated_length
-
-    # Format the string (header)
-    vm_list_output.append(
-        '{bold}{vm_name: <{vm_name_length}} {vm_uuid: <{vm_uuid_length}} \
-{vm_state_colour}{vm_state: <{vm_state_length}}{end_colour} \
-{vm_networks: <{vm_nets_length}} \
-{vm_memory: <{vm_ram_length}} {vm_vcpu: <{vm_vcpu_length}} \
-{vm_node: <{vm_node_length}} \
-{vm_migrated: <{vm_migrated_length}}{end_bold}'.format(
-            vm_name_length=vm_name_length,
-            vm_uuid_length=vm_uuid_length,
-            vm_state_length=vm_state_length,
-            vm_nets_length=vm_nets_length,
-            vm_ram_length=vm_ram_length,
-            vm_vcpu_length=vm_vcpu_length,
-            vm_node_length=vm_node_length,
-            vm_migrated_length=vm_migrated_length,
-            bold=ansiprint.bold(),
-            end_bold=ansiprint.end(),
-            vm_state_colour='',
-            end_colour='',
-            vm_name='Name',
-            vm_uuid='UUID',
-            vm_state='State',
-            vm_networks='Networks',
-            vm_memory='RAM (M)',
-            vm_vcpu='vCPUs',
-            vm_node='Node',
-            vm_migrated='Migrated'
-        )
-    )
-            
-    # Format the string (elements)
-    for domain_information in vm_list:
-        if domain_information['state'] == 'start':
-            vm_state_colour = ansiprint.green()
-        elif domain_information['state'] == 'restart':
-            vm_state_colour = ansiprint.yellow()
-        elif domain_information['state'] == 'shutdown':
-            vm_state_colour = ansiprint.yellow()
-        elif domain_information['state'] == 'stop':
-            vm_state_colour = ansiprint.red()
-        elif domain_information['state'] == 'fail':
-            vm_state_colour = ansiprint.red()
-        else:
-            vm_state_colour = ansiprint.blue()
-
-        # Handle colouring for an invalid network config
-        raw_net_list = getNiceNetID(domain_information)
-        net_list = []
-        vm_net_colour = ''
-        for net_vni in raw_net_list:
-            net_exists = zkhandler.exists(zk_conn, '/networks/{}'.format(net_vni))
-            if not net_exists and net_vni != 'cluster':
-                vm_net_colour = ansiprint.red()
-            net_list.append(net_vni)
-
-        vm_list_output.append(
-            '{bold}{vm_name: <{vm_name_length}} {vm_uuid: <{vm_uuid_length}} \
-{vm_state_colour}{vm_state: <{vm_state_length}}{end_colour} \
-{vm_net_colour}{vm_networks: <{vm_nets_length}}{end_colour} \
-{vm_memory: <{vm_ram_length}} {vm_vcpu: <{vm_vcpu_length}} \
-{vm_node: <{vm_node_length}} \
-{vm_migrated: <{vm_migrated_length}}{end_bold}'.format(
-                vm_name_length=vm_name_length,
-                vm_uuid_length=vm_uuid_length,
-                vm_state_length=vm_state_length,
-                vm_nets_length=vm_nets_length,
-                vm_ram_length=vm_ram_length,
-                vm_vcpu_length=vm_vcpu_length,
-                vm_node_length=vm_node_length,
-                vm_migrated_length=vm_migrated_length,
-                bold='',
-                end_bold='',
-                vm_state_colour=vm_state_colour,
-                end_colour=ansiprint.end(),
-                vm_name=domain_information['name'],
-                vm_uuid=domain_information['uuid'],
-                vm_state=domain_information['state'],
-                vm_net_colour=vm_net_colour,
-                vm_networks=','.join(net_list),
-                vm_memory=domain_information['memory'],
-                vm_vcpu=domain_information['vcpu'],
-                vm_node=domain_information['node'],
-                vm_migrated=domain_information['migrated']
-            )
-        )
-
-    click.echo('\n'.join(sorted(vm_list_output)))
-
-    return True, ''
-

daemon-common/zkhandler.py

@@ -20,10 +20,9 @@
 #
 ###############################################################################
 
-import kazoo.client
+import time
 import uuid
 
-import client_lib.ansiprint as ansiprint
 
 # Exists function
 def exists(zk_conn, key):
@@ -33,22 +32,66 @@ def exists(zk_conn, key):
     else:
         return False
 
+
 # Child list function
 def listchildren(zk_conn, key):
     children = zk_conn.get_children(key)
     return children
 
+
 # Delete key function
 def deletekey(zk_conn, key, recursive=True):
     zk_conn.delete(key, recursive=recursive)
 
+
+# Rename key recursive function
+def rename_key_element(zk_conn, zk_transaction, source_key, destination_key):
+    data_raw = zk_conn.get(source_key)
+    data = data_raw[0]
+    zk_transaction.create(destination_key, data)
+
+    if zk_conn.get_children(source_key):
+        for child_key in zk_conn.get_children(source_key):
+            child_source_key = "{}/{}".format(source_key, child_key)
+            child_destination_key = "{}/{}".format(destination_key, child_key)
+            rename_key_element(zk_conn, zk_transaction, child_source_key, child_destination_key)
+
+    zk_transaction.delete(source_key)
+
+
+# Rename key function
+def renamekey(zk_conn, kv):
+    # Start up a transaction
+    zk_transaction = zk_conn.transaction()
+
+    # Proceed one KV pair at a time
+    for source_key in sorted(kv):
+        destination_key = kv[source_key]
+
+        # Check if the source key exists or fail out
+        if not zk_conn.exists(source_key):
+            raise
+        # Check if the destination key exists and fail out
+        if zk_conn.exists(destination_key):
+            raise
+
+        rename_key_element(zk_conn, zk_transaction, source_key, destination_key)
+
+    # Commit the transaction
+    try:
+        zk_transaction.commit()
+        return True
+    except Exception:
+        return False
+
+
 # Data read function
 def readdata(zk_conn, key):
     data_raw = zk_conn.get(key)
     data = data_raw[0].decode('utf8')
-    meta = data_raw[1]
     return data
 
+
 # Data write function
 def writedata(zk_conn, kv):
     # Start up a transaction
@@ -87,14 +130,56 @@ def writedata(zk_conn, kv):
     except Exception:
         return False
 
+
 # Write lock function
 def writelock(zk_conn, key):
-    lock_id = str(uuid.uuid1())
-    lock = zk_conn.WriteLock('{}'.format(key), lock_id)
+    count = 1
+    while True:
+        try:
+            lock_id = str(uuid.uuid1())
+            lock = zk_conn.WriteLock('{}'.format(key), lock_id)
+            break
+        except Exception:
+            count += 1
+            if count > 5:
+                break
+            else:
+                time.sleep(0.5)
+                continue
     return lock
 
+
 # Read lock function
 def readlock(zk_conn, key):
-    lock_id = str(uuid.uuid1())
-    lock = zk_conn.ReadLock('{}'.format(key), lock_id)
+    count = 1
+    while True:
+        try:
+            lock_id = str(uuid.uuid1())
+            lock = zk_conn.ReadLock('{}'.format(key), lock_id)
+            break
+        except Exception:
+            count += 1
+            if count > 5:
+                break
+            else:
+                time.sleep(0.5)
+                continue
+    return lock
+
+
+# Exclusive lock function
+def exclusivelock(zk_conn, key):
+    count = 1
+    while True:
+        try:
+            lock_id = str(uuid.uuid1())
+            lock = zk_conn.Lock('{}'.format(key), lock_id)
+            break
+        except Exception:
+            count += 1
+            if count > 5:
+                break
+            else:
+                time.sleep(0.5)
+                continue
     return lock

debian/changelog

@@ -1,8 +1,75 @@
+pvc (0.9.6-0) unstable; urgency=high
+
+  * Fixes bug with migrations
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Tue, 17 Nov 2020 13:01:54 -0500
+
+pvc (0.9.5-0) unstable; urgency=high
+
+  * Fixes bug with line count in log follow
+  * Fixes bug with disk stat output being None
+  * Adds short pretty health output
+  * Documentation updates
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Tue, 17 Nov 2020 12:34:04 -0500
+
+pvc (0.9.4-0) unstable; urgency=high
+
+  * Fixes major bug in OVA parser
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Tue, 10 Nov 2020 15:33:50 -0500
+
+pvc (0.9.3-0) unstable; urgency=high
+
+  * Fixes bugs with image & OVA upload parsing
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Mon, 09 Nov 2020 10:28:15 -0500
+
+pvc (0.9.2-0) unstable; urgency=high
+
+  * Major linting of the codebase with flake8; adds linting tools
+  * Implements CLI-based modification of VM vCPUs, memory, networks, and disks without directly editing XML
+  * Fixes bug where `pvc vm log -f` would show all 1000 lines before starting
+  * Fixes bug in default provisioner libvirt schema (`drive` -> `driver` typo)
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Sun, 08 Nov 2020 02:03:29 -0500
+
+pvc (0.9.1-0) unstable; urgency=high
+
+  * Added per-VM migration method feature
+  * Fixed bug with provisioner system template listing
+
+ -- Joshua Boniface <joshua@boniface.me>  Thu, 29 Oct 2020 12:15:28 -0400
+
+pvc (0.9.0-0) unstable; urgency=high
+
+  * Numerous bugfixes and improvements
+
+ -- Joshua Boniface <joshua@boniface.me>  Sun, 18 Oct 2020 14:31:00 -0400
+
+pvc (0.8-1) unstable; urgency=high
+
+  * Fix bug with IPv6 being enabled on bridged interfaces
+
+ -- Joshua Boniface <joshua@boniface.me>  Thu, 15 Oct 2020 11:02:24 -0400
+
+pvc (0.8-0) unstable; urgency=medium
+
+  * Numerous bugfixes and improvements
+
+ -- Joshua Boniface <joshua@boniface.me>  Tue, 11 Aug 2020 12:12:07 -0400
+
+pvc (0.7-0) unstable; urgency=medium
+
+  * Numerous bugfixes and improvements
+
+ -- Joshua Boniface <joshua@boniface.me>  Sat, 15 Feb 2020 23:24:17 -0500
+
 pvc (0.6-0) unstable; urgency=medium
 
   * Numerous improvements, implementation of provisioner and API client
 
- -- Joshua Boniface <joshua@boniface.me>  Sat, 08 Feb 2019 18:26:58 -0500
+ -- Joshua Boniface <joshua@boniface.me>  Sat, 08 Feb 2020 18:26:58 -0500
 
 pvc (0.5-0) unstable; urgency=medium
 

debian/control

@@ -6,34 +6,34 @@ Standards-Version: 3.9.8
 Homepage: https://www.boniface.me
 X-Python3-Version: >= 3.2
 
-Package: pvc-daemon
+Package: pvc-daemon-node
 Architecture: all
-Depends: systemd, pvc-client-common, python3-kazoo, python3-psutil, python3-apscheduler, python3-libvirt, python3-psycopg2, python3-dnspython, python3-yaml, python3-distutils, ipmitool, libvirt-daemon-system, arping, vlan, bridge-utils, dnsmasq, nftables, pdns-server, pdns-backend-pgsql
+Depends: systemd, pvc-daemon-common, python3-kazoo, python3-psutil, python3-apscheduler, python3-libvirt, python3-psycopg2, python3-dnspython, python3-yaml, python3-distutils, python3-rados, python3-gevent, ipmitool, libvirt-daemon-system, arping, vlan, bridge-utils, dnsmasq, nftables, pdns-server, pdns-backend-pgsql
 Suggests: pvc-client-api, pvc-client-cli
-Description: Parallel Virtual Cluster virtualization daemon (Python 3)
+Description: Parallel Virtual Cluster node daemon (Python 3)
  A KVM/Zookeeper/Ceph-based VM and private cloud manager
  .
  This package installs the PVC node daemon
 
-Package: pvc-client-common
+Package: pvc-daemon-api
+Architecture: all
+Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml, python3-flask-migrate, python3-flask-script, fio
+Description: Parallel Virtual Cluster API daemon (Python 3)
+ A KVM/Zookeeper/Ceph-based VM and private cloud manager
+ .
+ This package installs the PVC API daemon
+
+Package: pvc-daemon-common
 Architecture: all
 Depends: python3-kazoo, python3-psutil, python3-click, python3-lxml
-Description: Parallel Virtual Cluster common client libraries (Python 3)
+Description: Parallel Virtual Cluster common libraries (Python 3)
  A KVM/Zookeeper/Ceph-based VM and private cloud manager
  .
- This package installs the common client libraries
-
-Package: pvc-client-api
-Architecture: all
-Depends: systemd, pvc-client-common, python3-yaml, python3-flask, python3-flask-restful, python3-gevent, python3-celery, python-celery-common, python3-distutils, redis, python3-redis
-Description: Parallel Virtual Cluster API client (Python 3)
- A KVM/Zookeeper/Ceph-based VM and private cloud manager
- .
- This package installs the PVC API client daemon
+ This package installs the common libraries for the daemon and API
 
 Package: pvc-client-cli
 Architecture: all
-Depends: python3-requests, python3-yaml, python3-lxml
+Depends: python3-requests, python3-requests-toolbelt, python3-yaml, python3-lxml, python3-click
 Description: Parallel Virtual Cluster CLI client (Python 3)
  A KVM/Zookeeper/Ceph-based VM and private cloud manager
  .

debian/pvc-client-api.install

@@ -1,6 +0,0 @@
-client-api/pvc-api.py usr/share/pvc
-client-api/pvc-api.sample.yaml etc/pvc
-client-api/api_lib usr/share/pvc
-client-api/pvc-api.service lib/systemd/system
-client-api/pvc-provisioner-worker.service lib/systemd/system
-client-api/provisioner usr/share/pvc

debian/pvc-client-api.postinst

@@ -1,20 +0,0 @@
-#!/bin/sh
-
-# Install client binary to /usr/bin via symlink
-ln -s /usr/share/pvc/api.py /usr/bin/pvc-api
-
-# Reload systemd's view of the units
-systemctl daemon-reload
-
-# Restart the main daemon (or warn on first install)
-if systemctl is-active --quiet pvc-api.service; then
-    systemctl restart pvc-api.service
-else
-    echo "NOTE: The PVC client API daemon (pvc-api.service) has not been started; create a config file at /etc/pvc/pvc-api.yaml then start it."
-fi
-# Restart the worker daemon (or warn on first install)
-if systemctl is-active --quiet pvc-provisioner-worker.service; then
-    systemctl restart pvc-provisioner-worker.service
-else
-    echo "NOTE: The PVC provisioner worker daemon (pvc-provisioner-worker.service) has not been started; create a config file at /etc/pvc/pvc-api.yaml then start it."
-fi

debian/pvc-client-cli.install

@@ -1,2 +1,3 @@
 client-cli/pvc.py usr/share/pvc
 client-cli/cli_lib usr/share/pvc
+client-cli/scripts usr/share/pvc

debian/pvc-client-common.install

@@ -1 +0,0 @@
-client-common/* usr/share/pvc/client_lib

debian/pvc-daemon-api.install

@@ -0,0 +1,9 @@
+api-daemon/pvcapid.py usr/share/pvc
+api-daemon/pvcapid-manage.py usr/share/pvc
+api-daemon/pvc-api-db-upgrade usr/share/pvc
+api-daemon/pvcapid.sample.yaml etc/pvc
+api-daemon/pvcapid usr/share/pvc
+api-daemon/pvcapid.service lib/systemd/system
+api-daemon/pvcapid-worker.service lib/systemd/system
+api-daemon/provisioner usr/share/pvc
+api-daemon/migrations usr/share/pvc

debian/pvc-daemon-api.postinst

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# Reload systemd's view of the units
+systemctl daemon-reload
+
+# Restart the main daemon and apply database migrations (or warn on first install)
+if systemctl is-active --quiet pvcapid.service; then
+    systemctl stop pvcapid-worker.service
+    systemctl stop pvcapid.service
+    /usr/share/pvc/pvc-api-db-upgrade
+    systemctl start pvcapid.service
+    systemctl start pvcapid-worker.service
+else
+    echo "NOTE: The PVC client API daemon (pvcapid.service) and the PVC provisioner worker daemon (pvcapid-worker.service) have not been started; create a config file at /etc/pvc/pvcapid.yaml, then run the database configuration (/usr/share/pvc/pvc-api-db-upgrade) and start them manually."
+fi

debian/pvc-daemon-api.prerm

@@ -1,4 +1,4 @@
 #!/bin/sh
 
 # Remove client binary symlink
-rm -f /usr/bin/pvc-api
+rm -f /usr/bin/pvcapid

debian/pvc-daemon-common.install

@@ -0,0 +1 @@
+daemon-common/* usr/share/pvc/daemon_lib

debian/pvc-daemon-node.install

@@ -0,0 +1,7 @@
+node-daemon/pvcnoded.py usr/share/pvc
+node-daemon/pvcnoded.sample.yaml etc/pvc
+node-daemon/pvcnoded usr/share/pvc
+node-daemon/pvcnoded.service lib/systemd/system
+node-daemon/pvc.target lib/systemd/system
+node-daemon/pvc-flush.service lib/systemd/system
+node-daemon/monitoring usr/share/pvc

debian/pvc-daemon-node.postinst

@@ -4,8 +4,8 @@
 systemctl daemon-reload
 
 # Enable the service and target
-systemctl enable /lib/systemd/system/pvcd.service
-systemctl enable /lib/systemd/system/pvcd.target
+systemctl enable /lib/systemd/system/pvcnoded.service
+systemctl enable /lib/systemd/system/pvc.target
 
 # Inform administrator of the autoflush daemon if it is not enabled
 if ! systemctl is-active --quiet pvc-flush.service; then
@@ -13,8 +13,8 @@ if ! systemctl is-active --quiet pvc-flush.service; then
 fi
 
 # Inform administrator of the service restart/startup not occurring automatically
-if systemctl is-active --quiet pvcd.service; then
-    echo "NOTE: The PVC node daemon (pvcd.service) has not been restarted; this is up to the administrator."
+if systemctl is-active --quiet pvcnoded.service; then
+    echo "NOTE: The PVC node daemon (pvcnoded.service) has not been restarted; this is up to the administrator."
 else
-    echo "NOTE: The PVC node daemon (pvcd.service) has not been started; create a config file at /etc/pvc/pvcd.yaml then start it."
+    echo "NOTE: The PVC node daemon (pvcnoded.service) has not been started; create a config file at /etc/pvc/pvcnoded.yaml then start it."
 fi

debian/pvc-daemon-node.prerm

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# Disable the services
+systemctl disable pvcnoded.service
+systemctl disable pvc.target

debian/pvc-daemon.install

@@ -1,6 +0,0 @@
-node-daemon/pvcd.py usr/share/pvc
-node-daemon/pvcd.sample.yaml etc/pvc
-node-daemon/pvcd usr/share/pvc
-node-daemon/pvcd.target lib/systemd/system
-node-daemon/pvcd.service lib/systemd/system
-node-daemon/pvc-flush.service lib/systemd/system

debian/pvc-daemon.prerm

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-# Disable the services
-systemctl disable pvcd.service
-systemctl disable pvcd.target

docs/about.md

@@ -1,67 +1,149 @@
-# About the Parallel Virtual Cluster suite
+# About the Parallel Virtual Cluster system
 
-## Project Goals and Philosophy
+- [About the Parallel Virtual Cluster system](#about-the-parallel-virtual-cluster-system)
+  * [Project Motivation](#project-motivation)
+  * [Building Blocks](#building-blocks)
+  * [Cluster Architecture](#cluster-architecture)
+  * [Clients](#clients)
+    + [API Client](#api-client)
+    + [Direct Bindings](#direct-bindings)
+    + [CLI Client](#cli-client)
+  * [Deployment](#deployment)
+  * [Frequently Asked Questions](#frequently-asked-questions)
+    + [General Questions](#general-questions)
+    + [Feature Questions](#feature-questions)
+    + [Storage Questions](#storage-questions)
+  * [About The Author](#about-the-author)
+
+This document contains information about the project itself, the software stack, its motivations, and a number of frequently-asked questions.
+
+## Project Motivation
 
 Server management and system administration have changed significantly in the last decade. Computing as a resource is here, and software-defined is the norm. Gone are the days of pet servers, of tweaking configuration files by hand, and of painstakingly installing from ISO images in 52x CD-ROM drives. This is a brave new world.
 
-As part of this trend, the rise of IaaS (Infrastructure as a Service) has created an entirely new way for administrators and, increasingly, developers, to interact with servers. They need to be able to provision virtual machines easily and quickly, to ensure those virtual machines are reliable and consistent, and to avoid downtime wherever possible.
+As part of this trend, the rise of IaaS (Infrastructure as a Service) has created an entirely new way for administrators and, increasingly, developers, to interact with servers. They need to be able to provision virtual machines easily and quickly, to ensure those virtual machines are reliable and consistent, and to avoid downtime wherever possible. Even in a world of containers, VMs are still important, and are not going away, so some virtual management solution is a must.
 
-However, the state of the Free Software, virtual management ecosystem at the start of 2020 is quite disappointing. On the one hand are the giant, IaaS products like OpenStack and CloudStack. These are massive pieces of software, featuring dozens of interlocking parts, designed for massive clusters and public cloud deployments. They're great for a "hyperscale" provider, a large-scale SaaS/IaaS provider, or an enterprise. But they're not designed for small teams or small clusters. On the other hand, tools like Proxmox, oVirt, and even good old fashioned shell scripts are barely scalable, are showing their age, and have become increasingly unwieldy for advanced use-cases - great for one server, not so great for 9 in a highly-available cluster. Not to mention the constant attempts to monetize by throwing features behind Enterprise subscriptions. In short, there is a massive gap between the old-style, pet-based virtualization and the modern, large-scale, IaaS-type virtualization. This is not to mention the well-entrenched, proprietary solutions like VMWare and Nutanix which provide many of the features a small cluster administrator requires, but can be prohibitively expensive for small organizations.
+However, the current state of this ecosystem is lacking. At present there are 3 primary categories: the large "Stack" open-source projects, the smaller traditional "VM management" open-source projects, and the entrenched proprietary solutions.
 
-PVC aims to bridge these gaps. As a Python 3-based, fully-Free Software, scalable, and redundant private "cloud" that isn't afraid to say it's for small clusters, PVC is able to provide the simple, easy-to-use, small cluster you need today, with minimal administrator work, while being able to scale as your system grows, supporting hundreds or thousands of VMs across dozens of nodes. High availability is baked right into the core software at every layer, giving you piece of mind about your cluster, and ensuring that your systems keep running no matter what happens. And the interface couldn't be easier - a straightforward Click-based CLI and a Flask-based HTTP API provide access to the cluster for you to manage, either directly or though scripts or WebUIs. And since everything is Free Software, you can always inspect it, customize it to your use-case, add features, and contribute back to the community if you so choose.
+At the high end of the open-source ecosystem, are the "Stacks": OpenStack, CloudStack, and their numerous "vendorware" derivatives. These are large, unwieldy projects with dozens or hundreds of pieces of software to deploy in production, and can often require a large team just to understand and manage them. They're great if you're a large enterprise, building a public cloud, or have a team to get you going. But if you just want to run a small- to medium-sized virtual cluster for your SMB or ISP, they're definitely overkill and will cause you more headaches than they will solve long-term.
 
-PVC provides all the features you'd expect of a "cloud" system - easy management of VMs, including live migration between nodes for maximum uptime; virtual networking support using either vLANs or EVPN-based VXLAN; shared, redundant, object-based storage using Ceph, and a Python function library and convenient API interface for building your own interfaces. It is able to do this without being excessively complex, and without making sacrifices for legacy ideas.
+At the low end of the open source ecosystem, are what I call the "traditional tools". The biggest name in this space is ProxMox, though other, mostly defunct projects like Ganeti, tangential projects like Corosync/Pacemaker, and even traditional "I just use scripts" methods fit as well. These projects are great if you want to run a small server or homelab, but they quickly get unwieldy, though for the opposite reason from the Stacks: they're too simplistic, designed around single-host models, and when they provide redundancy at all it is often haphazard and nowhere near production-grade.
 
-If you need to run virtual machines, and don't have the time to learn the Stacks, the patience to deal with the old-style FOSS tools, or the money to spend on proprietary solutions, PVC might be just what you're looking for.
+Finally, the proprietary solutions like VMWare and Nutanix have entrenched themselves in the industry. They're excellent pieces of software providing just about anything you would need, but this comes at a significant cost, both in terms of money and also in software freedom and vendor lock-in. The licensing costs of Nutanix for instance can often make even enterprise-grade customers' accountants' heads spin.
+
+PVC seeks to bridge the gaps between these 3 categories. It is fully Free Software like the first two categories, and even more so - PVC is committed to never be "open-core" software and to never hide a single feature behind a paywall; it is able to scale from very small (1 or 3 node) clusters up to a dozen or more nodes, bridging the first two categories as effortlessly as the third does; it makes use of a hyperconverged architecture like ProxMox or Nuntanix to avoid wasting hardware resources on dedicated controller, hypervisor, and storage nodes; it is redundant at every layer from the ground-up, something that is not designed into any other free solution, and is able to tolerate the loss any single disk or entire node with barely a blip, all without administrator intervention; and finally, it is designed to be as simple to use as possible, with an Ansible-based node management framework, a RESTful API client interface, and a consistent, self-documenting CLI administration tool, allowing an administrator to create and manage their cluster quickly and simply, and then get on with more interesting things.
+
+In short, it is a Free Software, scalable, redundant, self-healing, and self-managing private cloud solution designed with administrator simplicity in mind.
+
+## Building Blocks
+
+PVC is build from a number of other, open source components. The main system itself is a series of software daemons (services) written in Python 3, with the CLI interface also written in Python 3.
+
+Virtual machines themselves are run with the Linux KVM subsystem via the Libvirt virtual machine management library. This provides the maximum flexibility and compatibility for running various guest operating systems in multiple modes (fully-virtualized, para-virtualized, virtio-enabled, etc.).
+
+To manage cluster state, PVC uses Zookeeper. This is an Apache project designed to provide a highly-available and always-consistent key-value database. The various daemons all connect to the distributed Zookeeper database to both obtain details about cluster state, and to manage that state. For instance the node daemon watches Zookeeper for information on what VMs to run, networks to create, etc., while the API writes information to Zookeeper in response to requests.
+
+Additional relational database functionality, specifically for the DNS aggregation subsystem and the VM provisioner, is provided by the PostgreSQL database and the Patroni management tool, which provides automatic clustering and failover for PostgreSQL database instances.
+
+Node network routing for managed networks providing EBGP VXLAN and route-learning is provided by FRRouting, a descendant project of Quaaga and GNU Zebra.
+
+The storage subsystem is provided by Ceph, a distributed object-based storage subsystem with extensive scalability, self-managing, and self-healing functionality. The Ceph RBD (Rados Block Device) subsystem is used to provide VM block devices similar to traditional LVM or ZFS zvols, but in a distributed, shared-storage manner.
+
+All the components are designed to be run on top of Debian GNU/Linux, specifically Debian 10.X "Buster", with the SystemD system service manager. This OS provides a stable base to run the various other subsystems while remaining truly Free Software, while SystemD provides functionality such as automatic daemon restarting and complex startup/shutdown ordering.
 
 ## Cluster Architecture
 
-A PVC cluster is based around "nodes", which are physical servers on which the various daemons, storage, networks, and virtual machines run. Each node is self-contained; it is able to perform any and all cluster functions if needed, and there is no segmentation of function between different types of physical hosts.
+A PVC cluster is based around "nodes", which are physical servers on which the various daemons, storage, networks, and virtual machines run. Each node is self-contained and is able to perform any and all cluster functions if needed; there is no segmentation of function between different types of physical hosts.
 
-A limited number of nodes, called "coordinators", are statically configured to provide additional services for the cluster. All databases for instance run on the coordinators, but not other nodes. This prevents any issues with scaling database clusters across dozens of hosts, while still retaining maximum redundancy. In a standard configuration, 3 or 5 nodes are designated as coordinators, and additional nodes connect to the coordinators for database access where required. For quorum purposes, there should always be an odd number of coordinators, and exceeding 5 is likely not required even for large clusters. PVC also supports a single node cluster format for extremely small clusters, homelabs, or testing where redundancy is not required.
+A limited number of nodes, called "coordinators", are statically configured to provide additional services for the cluster. For instance, all databases, FRRouting instances, and Ceph management daemons run only on the set of cluster coordinators. At cluster bootstrap, 1 (testing-only), 3 (small clusters), or 5 (large clusters) nodes may be chosen as the coordinators. Other nodes can then be added as "hypervisor" nodes, which then provide only block device (storage) and VM (compute) functionality by connecting to the set of coordinators. This limits the scaling problem of the databases while ensuring there is still maximum redundancy and resiliency for the core cluster services. Which nodes are designated as coordinators can be changed should the administrator so desire, simply by installing the required software on additional nodes, though this is not recommended (the Ceph system in particular is cumbersome to reconfigure).
 
-The primary database for PVC is Zookeeper, a highly-available key-value store designed with consistency in mind. Each node connects to the Zookeeper cluster running on the coordinators to send and receive data from the rest of the cluster. The API client (and Python function library) interface with this Zookeeper cluster directly to configure and obtain state about the various objects in the cluster. This database is the central authority for all nodes.
+During runtime, one coordinator is elected the "primary" for the cluster. This designation can shift dynamically in response to cluster events, or be manually migrated by an administrator. The coordinator takes on a number of roles for which only one host may be active at once, for instance to provide DHCP services to managed client networks or to interface with the API.
 
-Nodes are networked together via at least 3 different networks, set during bootstrap. The first is the "upstream" network, which provides upstream access for the nodes, for instance Internet connectivity, sending routes to client networks to upstream routers, etc. This should usually be a private/firewalled network to prevent unauthorized access to the cluster. The second is the "cluster" network, which is a private RFC1918 network that is unrouted and that nodes use to communicate between one another for Zookeeper access, Libvirt migrations, EVPN VXLAN tunnels, etc. The third is the "storage" network, which is used by the Ceph storage cluster for inter-OSD communication, allowing it to be separate from the main cluster network for maximum performance flexibility.
+Nodes are networked together via a set of statically-configured networks. At a minimum, 2 discrete networks are required, with an optional 3rd. The "upstream" network is the primary network for the nodes, and provides functions such as upstream Internet access, routing to and from the cluster nodes, and management via the API; it may be either a firewalled public or NAT'd RFC1918 network, but should never be exposed directly to the Internet. The "cluster" network is an unrouted RFC1918 network which provides inter-node communication for managed client network traffic (VXLANs), cross-node routing, VM migration and failover, and database replication and access. Finally, though optionally collapsed with the "cluster" network, the "storage" network is another unrouted RFC1918 network which provides a dedicated logical and/or physical link between the nodes for storage traffic, including VM block device storage traffic, inter-OSD replication traffic, and Ceph heartbeat traffic, thus allowing it to be completely isolated from the other networks for maximum performance. With each network is a single "floating" IP address which follows the primary coordinator, providing a single interface to the cluster. Once configured, the cluster is then able to create additional networks of two kinds, "bridged" traditional vLANs and "managed" routed VXLANs, to provide network access to VMs.
 
-Further information about the general cluster architecture can be found at the [cluster architecture page](/architecture/cluster).
+Further information about the general cluster architecture, including important considerations for node specifications/sizing and network configuration, can be found at the [cluster architecture page](/cluster-architecture).
 
-## Node Architecture
+## Clients
 
-Within each node, the PVC daemon is a single Python 3 program which handles all node functionality, including networking, starting cluster services, managing creation/removal of VMs, networks, and storage, and providing utilization statistics and information to the cluster.
+### API Client
 
-The daemon uses an object-oriented approach, with most cluster objects being represented by class objects of a specific type. Each node has a full view of all cluster objects and can interact with them based on events from the cluster as needed.
+The API client is a Flask-based RESTful API and is the core interface to PVC. By default the API will run on the primary coordinator, listening on TCP port 7370 on the "upstream" network floating IP address. All other clients communicate with this API to perform actions against the cluster. The API features basic authentication using UUID-based API keys to prevent unauthorized access, and can optionally be configured with full TLS encryption to provide integrity and confidentiality across public networks.
 
-Further information about the node daemon architecture can be found at the [daemon architecture page](/architecture/daemon).
+The API generally accepts all requests as HTTP form requests following standard RESTful guidelines, supporting arguments in the URI string or, with limited exceptions, in the message body. The API returns JSON response bodies to all requests consisting either of the information requested, or a `{ "message": "text" }` construct to pass informational status messages back to the client.
 
-## Client Architecture
+The API client manual can be found at the [API manual page](/manuals/api), and the full API documentation can be found at the [API reference page](/manuals/api-reference.html).
 
-### API client
+### Direct Bindings
 
-The API client is the core interface to PVC. It is a Flask RESTful API interface capable of performing all functions, and by default runs on the primary coordinator listening on port 7370 at the upstream floating IP address. Other clients, such as the CLI client, connect to the API to perform actions against the cluster. The API features a basic key-based authentication mechanism to prevent unauthorized access to the cluster if desired, and can also provide TLS-encrypted access for maximum security over public networks.
+The API client uses a dedicated set of Python libraries, packaged as the `pvc-daemon-common` Debian package, to communicate with the cluster. It is thus possible to build custom Python clients that directly interface with the PVC cluster, without having to get "into the weeds" of the Zookeeper or PostgreSQL databases.
 
-The API accepts all requests as HTTP form requests, supporting arguments both in the URI string as well as in the POST/PUT body. The API returns JSON response bodies to all requests.
+### CLI Client
 
-The API client manual can be found at the [API manual page](/manuals/api), and the [API documentation page](/manuals/api-reference.html).
+The CLI client is a Python Click application, which provides a convenient CLI interface to the API client. It supports connecting to multiple clusters from a single instance, with or without authentication and over both HTTP or HTTPS, including a special "local" cluster if the client determines that an API configuration exists on the local host. Information about the configured clusters is stored in a local JSON document, and a default cluster can be set with an environment variable.
 
-### Direct bindings
+The CLI client is self-documenting using the `-h`/`--help` arguments throughout, easing the administrator learning curve and providing easy access to command details. A short manual can also be found at the [CLI manual page](/manuals/cli).
 
-The API client uses a dedicated, independent set of functions to perform the actual communication with the cluster, which is packaged separately as the `pvc-client-common` package. These functions can be used directly by 3rd-party Python interfaces for PVC if desired.
+## Deployment
 
-### CLI client
+The overall management, deployment, bootstrapping, and configuring of nodes is accomplished via a set of Ansible roles and playbooks, found in the [`pvc-ansible` repository](https://github.com/parallelvirtualcluster/pvc-ansible), and nodes are installed via a custom installer ISO generated by the [`pvc-installer` repository](https://github.com/parallelvirtualcluster/pvc-installer). Once the cluster is set up, nodes can be added, replaced, updated, or reconfigured using this Ansible framework.
 
-The CLI client interface is a Click application, which provides a convenient CLI interface to the API client. It supports connecting to multiple clusters, over both HTTP and HTTPS and with authentication, including a special "local" cluster if the client determines that an `/etc/pvc/pvc-api.yaml` configuration exists on the host.
+The Ansible configuration and architecture manual can be found at the [Ansible manual page](/manuals/ansible).
 
-The CLI client is self-documenting using the `-h`/`--help` arguments, though a short manual can be found at the [CLI manual page](/manuals/cli).
+## Frequently Asked Questions
 
-## Deployment architecture
+### General Questions
 
-The overall management, deployment, bootstrapping, and configuring of nodes is accomplished via a set of Ansible roles, found in the [`pvc-ansible` repository](https://github.com/parallelvirtualcluster/pvc-ansible), and nodes are installed via a custom installer ISO generated by the [`pvc-installer` repository](https://github.com/parallelvirtualcluster/pvc-installer). Once the cluster is set up, nodes can be added, replaced, or updated using this Ansible framework.
+#### What is it?
 
-Further information about the Ansible deployment architecture can be found at the [Ansible architecture page](/architecture/ansible).
+PVC is a virtual machine management suite designed around high-availability and ease-of-use. It can be considered an alternative to OpenStack, ProxMox, Nutanix, and other similar solutions that manage not just the VMs, but the surrounding infrastructure as well.
 
-The Ansible configuration manual can be found at the [Ansible manual page](/manuals/ansible).
+#### Why would you make this?
 
-## About the author
+After becoming frustrated by numerous other management tools, I discovered that what I wanted didn't exist as FLOSS software, so I built it myself. Since then, I have also been able to leverage PVC both for my own purposes as well as for my employer, a win-win for the project.
+
+#### Is PVC right for me?
+
+PVC might be right for you if:
+
+1. You need KVM-based VMs.
+2. You want management of storage and networking (a.k.a. "batteries-included") in the same tool.
+3. You want hypervisor-level redundancy, able to tolerate hypervisor downtime seamlessly, for all elements of the stack.
+
+I built PVC for my homelab first, found a perfect use-case with my employer, and think it might be useful to you too.
+
+#### Is 3 hypervisors really the minimum?
+
+For a redundant cluster, yes. PVC requires a majority quorum for proper operation at various levels, and the smallest possible majority quorum is 2-of-3; thus 3 nodes is the safe minimum. That said, you can run PVC on a single node for testing/lab purposes without host-level redundancy, should you wish to do so, and it might also be possible to run 2 "main" systems with a 3rd "quorum observer" hosting only the management tools but no VMs, however this is not officially supported.
+
+### Feature Questions
+
+#### Does PVC support containers (Docker/Kubernetes/LXC/etc.)?
+
+No, not directly. PVC supports only KVM VMs. To run containers, you would need to run a VM which then runs your containers. For instance PVC makes an excellent underlying layer for a virtual Kubernetes cluster, instead of bare hardware.
+
+#### Does PVC have a WebUI?
+
+Not yet. Right now, PVC management is done exclusively with the CLI interface to the API. A WebUI can and likely will be built in the future, but I'm not a frontend developer and I do not consider this a personal priority. As of late 2020 the API is generally stable, so I would welcome 3rd party assistance here.
+
+### Storage Questions
+
+#### Can I use RAID-5/RAID-6 with PVC?
+
+The short answer is no. The long answer is: Ceph, the storage backend used by PVC, does support "erasure coded" pools which implement a RAID-5-like (striped with distributed parity) functionality, but PVC does not support this for several reasons, mostly related to ease of management and performance. If you use PVC, you must accept at the very least a 2x storage penalty, and for true multi-node safety and resiliency, a 3x storage penalty for VM storage. This is a trade-off of the architecture and should be taken into account when sizing storage in nodes.
+
+#### Can I use spinning HDDs with PVC?
+
+You can, but you won't like the results. SSDs, and specifically datacentre-grade SSDs for resiliency, are required to obtain any sort of reasonable performance when running multiple VMs. The higher-performance the drives, the faster the storage.
+
+#### What network speed does PVC require?
+
+For optimal performance, nodes should use at least 10-Gigabit Ethernet network interfaces wherever possible, and on large clusters a dedicated 10-Gigabit "storage" network, separate from the "upstream"/"cluster" networks, is strongly recommended. The storage system performance, especially for writes, is more heavily bottlenecked by the network speed than the actual storage device speed when speaking of high-performance disks. 1-Gigabit Ethernet will be sufficient for some use-cases and is sufficient for the non-storage networks (VM traffic notwithstanding), but storage performance will become severely limited as the cluster grows. Even slower network speeds (e.g. 100-Megabit) are not sufficient for PVC to operate properly except in very limited testing scenarios.
+
+#### What Ceph version does PVC use?
+
+PVC requires Ceph 14.x (Nautilus). The official PVC repository at https://repo.bonifacelabs.ca includes Ceph 14.2.x (updated regularly), since Debian Buster by default includes only 12.x (Luminous).
+
+## About The Author
+
+PVC is written by [Joshua](https://www.boniface.me) [M.](https://bonifacelabs.ca) [Boniface](https://github.com/joshuaboniface). A Linux system administrator by trade, Joshua is always looking for the best solutions to his user's problems, be they developers or end users. PVC grew out of his frustration with the various FOSS virtualization tools, as well as and specifically, the constant failures of Pacemaker/Corosync to gracefully manage a virtualization cluster. He started work on PVC at the end of May 2018 as a simple alternative to a Corosync/Pacemaker-managed virtualization cluster, and has been growing the feature set and stability of the system ever since.
 
-PVC is written by [Joshua](https://www.boniface.me) [M.](https://bonifacelabs.ca) [Boniface](https://github.com/joshuaboniface). A Linux system administrator by trade, Joshua is always looking for the best solutions to his user's problems, be they developers or end users. PVC grew out of his frustration with the various FOSS virtualization tools, as well as and specifically, the constant failures of Pacemaker/Corosync to gracefully manage a virtualization cluster. He started work on PVC at the end of May 2018 as a simple alternative to a Corosync/Pacemaker-managed virtualization cluster, and has been growing the feature set in starts and stops ever since.

docs/architecture/ansible.md

@@ -1,43 +0,0 @@
-# PVC Ansible architecture
-
-The PVC Ansible setup and management framework is written in Ansible. It consists of two roles: `base` and `pvc`.
-
-## Base role
-
-The Base role configures a node to a specific, standard base Debian system, with a number of PVC-specific tweaks. Some examples include:
-
-* Installing the custom PVC repository at Boniface Labs.
-
-* Removing several unnecessary packages and installing numerous additional packages.
-
-* Automatically configuring network interfaces based on the `group_vars` configuration.
-
-* Configuring several general `sysctl` settings for optimal performance.
-
-* Installing and configuring rsyslog, postfix, ntpd, ssh, and fail2ban.
-
-* Creating the users specified in the `group_vars` configuration.
-
-* Installing custom MOTDs, bashrc files, vimrc files, and other useful configurations for each user.
-
-The end result is a standardized "PVC node" system ready to have the daemons installed by the PVC role.
-
-## PVC role
-
-The PVC role configures all the dependencies of PVC, including storage, networking, and databases, then installs the PVC daemon itself. Specifically, it will, in order:
-
-* Install Ceph, configure and bootstrap a new cluster if `bootstrap=yes` is set, configure the monitor and manager daemons, and start up the cluster ready for the addition of OSDs via the client interface (coordinators only).
-
-* Install, configure, and if `bootstrap=yes` is set, bootstrap a Zookeeper cluster (coordinators only).
-
-* Install, configure, and if `bootstrap=yes` is set`, bootstrap a Patroni PostgreSQL cluster for the PowerDNS aggregator (coordinators only).
-
-* Install and configure Libvirt.
-
-* Install and configure FRRouting.
-
-* Install and configure the main PVC daemon and API client, including initializing the PVC cluster (`pvc init`).
-
-## Completion
-
-Once the entire playbook has run for the first time against a given host, the host will be rebooted to apply all the configured services. On startup, the system should immediately launch the PVC daemon, check in to the Zookeeper cluster, and become ready. The node will be in `flushed` state on its first boot; the administrator will need to run `pvc node unflush <node>` to set the node into active state ready to handle virtual machines.

docs/architecture/api.md

@@ -1,7 +0,0 @@
-# PVC API architecture
-
-The PVC API is a standalone client application for PVC. It interfaces directly with the Zookeeper database to manage state.
-
-The API is built using Flask and is packaged in the Debian package `pvc-client-api`. The API depends on the common client functions of the `pvc-client-common` package as does the CLI client.
-
-Details of the API interface can be found in [the manual](/manuals/api).

docs/architecture/cli.md

@@ -1,7 +0,0 @@
-# PVC CLI architecture
-
-The PVC CLI is a standalone client application for PVC. It interfaces with the PVC API, via a configurable list of clusters with customizable hosts, ports, addresses, and authentication.
-
-The CLI is build using Click and is packaged in the Debian package `pvc-client-cli`. The CLI does not depend on any other PVC components and can be used independently on arbitrary systems.
-
-The CLI is self-documenting, however [the manual](/manuals/cli) details the required configuration.

docs/architecture/cluster.md

@@ -1,169 +0,0 @@
-# PVC Cluster Architecture considerations
-
-This document contains considerations the administrator should make when preparing for and building a PVC cluster. It includes four main subsections: node specifications, storage specifications, network layout, and node layout, plus a fifth section featuring diagrams of 3 example topologies.
-
-## Node Specifications: Considering the size of nodes
-
-Each node in the cluster must be sized based on the needs of the cluster and the load placed on it. In general, taller nodes are better for performance and allow for a more powerful cluster on less hardware, though the needs of each specific environment and workload my affect this differently.
-
-At a bare minimum, each node should have the following specifications:
-
-* 12x 1.8GHz or better Intel/AMD cores from at least the Nehalem/Bulldozer eras (~2008 or newer)
-* 48GB of RAM
-* 2x 1Gbps Ethernet interfaces
-* 1x 10GB+ system disk (SSD/HDD/USB/SD/eMMC flash)
-* 1x 400GB+ OSD data disk (SSD)
-
-For a cluster of 3 such nodes, this will provide a total of:
-
-* 36 total CPU cores
-* 144GB RAM
-* 400GB usable Ceph storage space (`copies=3`)
-
-Of this, some amount of CPU and RAM will be used by the storage subsystem and the PVC daemons themselves, meaning that the total available for virtual machines is slightly less. Generally, each OSD data disk will consume 1 vCPU at load and 1-2GB RAM, so nodes should be sized not only according to the VM workload, but the number of storage disks per node. Additionally the coordinator databases will use additional RAM and CPU resources of up to 1-4GB per node, though there is generally little need to spec coordinators any larger than non-coordinator nodes and the VM automatic node selection process will take used RAM into account by default.
-
-## Storage Layout: Ceph and OSDs
-
-The Ceph subsystem of PVC, if enabled, creates a "hyperconverged" setup whereby storage and VM hypervisor functions are collocated onto the same physical servers. The performance of the storage must be taken into account when sizing the nodes as mentioned above.
-
-The Ceph system is laid out similar to the other daemons. The Ceph Monitor and Manager functions are delegated to the Coordinators over the cluster network, with all nodes connecting to these hosts to obtain the CRUSH maps and select OSD disks. OSDs are then distributed on all hosts, including non-coordinator hypervisors, and communicate with clients over the cluster network and with each other (for replication, failover, etc.) over the storage network.
-
-PVC Ceph pools make use of the replication mechanism of Ceph to store multiple copies of each object, thus ensuring that data is always available even when a host is unavailable. Note that, mostly for performance reasons related to rewrites and random I/O, erasure coding is *not* supported in PVC.
-
-The default replication level for a new pool is `copies=3, mincopies=2`. This will store 3 copies of each object, with a host-level failure domain, and will allow I/O as long as 2 copies are available. Thus, in a cluster of any size, all data is fully available even if a single host becomes unavailable. It will however use 3x the space for each piece of data stored, which must be considered when sizing the disk space for the cluster: a pool in this configuration, running on 3 nodes each with a single 400GB disk, will effectively have 400GB of total space available for use. Additionally, new disks must be added in groups of 3 spread across the nodes in order to be able to take advantage of the additional space, since each write will require creating 3 copies across each of the 3 hosts.
-
-Non-default values can also be set at pool creation time. For instance, one could create a `copies=3, mincopies=1` pool, which would allow I/O with two hosts down but leaves the cluster susceptible to a write hole should a disk fail in this state. Alternatively, for more resilience, one could create a `copies=4, mincopies=2` pool, which will allow 2 hosts to fail without a write hole, but would consume 4x the space for each piece of data stored and require new disks to be added in groups of 4 instead. Practically any combination of values is possible, however these 3 are the most relevant for most use-cases, and for most, especially small, clusters, the default is sufficient to provide solid redundancy and guard against host failures until the administrator can respond.
-
-Replication levels cannot be changed within PVC once a pool is created, however they can be changed via manual Ceph commands on a coordinator should the administrator require this. In any case, the administrator should carefully consider sizing, failure domains, and performance when selecting storage devices to ensure the right level of resiliency versus data usage for their use-case and cluster size.
-
-## Network Layout: Considering the required networks
-
-A PVC cluster needs, at minimum, 3 networks in order to function properly. Each of the three networks and its function is detailed below. An additional two sections cover the two kinds of client networks and the considerations for them.
-
-### Physical network considerations
-
-At a minimum, a production PVC cluster should use at least two 1Gbps Ethernet interfaces, connected in an LACP or active-backup bond on one or more switches. On top of this bond, the various cluster networks should be configured as vLANs.
-
-More advanced physical network layouts are also possible. For instance, one could have two isolated networks. On the first network, each node has two 10Gbps Ethernet interfaces, which are combined in a bond across two redundant switch fabrics and that handle the upstream and cluster networks. On the second network, each node has an additional two 10Gbps, which are also combined in a bond across the redundant switch fabrics and handle the storage network. This configuration could support up to 10Gbps of aggregate client traffic while also supporting 10Gbps of aggregate storage traffic. Even more complex network configurations are possible if the cluster requires such performance. See the [Example Configurations](#example-configurations) section for some examples.
-
-### Upstream: Connecting the nodes to the wider world
-
-The upstream network functions as the main upstream for the cluster nodes, providing Internet access and a way to route managed client network traffic out of the cluster. In most deployments, this should be an RFC1918 private subnet with an upstream router which can perform NAT translation and firewalling as required, both for the cluster nodes themselves, but also for the RFC1918 managed client networks.
-
-The floating IP address in the upstream network can be used as a single point of communication with the PVC cluster from other upstream sources, for instance to access the DNS aggregator instance or the API if configured. For this reason the network should generally be protected from unauthorized access via a firewall.
-
-Nodes in this network are generally assigned static IP addresses which are configured at node install time and in the [Ansible deployment configuration](/manuals/ansible).
-
-The upstream router should be able to handle static routes to the PVC cluster, or form a BGP neighbour relationship with the coordinator nodes and/or floating IP address to learn routes to the managed client networks.
-
-The upstream network should generally be large enough to contain:
-
-0. The upstream router(s)
-0. The nodes themselves
-0. In most deployments, the node IPMI management interfaces.
-
-For example, for a 3+ node cluster, up to about 90 nodes, the following configuration might be used:
-
-| Description | Address |
-|-------------|---------|
-| Upstream network | 10.0.0.0/24 |
-| Router VIP address | 10.0.0.1 |
-| Router 1 address | 10.0.0.2 |
-| Router 2 address | 10.0.0.3 |
-| PVC floating address | 10.0.0.10 |
-| node1 | 10.0.0.11 |
-| node2 | 10.0.0.12 |
-| etc.  | etc. |
-| node1-ipmi | 10.0.0.111 |
-| node2-ipmi | 10.0.0.112 |
-| etc.  | etc. |
-
-For even larger clusters, a `/23` or even larger network may be used.
-
-### Cluster: Connecting the nodes with each other
-
-The cluster network is an unrouted private network used by the PVC nodes to communicate with each other for database access, Libvirt migration, and storage client traffic. It is also used as the underlying interface for the BGP EVPN VXLAN interfaces used by managed client networks.
-
-The floating IP address in the cluster network can be used as a single point of communication with the primary node.
-
-Nodes in this network are generally assigned IPs automatically based on their node number (e.g. node1 at `.1`, node2 at `.2`, etc.). The network should be large enough to include all nodes sequentially.
-
-Generally the cluster network should be completely separate from the upstream network, either a separate physical interface (or set of bonded interfaces) or a dedicated vLAN on an underlying physical device.
-
-### Storage: Connecting Ceph OSD with each other
-
-The storage network is an unrouted private network used by the PVC node storage OSDs to communicated with each other, without using the main cluster network and introducing potentially large amounts of traffic there.
-
-Nodes in this network are generally assigned IPs automatically based on their node number. The network should be large enough to include all nodes sequentially.
-
-The administrator may choose to collocate the storage network on the same physical interface as the cluster network, or on a separate physical interface. This should be decided based on the size of the cluster and the perceived ratios of client network versus storage traffic. In large (>3 node) or storage-intensive clusters, this network should generally be a separate set of fast physical interfaces, separate from both the upstream and cluster networks, in order to maximize and isolate the storage bandwidth.
-
-### Bridged (unmanaged) Client Networks
-
-The first type of client network is the unmanaged bridged network. These networks have a separate vLAN on the device underlying the cluster network, which is created when the network is configured. VMs are then bridged into this vLAN.
-
-With this client network type, PVC does no management of the network. This is left entirely to the administrator. It requires switch support and the configuration of the vLANs on the switchports of each node's cluster network before enabling the network.
-
-### VXLAN (managed) Client Networks
-
-The second type of client network is the managed VXLAN network. These networks make use of BGP EVPN, managed by route reflection on the coordinators, to create virtual layer 2 Ethernet tunnels between all nodes in the cluster. VXLANs are then run on top of these virtual layer 2 tunnels, with the primary PVC node providing routing, DHCP, and DNS functionality to the network via a single IP address.
-
-With this client network type, PVC is in full control of the network. No vLAN configuration is required on the switchports of each node's cluster network as the virtual layer 2 tunnel travels over the cluster layer 3 network. All client network traffic destined for outside the network will exit via the upstream network of the primary coordinator node; note that this may introduce a bottleneck and tromboning if there is a large amount of external and/or inter-network traffic on the cluster. The administrator should consider this carefully when sizing the cluster network.
-
-Future PVC versions may support other client network types, such as direct-routing between VMs.
-
-## Node Layout: Considering how nodes are laid out
-
-A production-grade PVC cluster requires 3 nodes running the PVC Daemon software. 1-node clusters are supported for very small clusters, homelabs, and testing, but provide no redundancy; they should not be used in production situations.
-
-### Node Functions: Coordinators versus Hypervisors
-
-Within PVC, a given node can have one of two main functions: it can be a "Coordinator" or a "Hypervisor".
-
-#### Coordinators
-
-Coordinators are a special set of 3 or 5 nodes with additional functionality. The coordinator nodes run, in addition to the PVC software itself, a number of databases and additional functions which are required by the whole cluster. An odd number of coordinators is *always* required to maintain quorum, though there are diminishing returns when creating more than 3. These additional functions are:
-
-0. The Zookeeper database containing the cluster state and configuration
-0. The DNS aggregation Patroni PostgreSQL database containing DNS records for all client networks
-0. The FRR EBGP route reflectors and upstream BGP peers
-
-In addition to these functions, coordinators can usually also run all other PVC node functions.
-
-The set of coordinator nodes is generally configured at cluster bootstrap, initially with 3 nodes, which are then bootstrapped together to form a basic 3-node cluster. Additional nodes, either as coordinators or as hypervisors, can then be added to the running cluster to bring it up to its final size, either immediately or as the needs of the cluster change.
-
-##### The Primary Coordinator
-
-Within the set of coordinators, a single primary coordinator is elected and shuffles around the cluster as nodes start and stop. Which coordinator is primary can be selected by the administrator manually, or via a simple election process within the cluster. Once a node becomes primary, it will remain so until told not to be. This coordinator is responsible for some additional functionality in addition to the other coordinators. These additional functions are:
-
-0. The floating IPs in the main networks
-0. The default gateway IP for each managed client network
-0. The DNSMasq instance handling DHCP and DNS for each managed client network
-0. The API and provisioner clients and workers
-
-#### Hypervisors
-
-Hypervisors consist of all other PVC nodes in the cluster. For small clusters (3 nodes), there will generally not be any non-coordinator nodes, though adding a 4th would require it to be a hypervisor to preserve quorum between the coordinators. Larger clusters should generally add new nodes as Hypervisors rather than coordinators to preserve the small set of coordinator nodes previously mentioned.
-
-## Example Configurations
-
-This section provides diagrams of 3 possible node configurations, providing an idea of the sort of cluster topologies supported by PVC.
-
-#### Basic 3-node cluster
-
-![3-node cluster](/images/3-node-cluster.png)
-
-*Above: A diagram of a simple 3-node cluster; all nodes are coordinators, single 1Gbps network interface per node, collapsed cluster and storage networks*
-
-#### Mid-sized 8-node cluster with 3 coordinators
-
-![8-node cluster](/images/8-node-cluster.png)
-
-*Above: A diagram of a mid-sized 8-node cluster with 3 coordinators, dual bonded 10Gbps network interfaces per node*
-
-#### Large 17-node cluster with 5 coordinators
-
-![17-node cluster](/images/17-node-cluster.png)
-
-*Above: A diagram of a large 17-node cluster with 5 coordinators, dual bonded 10Gbps network interfaces per node for both cluster/upstream and storage networks*
-

docs/architecture/daemon.md

@@ -1,53 +0,0 @@
-# PVC Node Daemon architecture
-
-The PVC Node Daemon is the heart of the PVC system and runs on each node to manage the state of the node and its configured resources. The daemon connects directly to the Zookeeper cluster for coordination and state.
-
-The node daemon is build using Python 3.X and is packaged in the Debian package `pvc-daemon`.
-
-Configuration of the daemon is documented in [the manual](/manuals/daemon), however it is recommended to use the [Ansible configuration interface](/manuals/ansible) to configure the PVC system for you from scratch.
-
-## Overall architecture
-
-The PVC daemon is object-oriented - each cluster resource is represented by an Object, which is then present on each node in the cluster. This allows state changes to be reflected across the entire cluster should their data change.
-
-During startup, the system scans the Zookeeper database and sets up the required objects. The database is then watched in real-time for additional changes to the database information.
-
-## Startup sequence
-
-The daemon startup sequence is documented below. The main daemon entry-point is `Daemon.py` inside the `pvcd` folder, which is called from the `pvcd.py` stub file.
-
-0. The configuration is read from `/etc/pvc/pvcd.yaml` and the configuration object set up.
-
-0. Any required filesystem directories, mostly dynamic directories, are created.
-
-0. The logger is set up. If file logging is enabled, this is the state when the first log messages are written.
-
-0. Host networking is configured based on the `pvcd.yaml` configuration file. In a normal cluster, this is the point where the node will become reachable on the network as all networking is handled by the PVC node daemon.
-
-0. Sysctl tweaks are applied to the host system, to enable routing/forwarding between nodes via the host.
-
-0. The node determines its coordinator state and starts the required daemons if applicable. In a normal cluster, this is the point where the dependent services such as Zookeeper, FRR, and Ceph become available. After this step, the daemon waits 5 seconds before proceeding to give these daemons a chance to start up.
-
-0. The daemon connects to the Zookeeper cluster and starts its listener. If the Zookeeper cluster is unavailable, it will wait some time before abandoning the attempt and starting again from step 1.
-
-0. Termination handling/cleanup is configured.
-
-0. The node checks if it is already present in the Zookeeper cluster; if not, it will add itself to the database. Initial static options are also updated in the database here. The daemon state transitions from `stop` to `init`.
-
-0. The node checks if Libvirt is accessible.
-
-0. The node starts up the NFT firewall if applicable and configures the base rule-set.
-
-0. The node ensures that `dnsmasq` is stopped (legacy check, might be safe to remove eventually).
-
-0. The node begins setting up the object representations of resources, in order:
-
-    a. Node entries
-
-    b. Network entries, creating client networks and starting them as required.
-
-    c. Domain (VM) entries, starting up the VMs as required.
-
-    d. Ceph storage entries (OSDs, Pools, Volumes, Snapshots).
-
-0. The node activates its keepalived timer and begins sending keepalive updates to the cluster. The daemon state transitions from `init` to `run` and the system has started fully.

docs/cluster-architecture.md

@@ -0,0 +1,251 @@
+# PVC Cluster Architecture considerations
+
+- [PVC Cluster Architecture considerations](#pvc-cluster-architecture-considerations)
+  * [Node Specifications: Considering the size of nodes](#node-specifications--considering-the-size-of-nodes)
+  * [Storage Layout: Ceph and OSDs](#storage-layout--ceph-and-osds)
+  * [Physical network considerations](#physical-network-considerations)
+  * [Network Layout: Considering the required networks](#network-layout--considering-the-required-networks)
+    + [PVC system networks](#pvc-system-networks)
+        - [Upstream: Connecting the nodes to the wider world](#upstream--connecting-the-nodes-to-the-wider-world)
+        - [Cluster: Connecting the nodes with each other](#cluster--connecting-the-nodes-with-each-other)
+        - [Storage: Connecting Ceph OSD with each other](#storage--connecting-ceph-osd-with-each-other)
+    + [PVC client networks](#pvc-client-networks)
+        - [Bridged (unmanaged) Client Networks](#bridged--unmanaged--client-networks)
+        - [VXLAN (managed) Client Networks](#vxlan--managed--client-networks)
+        - [Other Client Networks](#other-client-networks)
+  * [Node Layout: Considering how nodes are laid out](#node-layout--considering-how-nodes-are-laid-out)
+    + [Node Functions: Coordinators versus Hypervisors](#node-functions--coordinators-versus-hypervisors)
+        - [Coordinators](#coordinators)
+            * [The Primary Coordinator](#the-primary-coordinator)
+        - [Hypervisors](#hypervisors)
+    + [Geographic redundancy](#geographic-redundancy)
+  * [Example Configurations](#example-configurations)
+    + [Basic 3-node cluster](#basic-3-node-cluster)
+    + [Mid-sized 8-node cluster with 3 coordinators](#mid-sized-8-node-cluster-with-3-coordinators)
+    + [Large 17-node cluster with 5 coordinators](#large-17-node-cluster-with-5-coordinators)
+
+This document contains considerations the administrator should make when preparing for and building a PVC cluster. It is important that prospective PVC administrators read this document *thoroughly* before deploying a cluster to ensure they understand the requirements, caveats, and important details about how PVC operates.
+
+## Node Specifications: Considering the size of nodes
+
+PVC nodes, especially coordinator nodes, run a significant number of software applications in addition to the virtual machines (VMs). It is therefore extremely important to size the systems correctly for the expected workload while planning both for redundancy and future capacity. In general, taller nodes are better for performance, providing a more powerful cluster on fewer physical machines, though each workload may be different in this regard.
+
+The following table provides bare-minimum, recommended, and optimal specifications for a cluster. The bare-minimum specification would be suitable for testing or a small lab, but not for production use. The recommended specification would be suitable for a small production cluster running lightweight VMs. The optimal cluster would be the ideal for running a demanding, resource-intensive production cluster. Note that these are the minimum resources required, and actual usage will likely require more resources than those presented here - this is mostly to show the minimums for each specified configuration (i.e. testing, light production, heavy production).
+
+| Resource | Minimum | Recommended | Optimal|
+|--------------|-----------|---------------|----------|
+| CPU generation | Intel Nehalem (2008) / AMD Bulldozer (2011) | Intel Sandy Bridge (2011) / AMD Naples (2017) | Intel Haswell (2013) / AMD Rome (2019) |
+| CPU cores (per node) | 4x @1.8GHz | 8x @2.0GHz | 12x @2.2 GHz |
+| RAM (per node) | 16GB | 48GB | 64GB |
+| System disk (SSD/HDD/USB/SD/eMMC) | 1x 10GB | 2x 10GB RAID-1 | 2x 32GB RAID-1 |
+| Data disk (SSD only) | 1x 200GB | 1x 400GB | 2x 400GB |
+| Network interfaces | 1x 1Gbps | 2x 1Gbps LAG | 2x 10Gbps LAG |
+| Total CPU cores (healthy) | 12x | 24x | 36x |
+| Total CPU cores (n-1) | 8x | 16x | 24x |
+| Total RAM (healthy) | 48GB | 144GB | 192GB |
+| Total RAM (n-1) | 32GB | 96GB | 128GB |
+| Total disk space | 200GB | 400GB | 800GB |
+
+Of these totals, some amount of CPU and RAM will be used by the storage subsystem and the PVC daemons themselves, meaning that the total available for virtual machines is slightly less. Generally, each OSD data disk will consume 1 vCPU at load and 1-2GB RAM, so nodes should be sized not only according to the VM workload, but the number of storage disks per node. Additionally the coordinator databases will use additional RAM and CPU resources of up to 1-4GB per node, though there is generally little need to spec coordinators any larger than non-coordinator nodes and the VM automatic node selection process will take used RAM into account by default.
+
+### System Disks
+
+The system disk(s) chosen are important to consider, especially for coordinators. Ideally, an SSD, or two SSDs in RAID-1/mirroring are recommended for system disks. This helps ensure optimal performance for the system (e.g. swap space) and PVC components such as databases as well as the Ceph caches.
+
+It is possible to run PVC on slower disks, for instance HDDs, USB drives, SD cards, or eMMC flash. For hypervisor-only nodes this will be acceptable; however for coordinators be advised that the performance of some aspects of the system may suffer as a result, and the longevity of the storage media must be carefully considered. RAID-1/mirroring is strongly recommended for these storage media as well, especially on coordinator nodes.
+
+### n-1 Redundancy
+
+Care should be taken to examine the "healthy" versus "n-1" total resource availability. Under normal operation, PVC will use all available resources and distribute VMs across all cluster nodes. However, during single-node failure or maintenance conditions, all VMs will be required to run on the remaining hypervisors. Thus, care should be taken during planning to ensure there is sufficient resources for the expected workload of the cluster.
+
+The general rule for available resource capacity planning can be though of as "1/3 of the total disks space, 2/3 of the total RAM, 2/3 of the total CPUs" for a 3-node cluster.
+
+For memory provisioning of VMs, PVC will warn the administrator, via a Degraded cluster state, if the "n-1" RAM quantity is exceeded by the total maximum allocation of all running VMs. This situation can be worked around with sufficient swap space on nodes to ensure there is overflow, however the warning cannot be overridden. If nodes are of mismatched sizes, the "n-1" RAM quantity is calculated by removing (one of) the largest node in the cluster and adding the remaining nodes' RAM counts together.
+
+### Operating System and Architecture
+
+As an underlying OS, only Debian 10 "Buster" is supported by PVC. This is the operating system installed by the PVC [node installer](https://github.com/parallelvirtualcluster/pvc-installer) and expected by the PVC [Ansible configuration system](https://github.com/parallelvirtualcluster/pvc-ansible). Ubuntu or other Debian-derived distributions may work, but are not officially supported. PVC also makes use of a custom repository to provide the PVC software and an updated version of Ceph beyond what is available in the base operating system, and this is only compatible officially with Debian 10 "Buster".
+
+Currently, only the `amd64` (Intel 64 or AMD64) architecture is officially supported by PVC. Given the cross-platform nature of Python and the various software components in Debian, it may work on `armhf` or `arm64` systems as well, however this has not been tested by the author.
+
+## Storage Layout: Ceph and OSDs
+
+The Ceph subsystem of PVC, if enabled, creates a "hyperconverged" cluster whereby storage and VM hypervisor functions are collocated onto the same physical servers. The performance of the storage must be taken into account when sizing the nodes as mentioned above.
+
+The Ceph system is laid out similar to the other daemons. The Ceph Monitor and Manager functions are delegated to the Coordinators over the storage network, with all nodes connecting to these hosts to obtain the CRUSH maps and select OSD disks. OSDs are then distributed on all hosts, including non-coordinator hypervisors, and communicate with clients and each other over the storage network.
+
+Disks must be balanced across all nodes. Therefore, adding 1 disk to 1 node is not sufficient; 1 disk must be added to all nodes at the same time for the available space to increase. Ideally, disk sizes should also be identical across all storage disks, though the weight of each disk can be configured when added to the cluster. Generally speaking, fewer larger disks are preferable to many smaller disks to minimize storage resource utilization, however slightly more storage performance can be gained from using many small disks; the administrator should therefore always aim to choose the biggest disks they can and grow by adding more identical disks as space or performance needs grow.
+
+PVC Ceph pools make use of the replication mechanism of Ceph to store multiple copies of each object, thus ensuring that data is always available even when a host is unavailable. Only "replica"-based Ceph redundancy is supported by PVC; erasure coded pools are not supported due to major performance impacts related to rewrites and random I/O.
+
+The default replication level for a new pool is `copies=3, mincopies=2`. This will store 3 copies of each object, with a host-level failure domain, and will allow I/O as long as 2 copies are available. Thus, in a cluster of any size, all data is fully available even if a single host becomes unavailable. It will however use 3x the space for each piece of data stored, which must be considered when sizing the disk space for the cluster: a pool in this configuration, running on 3 nodes each with a single 400GB disk, will effectively have 400GB of total space available for use. As mentioned above, new disks must also be added in groups across nodes equal to the total number of `copies` to ensure new space is usable.
+
+Non-default values can also be set at pool creation time. For instance, one could create a `copies=3, mincopies=1` pool, which would allow I/O with two hosts down but leaves the cluster susceptible to a write hole should a disk fail in this state. Alternatively, for more resilience, one could create a `copies=4, mincopies=3` pool, which will allow 2 hosts to fail without a write hole, but would consume 4x the space for each piece of data stored and require new disks to be added in groups of 4 instead. Practically any combination of values is possible, however these 3 are the most relevant for most use-cases, and for most, especially small, clusters, the default is sufficient to provide solid redundancy and guard against host failures until the administrator can respond.
+
+Replication levels cannot be changed within PVC once a pool is created, however they can be changed via manual Ceph commands on a coordinator should the administrator require this. In any case, the administrator should carefully consider sizing, failure domains, and performance when selecting storage devices to ensure the right level of resiliency versus data usage for their use-case and cluster size.
+
+## Physical network considerations
+
+At a minimum, a production PVC cluster should use at least two 1Gbps Ethernet interfaces, connected in an LACP or active-backup bond on one or more switches. On top of this bond, the various cluster networks are configured as 802.3q vLANs. PVC is be able to support configurations without 802.1q vLAN support using multiple physical interfaces and no bridged client networks, but this is strongly discouraged due to the added complexity this introduces; the switches chosen for the cluster should include these requirements as a minimum.
+
+More advanced physical network layouts are also possible. For instance, one could have two isolated networks. On the first network, each node has two 10Gbps Ethernet interfaces, which are combined in a bond across two redundant switch fabrics and that handle the upstream and cluster networks. On the second network, each node has an additional two 10Gbps, which are also combined in a bond across the redundant switch fabrics and handle the storage network. This configuration could support up to 10Gbps of aggregate client traffic while also supporting 10Gbps of aggregate storage traffic. Even more complex network configurations are possible if the cluster requires such performance. See the [Example Configurations](#example-configurations) section for some examples.
+
+Only Ethernet networks are supported by PVC. More exotic interconnects such as Infiniband are not supported by default, and must be manually set up with Ethernet (e.g. EoIB) layers on top to be usable with PVC.
+
+PVC manages the IP addressing of all nodes itself and creates the required addresses during node daemon startup; thus, the on-boot network configuration of each interface should be set to "manual" with no IP addresses configured.
+
+## Network Layout: Considering the required networks
+
+A PVC cluster needs several different networks to operate properly; they are described in detail below and the administrator should ensure they account for all the required networks when planning the cluster.
+
+### PVC system networks
+
+#### Upstream: Connecting the nodes to the wider world
+
+The upstream network functions as the main upstream for the cluster nodes, providing Internet access and a way to route managed client network traffic out of the cluster. In most deployments, this should be an RFC1918 private subnet with an upstream router which can perform NAT translation and firewalling as required, both for the cluster nodes themselves, and also for any RFC1918 managed client networks.
+
+The floating IP address in the cluster network can be used as a single point of communication with the active primary node, for instance to access the DNS aggregator instance or the management API. PVC provides only limited access control mechanisms to the API interface, so the upstream network should always be protected by a firewall; running PVC directly accessible on the Internet is strongly discouraged and may post a serious security risk, and all access should be restricted to the smallest possible set of remote systems.
+
+Nodes in this network are generally assigned static IP addresses which are configured at node install time and in the [Ansible deployment configuration](/manuals/ansible).
+
+The upstream router should be able to handle static routes to the PVC cluster, or form a BGP neighbour relationship with the coordinator nodes and/or floating IP address to learn routes to the managed client networks.
+
+The upstream network should generally be large enough to contain:
+
+0. The upstream router(s)
+0. The nodes themselves
+0. In most deployments, the node IPMI management interfaces.
+
+For example, for a 3+ node cluster, up to about 90 nodes, the following configuration might be used:
+
+| Description | Address |
+|-------------|---------|
+| Upstream network | 10.0.0.0/24 |
+| Router VIP address | 10.0.0.1 |
+| Router 1 address | 10.0.0.2 |
+| Router 2 address | 10.0.0.3 |
+| PVC floating address | 10.0.0.10 |
+| node1 | 10.0.0.11 |
+| node2 | 10.0.0.12 |
+| etc.  | etc. |
+| node1-ipmi | 10.0.0.111 |
+| node2-ipmi | 10.0.0.112 |
+| etc.  | etc. |
+
+For even larger clusters, a `/23` or even larger network may be used.
+
+#### Cluster: Connecting the nodes with each other
+
+The cluster network is an unrouted private network used by the PVC nodes to communicate with each other for database access and Libvirt migrations. It is also used as the underlying interface for the BGP EVPN VXLAN interfaces used by managed client networks.
+
+The floating IP address in the cluster network can be used as a single point of communication with the active primary node.
+
+Nodes in this network are generally assigned IPs automatically based on their node number (e.g. node1 at `.1`, node2 at `.2`, etc.). The network should be large enough to include all nodes sequentially.
+
+Generally the cluster network should be completely separate from the upstream network, either a separate physical interface (or set of bonded interfaces) or a dedicated vLAN on an underlying physical device, but they can be collocated if required.
+
+#### Storage: Connecting Ceph daemons with each other and with OSDs
+
+The storage network is an unrouted private network used by the PVC node storage OSDs to communicated with each other, for Ceph management functionality, and for QEMU-to-Ceph disk access, without using the main cluster network and introducing potentially large amounts of traffic there.
+
+The floating IP address in the storage network can be used as a single point of communication with the active primary node, though this will generally be of little use.
+
+Nodes in this network are generally assigned IPs automatically based on their node number (e.g. node1 at `.1`, node2 at `.2`, etc.). The network should be large enough to include all nodes sequentially.
+
+The administrator may choose to collocate the storage network on the same physical interface as the cluster network, or on a separate physical interface. This should be decided based on the size of the cluster and the perceived ratios of client network versus storage traffic. In large (>3 node) or storage-intensive clusters, this network should generally be a separate set of fast physical interfaces, separate from both the upstream and cluster networks, in order to maximize and isolate the storage bandwidth. If the administrator does choose to collocate these networks, they may also share the same IP address, thus eliminating any distinction between the Cluster and Storage networks. The PVC software handles this natively when the Cluster and Storage IPs of a node are identical.
+
+### PVC client networks
+
+#### Bridged (unmanaged) Client Networks
+
+The first type of client network is the unmanaged bridged network. These networks have a separate vLAN on the device underlying the other networks, which is created when the network is configured. VMs are then bridged into this vLAN.
+
+With this client network type, PVC does no management of the network. This is left entirely to the administrator. It requires switch support and the configuration of the vLANs on the switchports of each node's physical interfaces before enabling the network.
+
+Generally, the same physical network interface will underlay both the cluster networks as well as bridged client networks. PVC does however support specifying a separate physical device for bridged client networks, for instance to separate these networks onto a different physical interface from the main cluster networks.
+
+#### VXLAN (managed) Client Networks
+
+The second type of client network is the managed VXLAN network. These networks make use of BGP EVPN, managed by route reflection on the coordinators, to create virtual layer 2 Ethernet tunnels between all nodes in the cluster. VXLANs are then run on top of these virtual layer 2 tunnels, with the active primary PVC node providing routing, DHCP, and DNS functionality to the network via a single IP address.
+
+With this client network type, PVC is in full control of the network. No vLAN configuration is required on the switchports of each node's physical interfaces, as the virtual layer 2 tunnel travels over the cluster layer 3 network. All client network traffic destined for outside the network will exit via the upstream network interface of the active primary coordinator node.
+
+NOTE: These networks may introduce a bottleneck and tromboning if there is a large amount of external and/or inter-network traffic on the cluster. The administrator should consider this carefully when deciding whether to use managed or bridged networks and properly evaluate the inter-network traffic requirements.
+
+#### Other Client Networks
+
+Future PVC versions may support other client network types, such as direct-routing between VMs.
+
+## Node Layout: Considering how nodes are laid out
+
+A production-grade PVC cluster requires at least 3 nodes running the PVC Daemon software. 1-node clusters are supported for very small clusters, home labs, and testing, but provide no redundancy; they should not be used in production situations.
+
+### Node Functions: Coordinators versus Hypervisors
+
+Within PVC, a given node can have one of two main functions: "Coordinator" or "Hypervisor".
+
+#### Coordinators
+
+Coordinators are a special set of 3 or 5 nodes with additional functionality. The coordinator nodes run, in addition to the PVC software itself, a number of databases and additional functions which are required by the whole cluster. An odd number of coordinators is *always* required to maintain quorum, though there are diminishing returns when creating more than 3. These additional functions are:
+
+0. The Zookeeper database cluster containing the cluster state and configuration
+0. The Patroni PostgreSQL database cluster containing DNS records for managed networks and provisioning configurations
+0. The FRR EBGP route reflectors and upstream BGP peers
+
+In addition to these functions, coordinators can usually also run all other PVC node functions.
+
+The set of coordinator nodes is generally configured at cluster bootstrap, initially with 3 nodes, which are then bootstrapped together to form a basic 3-node cluster. Additional nodes, either as coordinators or as hypervisors, can then be added to the running cluster to bring it up to its final size, either immediately or as the needs of the cluster change.
+
+##### The Primary Coordinator
+
+Within the set of coordinators, a single primary coordinator is elected at cluster startup and as nodes start and stop, or in response to administrative commands. Once a node becomes primary, it will remain so until it stops or is told not to be. This coordinator is responsible for some additional functionality in addition to the other coordinators. These additional functions are:
+
+0. The floating IPs in the main networks
+0. The default gateway IP for each managed client network
+0. The DNSMasq instance handling DHCP and DNS for each managed client network
+0. The API and provisioner clients and workers
+
+PVC gracefully handles transitioning primary coordinator state, to minimize downtime. Workers will continue to operate on the old coordinator if available after a switchover and the administrator should be aware of any active tasks before switching the active primary coordinator.
+
+#### Hypervisors
+
+Hypervisors consist of all other PVC nodes in the cluster. For small clusters (3 nodes), there will generally not be any non-coordinator nodes, though adding a 4th would require it to be a hypervisor to preserve quorum between the coordinators. Larger clusters should generally add new nodes as Hypervisors rather than coordinators to preserve the small set of coordinator nodes previously mentioned.
+
+### Geographic redundancy
+
+PVC supports geographic redundancy of nodes in order to facilitate disaster recovery scenarios when uptime is critical. Functionally, PVC behaves the same regardless of whether the 3 or more coordinators are in the same physical location, or remote physical locations.
+
+When using geographic redundancy, there are several caveats to keep in mind:
+
+* The Ceph storage subsystem is latency-sensitive. With the default replication configuration, at least 2 writes must succeed for the write to return a success, so the total write latency of a write on any system will be equal to the maximum latency between any two nodes. It is recommended to keep all PVC nodes as "close" as possible latency-wise or storage performance may suffer.
+
+* The inter-node PVC networks must be layer-2 networks (broadcast domains). These networks must be spanned to all nodes in all locations.
+
+* The number of sites and positioning of coordinators at those sites is important. A majority (at least 2 in a 3-coordinator cluster, or 3 in a 5-coordinator) of coordinators must be able to reach each other in a failure scenario for the cluster as a whole to remain functional. Thus, configurations such as 2 + 1 or 3 + 2 splits across 2 sites do *not* provide full redundancy, and the whole cluster will be down if the majority site is down. It is thus recommended to always have an odd number of sites to match the odd number of coordinators, for instance a 1 + 1 + 1 or 2 + 2 + 1 configuration. Also note that all hypervisors much be able to reach the majority coordinator group or their storage will be impacted as well.
+
+* Even if the PVC software itself is in an unmanageable state, VMs will continue to run if at all possible. However, since the storage subsystem makes use of the same quorum, losing more than half of the nodes will very likely result in storage interruption as well, which will affect running VMs.
+
+If these requirements cannot be fulfilled, it may be best to have separate PVC clusters at each site and handle service redundancy at a higher layer to avoid a major disruption.
+
+## Example Configurations
+
+This section provides diagrams of 3 possible node configurations. These diagrams can be extrapolated out to almost any possible configuration and number of nodes.
+
+#### Basic 3-node cluster
+
+![3-node cluster](/images/3-node-cluster.png)
+
+*Above: A diagram of a simple 3-node cluster; all nodes are coordinators, single 1Gbps network interface per node, collapsed cluster and storage networks*
+
+#### Mid-sized 8-node cluster with 3 coordinators
+
+![8-node cluster](/images/8-node-cluster.png)
+
+*Above: A diagram of a mid-sized 8-node cluster with 3 coordinators, dual bonded 10Gbps network interfaces per node*
+
+#### Large 17-node cluster with 5 coordinators
+
+![17-node cluster](/images/17-node-cluster.png)
+
+*Above: A diagram of a large 17-node cluster with 5 coordinators, dual bonded 10Gbps network interfaces per node for both cluster/upstream and storage networks*
+

docs/getting-started.md

@@ -6,6 +6,8 @@ This guide will walk you through setting up a simple 3-node PVC cluster from scr
 
 ### Part One - Preparing for bootstrap
 
+0. Read through the [Cluster Architecture documentation](/architecture/cluster). This documentation details the requirements and conventions of a PVC cluster, and is important to understand before proceeding.
+
 0. Download the latest copy of the [`pvc-installer`](https://github.com/parallelvirtualcluster/pvc-installer) and [`pvc-ansible`](https://github.com/parallelvirtualcluster/pvc-ansible) repositories to your local machine.
 
 0. In `pvc-ansible`, create an initial `hosts` inventory, using `hosts.default` as a template. You can manage multiple PVC clusters ("sites") from the Ansible repository easily, however for simplicity you can use the simple name `cluster` for your initial site. Define the 3 hostnames you will use under the site group; usually the provided names of `pvchv1`, `pvchv2`, and `pvchv3` are sufficient, though you may use any hostname pattern you wish. It is *very important* that the names all contain a sequential number, however, as this is used by various components.
@@ -53,19 +55,20 @@ This guide will walk you through setting up a simple 3-node PVC cluster from scr
 0. Perform the initial bootstrap. From the `pvc-ansible` repository directory, execute the following `ansible-playbook` command, replacing `<cluster_name>` with the Ansible group name from the `hosts` file. Make special note of the additional `bootstrap=yes` variable, which tells the playbook that this is an initial bootstrap run.  
     `$ ansible-playbook -v -i hosts pvc.yml -l <cluster_name> -e bootstrap=yes`
 
+    **WARNING:** Never rerun this playbook with the `-e bootstrap=yes` option against an active cluster. This will have unintended, disastrous consequences.
+
 0. Wait for the Ansible playbook run to finish. Once completed, the cluster bootstrap will be finished, and all 3 nodes will have rebooted into a working PVC cluster.
 
-0. Install the CLI client on your administrative host, and verify connectivity to the cluster, for instance by running the following command, which should show all 3 nodes as present and running:  
-    `$ pvc -z pvchv1:2181,pvchv2:2181,pvchv3:2181 node list`
+0. Install the CLI client on your administrative host, and add and verify connectivity to the cluster; this will also verify that the API is working. You will need to know the cluster upstream floating IP address here, and if you configured SSL or authentication for the API in your `group_vars`, adjust the first command as needed (see `pvc cluster add -h` for details).  
+    `$ pvc cluster add -a <upstream_floating_ip> mycluster`  
+    `$ pvc -c mycluster node list`
 
-0. Optionally, verify the API is listening on the `upstream_floating_ip` address configured in the cluster `group_vars`, for instance by running the following command which shows, in JSON format, the same information as in the previous step:  
-    `$ curl -X GET http://<upstream_floating_ip>:7370/api/v1`
+    We can also set a default cluster by exporting the `PVC_CLUSTER` environment variable to avoid requiring `-c cluster` with every subsequent command:  
+    `$ export PVC_CLUSTER="mycluster"`
 
 ### Part Four - Configuring the Ceph storage cluster
 
-All steps in this and following sections can be performed using either the CLI client or the HTTP API; for clarity, only the CLI commands are shown.
-
-0. Determine the Ceph OSD block devices on each host, via an `ssh` shell. For instance, check `/dev/disk/by-path` to show the block devices by their physical SAS/SATA bus location, and obtain the relevant `/dev/sdX` name for each disk you wish to be a Ceph OSD on each host.
+0. Determine the Ceph OSD block devices on each host, via an `ssh` shell. For instance, use `lsblk` or check `/dev/disk/by-path` to show the block devices by their physical SAS/SATA bus location, and obtain the relevant `/dev/sdX` name for each disk you wish to be a Ceph OSD on each host.
 
 0. Add each OSD device to each host. The general command is:  
     `$ pvc storage osd add --weight <weight> <node> <device>`
@@ -78,9 +81,11 @@ All steps in this and following sections can be performed using either the CLI c
     `$ pvc storage osd add --weight 1.0 pvchv3 /dev/sdb`  
     `$ pvc storage osd add --weight 1.0 pvchv3 /dev/sdc`   
 
-    **NOTE:** On the CLI, the `--weight` argument is optional, and defaults to `1.0`. In the API, it must be specified explicitly. OSD weights determine the relative amount of data which can fit onto each OSD. Under normal circumstances, you would want all OSDs to be of identical size, and hence all should have the same weight. If your OSDs are instead different sizes, the weight should be proportional to the size, e.g. `1.0` for a 100GB disk, `2.0` for a 200GB disk, etc. For more details, see the Ceph documentation.
+    **NOTE:** On the CLI, the `--weight` argument is optional, and defaults to `1.0`. In the API, it must be specified explicitly, but the CLI sets a default value. OSD weights determine the relative amount of data which can fit onto each OSD. Under normal circumstances, you would want all OSDs to be of identical size, and hence all should have the same weight. If your OSDs are instead different sizes, the weight should be proportional to the size, e.g. `1.0` for a 100GB disk, `2.0` for a 200GB disk, etc. For more details, see the Ceph documentation.
 
-    **NOTE:** OSD commands wait for the action to complete on the node, and can take some time (up to 30s normally). Be cautious of HTTP timeouts when using the API to perform these steps.
+    **NOTE:** OSD commands wait for the action to complete on the node, and can take some time.
+
+    **NOTE:** You can add OSDs in any order you wish, for instance you can add the first OSD to each node and then add the second to each node, or you can add all nodes' OSDs together at once like the example. This ordering does not affect the cluster in any way.
 
 0. Verify that the OSDs were added and are functional (`up` and `in`):  
     `$ pvc storage osd list`
@@ -91,19 +96,18 @@ All steps in this and following sections can be performed using either the CLI c
     For example, to create a pool named `vms` with 256 placement groups (a good default with 6 OSD disks), run the command as follows:  
     `$ pvc storage pool add vms 256`
 
-    **NOTE:** Ceph placement groups are a complex topic; as a general rule it's easier to grow than shrink, so start small and grow as your cluster grows. For more details see the Ceph documentation and the [placement group calculator](https://ceph.com/pgcalc/).
+    **NOTE:** Ceph placement groups are a complex topic; as a general rule it's easier to grow than shrink, so start small and grow as your cluster grows. The general formula is to calculate the ideal number of PGs is `pgs * maxcopies / osds = ~250`, then round `pgs` down to the closest power of 2; generally, you want as close to 250 PGs per OSD as possible, but no more than 250. With 3-6 OSDs, 256 is a good number, and with 9+ OSDs, 512 is a good number. Ceph will error if the total number exceeds the limit. For more details see the Ceph documentation and the [placement group calculator](https://ceph.com/pgcalc/).
 
-    **NOTE:** All PVC RBD pools use `copies=3` and `mincopies=2` for data storage. This provides, for each object, 3 copies of the data, with writes being accepted with 1 degraded copy. This provides maximum resiliency against single-node outages, but will use 3x the amount of storage for each unit stored inside the image. Take this into account when sizing OSD disks and VM images. This cannot be changed as any less storage will result in a non-HA cluster that could not handle a single node failure.
+    **NOTE:** As detailed in the [cluster architecture documentation](/cluster-architecture), you can also set a custom replica configuration for each pool if the default of 3 replica copies with 2 minimum copies is not acceptable. See `pvc storage pool add -h` or that document for full details.
 
 0. Verify that the pool was added:  
     `$ pvc storage pool list`
 
 ### Part Five - Creating virtual networks
 
-0. Determine a domain name, IPv4, and/or IPv6 network for your first client network, and any other client networks you may wish to create. For this guide we will create a single "managed" virtual client network with DHCP.
+0. Determine a domain name and IPv4, and/or IPv6 network for your first client network, and any other client networks you may wish to create. These networks should never overlap with the cluster networks. For full details on the client network types, see the [cluster architecture documentation](/cluster-architecture).
 
-0. Create the virtual network. The general command for an IPv4-only network with DHCP is:  
-    `$ pvc network add <vni_id> --type <type> --description <space-less_description> --domain <domain> --ipnet <ipv4_network_in_CIDR> --gateway <ipv4_gateway_address> --dhcp --dhcp-start <first_address> --dhcp-end <last_address>`
+0. Create the virtual network. There are many options here, so see `pvc network add -h` for details.  
 
     For example, to create the managed (EVPN VXLAN) network `100` with subnet `10.100.0.0/24`,  gateway `.1` and DHCP from `.100` to `.199`, run the command as follows:  
     `$ pvc network add 100 --type managed --description my-managed-network --domain myhosts.local --ipnet 10.100.0.0/24 --gateway 10.100.0.1 --dhcp --dhcp-start 10.100.0.100 --dhcp-end 10.100.0.199`
@@ -111,135 +115,27 @@ All steps in this and following sections can be performed using either the CLI c
     For another example, to create the static bridged (switch-configured, tagged VLAN, with no PVC management of IPs) network `200`, run the command as follows:  
     `$ pvc network add 200 --type bridged --description my-bridged-network`
 
+    **NOTE:** Network descriptions cannot contain spaces or special characters; keep them short, sweet, and dash or underscore delimited.
+
 0. Verify that the network(s) were added:  
     `$ pvc network list`
 
 0. On the upstream router, configure one of:
 
-    a) A BGP neighbour relationship with the `upstream_floating_address` to automatically learn routes.
+    a) A BGP neighbour relationship with the cluster upstream floating address to automatically learn routes.
 
-    b) Static routes for the configured client IP networks towards the `upstream_floating_address`.
+    b) Static routes for the configured client IP networks towards the cluster upstream floating address.
 
 0. On the upstream router, if required, configure NAT for the configured client IP networks.
 
 0. Verify the client networks are reachable by pinging the managed gateway from outside the cluster.
 
-### Part Six - Setting nodes ready and deploying a VM
 
-This section walks through deploying a simple Debian VM to the cluster with Debootstrap. Note that as of PVC version `0.5`, this is still a manual process, though automated deployment of VMs based on configuration templates and image snapshots is planned for version `0.6`. This section can be used as a basis for a scripted installer, or a manual process as the administrator sees fit.
+### You're Done!
 
 0. Set all 3 nodes to `ready` state, allowing them to run virtual machines. The general command is:  
     `$ pvc node ready <node>`
 
-0. Create an RBD image for the VM. The general command is:  
-    `$ pvc storage volume add <pool> <name> <size>`
+Congratulations, you now have a basic PVC storage cluster, ready to run your VMs.
 
-    For example, to create a 20GB disk for a VM called `test1` in the previously-configured pool `vms`, run the command as follows:  
-    `$ pvc storage volume add vms test1_disk0 20G`
-
-0. Verify the RBD image was created:  
-    `$ pvc storage volume list`
-
-0. On one of the PVC nodes, for example `pvchv1`, map the RBD volume to the local system:  
-    `$ ceph rbd map vms/test1_disk0`
-
-    The resulting disk device will be available at `/dev/rbd/vms/test1_disk0` or `/dev/rbd0`.
-
-0. Create a filesystem on the block device, for example `ext4`:  
-    `$ mkfs -t ext4 /dev/rbd/vms/test1_disk0`
-
-0. Create a temporary directory and mount the block device to it, using `mount` to find the directory:  
-    `$ mount /dev/rbd/vms/test1_disk0 $( mktemp -d )`  
-    `$ mount | grep rbd`
-
-0. Run a `debootstrap` installation to the volume:  
-    `$ debootstrap buster <temporary_mountpoint> http://ftp.mirror.debian.org/debian`
-
-0. Bind mount the various required directories to the new system:  
-    `$ mount --bind /dev <temporary_mountpoint>/dev`  
-    `$ mount --bind /dev/pts <temporary_mountpoint>/dev/pts`  
-    `$ mount --bind /proc <temporary_mountpoint>/proc`  
-    `$ mount --bind /sys <temporary_mountpoint>/sys`  
-    `$ mount --bind /run <temporary_mountpoint>/run`  
-
-0. Using `chroot`, configure the VM system as required, for instance installing packages or adding users:  
-    `$ chroot <temporary_mountpoint>`  
-    `[chroot]$ ...`
-
-0. Install the GRUB bootloader in the VM system, and install Grub to the RBD device:  
-    `[chroot]$ apt install grub-pc`  
-    `[chroot]$ grub-install /dev/rbd/vms/test1_disk0`
-
-0. Exit the `chroot` environment, unmount the temporary mountpoint, and unmap the RBD device:  
-    `[chroot]$ exit`  
-    `$ umount <temporary_mountpoint>`  
-    `$ rbd unmap /dev/rd0`
-
-0. Prepare a Libvirt XML configuration, obtaining the required Ceph storage secret and a new random VM UUID first. This example provides a very simple VM with 1 vCPU, 1GB RAM, the previously-configured network `100`, and the previously-configured disk `vms/test1_disk0`:  
-    `$ virsh secret-list`  
-    `$ uuidgen`  
-    `$ $EDITOR /tmp/test1.xml`
-
-    ```
-    <domain type='kvm'>
-      <name>test1</name>
-      <uuid>[INSERT GENERATED UUID]</uuid>
-      <description>Testing VM</description>
-      <memory unit='MiB'>1024</memory>
-      <vcpu>1</vcpu>
-      <os>
-        <type arch='x86_64' machine='pc-i440fx-2.7'>hvm</type>
-        <boot dev='hd'/>
-      </os>
-      <features>
-        <acpi/>
-        <apic/>
-        <pae/>
-      </features>
-      <clock offset='utc'/>
-      <on_poweroff>destroy</on_poweroff>
-      <on_reboot>restart</on_reboot>
-      <on_crash>restart</on_crash>
-      <devices>
-        <emulator>/usr/bin/kvm</emulator>
-        <controller type='usb' index='0'/>
-        <controller type='pci' index='0' model='pci-root'/>
-        <serial type='pty'/>
-        <console type='pty'/>
-        <disk type='network' device='disk'>
-          <driver name='qemu' discard='unmap'/>
-          <auth username='libvirt'>
-             <secret type='ceph' uuid='[INSERT CEPH STORAGE SECRET]'/>
-          </auth>
-          <source protocol='rbd' name='vms/test1_disk0'>
-            <host name='[INSERT FIRST COORDINATOR CLUSTER NETWORK FQDN' port='6789'/>
-            <host name='[INSERT FIRST COORDINATOR CLUSTER NETWORK FQDN' port='6789'/>
-            <host name='[INSERT FIRST COORDINATOR CLUSTER NETWORK FQDN' port='6789'/>
-          </source>
-          <target dev='sda' bus='scsi'/>
-        </disk>
-        <interface type='bridge'>
-          <mac address='52:54:00:12:34:56'/>
-          <source bridge='vmbr100'/>
-          <model type='virtio'/>
-        </interface>
-        <controller type='scsi' index='0' model='virtio-scsi'/>
-      </devices>
-    </domain>
-    ```
-
-    **NOTE:** This Libvirt XML is only a sample; it should be modified to fit the specifics of the VM. Alternatively to manual configuration, one can use a tool like `virt-manager` to generate valid Libvirt XML configurations for PVC to use.
-
-0. Define the VM in the PVC cluster:  
-    `$ pvc vm define /tmp/test1.xml`
-
-0. Verify the VM is present in the cluster:  
-    `$ pvc vm info test1`
-
-0. Start the VM and watch the console log:  
-    `$ pvc vm start test1`  
-    `$ pvc vm log -f test1`
-
-If all has gone well until this point, you should now be able to watch your new VM boot on the cluster, grab DHCP from the managed network, and run away doing its thing. You could now, for instance, move it permanently to another node with the `pvc vm move -t <node> test1` command, or temporarily with the `pvc vm migrate -t <node> test1` command and back again with the `pvc vm unmigrate test` command.
-
-For more details on what to do next, see the [CLI manual](/manuals/cli) for a full list of management functions, SSH into your new VM, and start provisioning more. Your new private cloud is now here!
+For next steps, see the [Provisioner manual](/manuals/provisioner) for details on how to use the PVC provisioner to create new Virtual Machines, as well as the [CLI manual](/manuals/cli) and [API manual](/manuals/api) for details on day-to-day usage of PVC.

docs/index.md

@@ -1,29 +1,68 @@
-# PVC - The Parallel Virtual Cluster suite
+# PVC - The Parallel Virtual Cluster system
 
 <p align="center">
 <img alt="Logo banner" src="https://git.bonifacelabs.ca/uploads/-/system/project/avatar/135/pvc_logo.png"/>
 <br/><br/>
 <a href="https://github.com/parallelvirtualcluster/pvc"><img alt="License" src="https://img.shields.io/github/license/parallelvirtualcluster/pvc"/></a>
 <a href="https://github.com/parallelvirtualcluster/pvc/releases"><img alt="Release" src="https://img.shields.io/github/release-pre/parallelvirtualcluster/pvc"/></a>
-<a href="https://git.bonifacelabs.ca/parallelvirtualcluster/pvc/pipelines"><img alt="Pipeline Status" src="https://git.bonifacelabs.ca/parallelvirtualcluster/pvc/badges/master/pipeline.svg"/></a>
 <a href="https://parallelvirtualcluster.readthedocs.io/en/latest/?badge=latest"><img alt="Documentation Status" src="https://readthedocs.org/projects/parallelvirtualcluster/badge/?version=latest"/></a>
 </p>
 
-PVC is a suite of Python 3 tools to manage virtualized clusters. It provides a fully-functional private cloud based on four key principles:
+PVC is a KVM+Ceph+Zookeeper-based, Free Software, scalable, redundant, self-healing, and self-managing private cloud solution designed with administrator simplicity in mind. It is built from the ground-up to be redundant at the host layer, allowing the cluster to gracefully handle the loss of nodes or their components, both due to hardware failure or due to maintenance. It is able to scale from a minimum of 3 nodes up to 12 or more nodes, while retaining performance and flexibility, allowing the administrator to build a small cluster today and grow it as needed.
 
-1. Be Free Software Forever (or Bust)
-2. Be Opinionated and Efficient and Pick The Best Software
-3. Be Scalable and Redundant but Not Hyperscale
-4. Be Simple To Use, Configure, and Maintain
+The major goal of PVC is to be administrator friendly, providing the power of Enterprise-grade private clouds like OpenStack, Nutanix, and VMWare to homelabbers, SMBs, and small ISPs, without the cost or complexity. It believes in picking the best tool for a job and abstracting it behind the cluster as a whole, freeing the administrator from the boring and time-consuming task of selecting the best component, and letting them get on with the things that really matter. Administration can be done from a simple CLI or via a RESTful API capable of building full-featured web frontends or additional applications, taking a self-documenting approach to keep the administrator learning curvet as low as possible. Setup is easy and straightforward with an [ISO-based node installer](https://github.com/parallelvirtualcluster/pvc-installer) and [Ansible role framework](https://github.com/parallelvirtualcluster/pvc-ansible) designed to get a cluster up and running as quickly as possible. Build your cloud in an hour, grow it as you need, and never worry about it: just add physical servers.
 
-It is designed to be an administrator-friendly but extremely powerful and rich modern private cloud system, but without the feature bloat and complexity of tools like OpenStack. With PVC, an administrator can provision, manage, and update a cluster of dozens or more hypervisors running thousands of VMs using a simple CLI tool, HTTP API, or web interface. PVC is based entirely on Debian GNU/Linux and Free-and-Open-Source tools, providing the glue to bootstrap, provision and manage the cluster, then getting out of the administrators' way.
+## Getting Started
 
-Your cloud, the best way; just add physical servers.
-
-To get started with PVC, read the [Cluster Architecture document](/architecture/cluster), then see [Installing](/installing) for details on setting up a set of PVC nodes, using [`pvc-ansible`](/manuals/ansible) to configure and bootstrap a cluster, and managing it with the [`pvc` cli](/manuals/cli) or [HTTP API](/manuals/api). For details on the project, its motivation, and architectural details, see [the About page](/about).
+To get started with PVC, please see the [About](https://parallelvirtualcluster.readthedocs.io/en/latest/about/) page for general information about the project, and the [Getting Started](https://parallelvirtualcluster.readthedocs.io/en/latest/getting-started/) page for details on configuring your cluster.
 
 ## Changelog
 
+#### v0.9.6
+
+  * Fixes bug with migrations
+
+#### v0.9.5
+
+  * Fixes bug with line count in log follow
+  * Fixes bug with disk stat output being None
+  * Adds short pretty health output
+  * Documentation updates
+
+#### v0.9.4
+
+  * Fixes major bug in OVA parser
+
+#### v0.9.3
+
+  * Fixes bugs with image & OVA upload parsing
+
+#### v0.9.2
+
+  * Major linting of the codebase with flake8; adds linting tools
+  * Implements CLI-based modification of VM vCPUs, memory, networks, and disks without directly editing XML
+  * Fixes bug where `pvc vm log -f` would show all 1000 lines before starting
+  * Fixes bug in default provisioner libvirt schema (`drive` -> `driver` typo)
+
+#### v0.9.1
+
+  * Added per-VM migration method feature
+  * Fixed bug with provisioner system template listing
+
+#### v0.9.0
+
+Numerous small improvements and bugfixes. This release is suitable for general use and is pre-release-quality software.
+
+This release introduces an updated version scheme; all future stable releases until 1.0.0 is ready will be made under this 0.9.z naming. This does not represent semantic versioning and all changes (feature, improvement, or bugfix) will be considered for inclusion in this release train.
+
+#### v0.8
+
+Numerous improvements and bugfixes. This release is suitable for general use and is pre-release-quality software.
+
+#### v0.7
+
+Numerous improvements and bugfixes, revamped documentation. This release is suitable for general use and is beta-quality software.
+
 #### v0.6
 
 Numerous improvements and bugfixes, full implementation of the provisioner, full implementation of the API CLI client (versus direct CLI client). This release is suitable for general use and is beta-quality software.

docs/manuals/ansible.md

@@ -1,3 +1,47 @@
+# PVC Ansible architecture
+
+The PVC Ansible setup and management framework is written in Ansible. It consists of two roles: `base` and `pvc`.
+
+## Base role
+
+The Base role configures a node to a specific, standard base Debian system, with a number of PVC-specific tweaks. Some examples include:
+
+* Installing the custom PVC repository at Boniface Labs.
+
+* Removing several unnecessary packages and installing numerous additional packages.
+
+* Automatically configuring network interfaces based on the `group_vars` configuration.
+
+* Configuring several general `sysctl` settings for optimal performance.
+
+* Installing and configuring rsyslog, postfix, ntpd, ssh, and fail2ban.
+
+* Creating the users specified in the `group_vars` configuration.
+
+* Installing custom MOTDs, bashrc files, vimrc files, and other useful configurations for each user.
+
+The end result is a standardized "PVC node" system ready to have the daemons installed by the PVC role.
+
+## PVC role
+
+The PVC role configures all the dependencies of PVC, including storage, networking, and databases, then installs the PVC daemon itself. Specifically, it will, in order:
+
+* Install Ceph, configure and bootstrap a new cluster if `bootstrap=yes` is set, configure the monitor and manager daemons, and start up the cluster ready for the addition of OSDs via the client interface (coordinators only).
+
+* Install, configure, and if `bootstrap=yes` is set, bootstrap a Zookeeper cluster (coordinators only).
+
+* Install, configure, and if `bootstrap=yes` is set`, bootstrap a Patroni PostgreSQL cluster for the PowerDNS aggregator (coordinators only).
+
+* Install and configure Libvirt.
+
+* Install and configure FRRouting.
+
+* Install and configure the main PVC daemon and API client, including initializing the PVC cluster (`pvc init`).
+
+## Completion
+
+Once the entire playbook has run for the first time against a given host, the host will be rebooted to apply all the configured services. On startup, the system should immediately launch the PVC daemon, check in to the Zookeeper cluster, and become ready. The node will be in `flushed` state on its first boot; the administrator will need to run `pvc node unflush <node>` to set the node into active state ready to handle virtual machines.
+
 # PVC Ansible configuration manual
 
 This manual documents the various `group_vars` configuration options for the `pvc-ansible` framework. We assume that the administrator is generally familiar with Ansible and its operation.

docs/manuals/api.md

@@ -1,3 +1,11 @@
+# PVC API architecture
+
+The PVC API is a standalone client application for PVC. It interfaces directly with the Zookeeper database to manage state.
+
+The API is built using Flask and is packaged in the Debian package `pvc-client-api`. The API depends on the common client functions of the `pvc-client-common` package as does the CLI client.
+
+Details of the API interface can be found in [the manual](/manuals/api).
+
 # PVC HTTP API manual
 
 The PVC HTTP API client is built with Flask, a Python framework for creating API interfaces, and run directly with the PyWSGI framework. It interfaces directly with the Zookeeper cluster to send and receive information about the cluster. It supports authentication configured statically via tokens in the configuration file as well as SSL. It also includes the provisioner client, an optional section that can be used to create VMs automatically using a set of templates and standardized scripts.
@@ -8,7 +16,7 @@ The [`pvc-ansible`](https://github.com/parallelvirtualcluster/pvc-ansible) frame
 
 ### SSL
 
-The API accepts SSL certificate and key files via the `pvc-api.yaml` configuration to enable SSL support for the API, which protects the data and query values from snooping or tampering. SSL is strongly recommended if using the API outside of a trusted local area network.
+The API accepts SSL certificate and key files via the `pvcapid.yaml` configuration to enable SSL support for the API, which protects the data and query values from snooping or tampering. SSL is strongly recommended if using the API outside of a trusted local area network.
 
 ### API authentication
 
@@ -148,7 +156,7 @@ curl -X GET http://localhost:7370/api/v1/provisioner/status/<task-id>
 
 ## API Daemon Configuration
 
-The API is configured using a YAML configuration file which is passed in to the API process by the environment variable `PVC_CONFIG_FILE`. When running with the default package and SystemD unit, this file is located at `/etc/pvc/pvc-api.yaml`.
+The API is configured using a YAML configuration file which is passed in to the API process by the environment variable `PVC_CONFIG_FILE`. When running with the default package and SystemD unit, this file is located at `/etc/pvc/pvcapid.yaml`.
 
 ### Conventions
 
@@ -156,7 +164,7 @@ The API is configured using a YAML configuration file which is passed in to the
 
 * Settings may `depends` on other settings. This indicates that, if one setting is enabled, the other setting is very likely `required` by that setting.
 
-### `pvc-api.yaml`
+### `pvcapid.yaml`
 
 Example configuration:
 
@@ -185,9 +193,9 @@ pvc:
         database:
             host: 10.100.0.252
             port: 5432
-            name: pvcprov
-            user: pvcprov
-            pass: pvcprov
+            name: pvcapi
+            user: pvcapi
+            pass: pvcapi
         queue:
             host: localhost
             port: 6379
@@ -286,7 +294,7 @@ The port of the PostgreSQL instance for the Provisioner database. Should always
 
 * *required*
 
-The database name for the Provisioner database. Should always be `pvcprov`.
+The database name for the Provisioner database. Should always be `pvcapi`.
 
 ##### `provisioner` → `database` → `user`
 

docs/manuals/cli.md

@@ -1,10 +1,18 @@
+# PVC CLI architecture
+
+The PVC CLI is a standalone client application for PVC. It interfaces with the PVC API, via a configurable list of clusters with customizable hosts, ports, addresses, and authentication.
+
+The CLI is build using Click and is packaged in the Debian package `pvc-client-cli`. The CLI does not depend on any other PVC components and can be used independently on arbitrary systems.
+
+The CLI is self-documenting, however [the manual](/manuals/cli) details the required configuration.
+
 # PVC CLI client manual
 
 The PVC CLI client is built with Click, a Python framework for creating self-documenting CLI applications. It interfaces with the PVC API.
 
 Use the `-h` option at any level of the `pvc` CLI command to receive help about the available commands and options.
 
-Before using the CLI on a non-PVC node system, at least one cluster must be added using the `pvc cluster` subcommands. Running the CLI on hosts which also run the PVC API (via its configuration at `/etc/pvc/pvc-api.yaml`) uses the special `local` cluster, reading information from the API configuration, by default.
+Before using the CLI on a non-PVC node system, at least one cluster must be added using the `pvc cluster` subcommands. Running the CLI on hosts which also run the PVC API (via its configuration at `/etc/pvc/pvcapid.yaml`) uses the special `local` cluster, reading information from the API configuration, by default.
 
 ## Configuration
 

docs/manuals/daemon.md

@@ -1,10 +1,64 @@
+# PVC Node Daemon architecture
+
+The PVC Node Daemon is the heart of the PVC system and runs on each node to manage the state of the node and its configured resources. The daemon connects directly to the Zookeeper cluster for coordination and state.
+
+The node daemon is build using Python 3.X and is packaged in the Debian package `pvc-daemon`.
+
+Configuration of the daemon is documented in [the manual](/manuals/daemon), however it is recommended to use the [Ansible configuration interface](/manuals/ansible) to configure the PVC system for you from scratch.
+
+## Overall architecture
+
+The PVC daemon is object-oriented - each cluster resource is represented by an Object, which is then present on each node in the cluster. This allows state changes to be reflected across the entire cluster should their data change.
+
+During startup, the system scans the Zookeeper database and sets up the required objects. The database is then watched in real-time for additional changes to the database information.
+
+## Startup sequence
+
+The daemon startup sequence is documented below. The main daemon entry-point is `Daemon.py` inside the `pvcnoded` folder, which is called from the `pvcnoded.py` stub file.
+
+0. The configuration is read from `/etc/pvc/pvcnoded.yaml` and the configuration object set up.
+
+0. Any required filesystem directories, mostly dynamic directories, are created.
+
+0. The logger is set up. If file logging is enabled, this is the state when the first log messages are written.
+
+0. Host networking is configured based on the `pvcnoded.yaml` configuration file. In a normal cluster, this is the point where the node will become reachable on the network as all networking is handled by the PVC node daemon.
+
+0. Sysctl tweaks are applied to the host system, to enable routing/forwarding between nodes via the host.
+
+0. The node determines its coordinator state and starts the required daemons if applicable. In a normal cluster, this is the point where the dependent services such as Zookeeper, FRR, and Ceph become available. After this step, the daemon waits 5 seconds before proceeding to give these daemons a chance to start up.
+
+0. The daemon connects to the Zookeeper cluster and starts its listener. If the Zookeeper cluster is unavailable, it will wait some time before abandoning the attempt and starting again from step 1.
+
+0. Termination handling/cleanup is configured.
+
+0. The node checks if it is already present in the Zookeeper cluster; if not, it will add itself to the database. Initial static options are also updated in the database here. The daemon state transitions from `stop` to `init`.
+
+0. The node checks if Libvirt is accessible.
+
+0. The node starts up the NFT firewall if applicable and configures the base rule-set.
+
+0. The node ensures that `dnsmasq` is stopped (legacy check, might be safe to remove eventually).
+
+0. The node begins setting up the object representations of resources, in order:
+
+    a. Node entries
+
+    b. Network entries, creating client networks and starting them as required.
+
+    c. Domain (VM) entries, starting up the VMs as required.
+
+    d. Ceph storage entries (OSDs, Pools, Volumes, Snapshots).
+
+0. The node activates its keepalived timer and begins sending keepalive updates to the cluster. The daemon state transitions from `init` to `run` and the system has started fully.
+
 # PVC Node Daemon manual
 
 The PVC node daemon ins build with Python 3 and is run directly on nodes. For details of the startup sequence and general layout, see the [architecture document](/architecture/daemon).
 
 ## Configuration
 
-The Daemon is configured using a YAML configuration file which is passed in to the API process by the environment variable `PVCD_CONFIG_FILE`. When running with the default package and SystemD unit, this file is located at `/etc/pvc/pvcd.yaml`.
+The Daemon is configured using a YAML configuration file which is passed in to the API process by the environment variable `PVCD_CONFIG_FILE`. When running with the default package and SystemD unit, this file is located at `/etc/pvc/pvcnoded.yaml`.
 
 For most deployments, the management of the configuration file is handled entirely by the [PVC Ansible framework](/manuals/ansible) and should not be modified directly. Many options from the Ansible framework map directly into the configuration options in this file.
 
@@ -14,7 +68,7 @@ For most deployments, the management of the configuration file is handled entire
 
 * Settings may `depends` on other settings. This indicates that, if one setting is enabled, the other setting is very likely `required` by that setting.
 
-### `pvcd.yaml`
+### `pvcnoded.yaml`
 
 Example configuration:
 
@@ -58,9 +112,9 @@ pvc:
       database:
         host: localhost
         port: 5432
-        name: pvcprov
-        user: pvcprov
-        pass: pvcprovPassw0rd
+        name: pvcapi
+        user: pvcapi
+        pass: pvcapiPassw0rd
   system:
     fencing:
       intervals:
@@ -225,7 +279,7 @@ The port of the PostgreSQL instance for the Provisioner database. Should always
 
 * *required*
 
-The database name for the Provisioner database. Should always be `pvcprov`.
+The database name for the Provisioner database. Should always be `pvcapi`.
 
 ##### `metadata` → `database` → `user`
 

docs/manuals/provisioner.md

@@ -1,4 +1,4 @@
-# PVC Provisioner API architecture
+# PVC Provisioner manual
 
 The PVC provisioner is a subsection of the main PVC API. IT interfaces directly with the Zookeeper database using the common client functions, and with the Patroni PostgreSQL database to store details. The provisioner also interfaces directly with the Ceph storage cluster, for mapping volumes, creating filesystems, and installing guests.
 
@@ -10,10 +10,18 @@ The purpose of the Provisioner API is to provide a convenient way for administra
 
 The Provisioner allows the administrator to constuct descriptions of VMs, called profiles, which include system resource specifications, network interfaces, disks, cloud-init userdata, and installation scripts. These profiles are highly modular, allowing the administrator to specify arbitrary combinations of the mentioned VM features with which to build new VMs.
 
-Currently, the provisioner supports creating VMs based off of installation scripts, or by cloning existing volumes. Future versions of PVC will allow the uploading of arbitrary images (either disk or ISO images) to cluster volumes, permitting even more flexibility in the installation of VMs.
+The provisioner supports creating VMs based off of installation scripts, by cloning existing volumes, and by uploading OVA image templates to the cluster.
 
 Examples in the following sections use the CLI exclusively for demonstration purposes. For details of the underlying API calls, please see the [API interface reference](/manuals/api-reference.html).
 
+# Deploying VMs from OVA images
+
+PVC supports deploying virtual machines from industry-standard OVA images. OVA images can be uploaded to the cluster with the `pvc provisioner ova` commands, and deployed via the created profile(s) using the `pvc provisioner create` command. Additionally, the profile(s) can be modified to suite your specific needs via the provisioner template system detailed below.
+
+# Deploying VMs from provisioner scripts
+
+PVC supports deploying virtual machines using administrator-provided scripts, using templates, profiles, and Cloud-init userdata to control the deployment process as desired. This deployment method permits the administrator to deploy POSIX-like systems such as Linux or BSD directly from a companion tool such as `debootstrap` on-demand and with maximum flexibility.
+
 ## Templates
 
 The PVC Provisioner features three categories of templates to specify the resources allocated to the virtual machine. They are: System Templates, Network Templates, and Disk Templates.

docs/manuals/swagger.json

@@ -62,6 +62,11 @@
                     "description": "The total number of snapshots in the storage cluster",
                     "type": "integer"
                 },
+                "storage_health": {
+                    "description": "The overall storage cluster health",
+                    "example": "Optimal",
+                    "type": "string"
+                },
                 "upstream_ip": {
                     "description": "The cluster upstream IP address in CIDR format",
                     "example": "10.0.0.254/24",
@@ -422,13 +427,17 @@
                 "memory": {
                     "properties": {
                         "allocated": {
-                            "description": "The total amount of RAM allocated to domains in MB",
+                            "description": "The total amount of RAM allocated to running domains in MB",
                             "type": "integer"
                         },
                         "free": {
                             "description": "The total free RAM on the node in MB",
                             "type": "integer"
                         },
+                        "provisioned": {
+                            "description": "The total amount of RAM provisioned to all domains (regardless of state) on this node in MB",
+                            "type": "integer"
+                        },
                         "total": {
                             "description": "The total amount of node RAM in MB",
                             "type": "integer"
@@ -554,6 +563,48 @@
             },
             "type": "object"
         },
+        "ova": {
+            "properties": {
+                "id": {
+                    "description": "Internal provisioner OVA ID",
+                    "type": "integer"
+                },
+                "name": {
+                    "description": "OVA name",
+                    "type": "string"
+                },
+                "volumes": {
+                    "items": {
+                        "id": "ova_volume",
+                        "properties": {
+                            "disk_id": {
+                                "description": "Disk identifier",
+                                "type": "string"
+                            },
+                            "disk_size_gb": {
+                                "description": "Disk size in GB",
+                                "type": "string"
+                            },
+                            "pool": {
+                                "description": "Pool containing the OVA volume",
+                                "type": "string"
+                            },
+                            "volume_format": {
+                                "description": "OVA image format",
+                                "type": "string"
+                            },
+                            "volume_name": {
+                                "description": "Storage volume containing the OVA image",
+                                "type": "string"
+                            }
+                        },
+                        "type": "object"
+                    },
+                    "type": "list"
+                }
+            },
+            "type": "object"
+        },
         "pool": {
             "properties": {
                 "name": {
@@ -757,6 +808,146 @@
             },
             "type": "object"
         },
+        "storagebenchmark": {
+            "properties": {
+                "benchmark_result": {
+                    "properties": {
+                        "test_name": {
+                            "properties": {
+                                "bandwidth": {
+                                    "properties": {
+                                        "max": {
+                                            "description": "The maximum bandwidth (KiB/s) measurement",
+                                            "type": "string (integer)"
+                                        },
+                                        "mean": {
+                                            "description": "The mean bandwidth (KiB/s) measurement",
+                                            "type": "string (float)"
+                                        },
+                                        "min": {
+                                            "description": "The minimum bandwidth (KiB/s) measurement",
+                                            "type": "string (integer)"
+                                        },
+                                        "numsamples": {
+                                            "description": "The number of samples taken during the test",
+                                            "type": "string (integer)"
+                                        },
+                                        "stdev": {
+                                            "description": "The standard deviation of bandwidth",
+                                            "type": "string (float)"
+                                        }
+                                    },
+                                    "type": "object"
+                                },
+                                "cpu": {
+                                    "properties": {
+                                        "ctxsw": {
+                                            "description": "The number of context switches during the test",
+                                            "type": "string (integer)"
+                                        },
+                                        "majfault": {
+                                            "description": "The number of major page faults during the test",
+                                            "type": "string (integer)"
+                                        },
+                                        "minfault": {
+                                            "description": "The number of minor page faults during the test",
+                                            "type": "string (integer)"
+                                        },
+                                        "system": {
+                                            "description": "The percentage of test time spent in system (kernel) space",
+                                            "type": "string (float percentage)"
+                                        },
+                                        "user": {
+                                            "description": "The percentage of test time spent in user space",
+                                            "type": "string (float percentage)"
+                                        }
+                                    },
+                                    "type": "object"
+                                },
+                                "iops": {
+                                    "properties": {
+                                        "max": {
+                                            "description": "The maximum IOPS measurement",
+                                            "type": "string (integer)"
+                                        },
+                                        "mean": {
+                                            "description": "The mean IOPS measurement",
+                                            "type": "string (float)"
+                                        },
+                                        "min": {
+                                            "description": "The minimum IOPS measurement",
+                                            "type": "string (integer)"
+                                        },
+                                        "numsamples": {
+                                            "description": "The number of samples taken during the test",
+                                            "type": "string (integer)"
+                                        },
+                                        "stdev": {
+                                            "description": "The standard deviation of IOPS",
+                                            "type": "string (float)"
+                                        }
+                                    },
+                                    "type": "object"
+                                },
+                                "latency": {
+                                    "properties": {
+                                        "max": {
+                                            "description": "The maximum latency measurement",
+                                            "type": "string (integer)"
+                                        },
+                                        "mean": {
+                                            "description": "The mean latency measurement",
+                                            "type": "string (float)"
+                                        },
+                                        "min": {
+                                            "description": "The minimum latency measurement",
+                                            "type": "string (integer)"
+                                        },
+                                        "stdev": {
+                                            "description": "The standard deviation of latency",
+                                            "type": "string (float)"
+                                        }
+                                    },
+                                    "type": "object"
+                                },
+                                "overall": {
+                                    "properties": {
+                                        "bandwidth": {
+                                            "description": "The average bandwidth (KiB/s)",
+                                            "type": "string (integer)"
+                                        },
+                                        "iops": {
+                                            "description": "The average IOPS",
+                                            "type": "string (integer)"
+                                        },
+                                        "iosize": {
+                                            "description": "The total size of the benchmark data",
+                                            "type": "string (integer)"
+                                        },
+                                        "runtime": {
+                                            "description": "The total test time in milliseconds",
+                                            "type": "string (integer)"
+                                        }
+                                    },
+                                    "type": "object"
+                                }
+                            },
+                            "type": "object"
+                        }
+                    },
+                    "type": "object"
+                },
+                "id": {
+                    "description": "The database ID of the test result",
+                    "type": "string (containing integer)"
+                },
+                "job": {
+                    "description": "The job name (an ISO date) of the test result",
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
         "system-template": {
             "properties": {
                 "id": {
@@ -866,9 +1057,25 @@
                                 "description": "The full name of the volume in \"pool/volume\" format",
                                 "type": "string"
                             },
+                            "rd_bytes": {
+                                "description": "The number of read bytes from the volume",
+                                "type": "integer"
+                            },
+                            "rd_req": {
+                                "description": "The number of read requests from the volume",
+                                "type": "integer"
+                            },
                             "type": {
                                 "description": "The type of volume",
                                 "type": "string"
+                            },
+                            "wr_bytes": {
+                                "description": "The number of write bytes to the volume",
+                                "type": "integer"
+                            },
+                            "wr_req": {
+                                "description": "The number of write requests to the volume",
+                                "type": "integer"
                             }
                         },
                         "type": "object"
@@ -902,6 +1109,51 @@
                     "description": "The assigned RAM of the VM in MB",
                     "type": "integer"
                 },
+                "memory_stats": {
+                    "properties": {
+                        "actual": {
+                            "description": "The total active memory of the VM in kB",
+                            "type": "integer"
+                        },
+                        "available": {
+                            "description": "The total amount of usable memory as seen by the domain in kB",
+                            "type": "integer"
+                        },
+                        "last_update": {
+                            "description": "Timestamp of the last update of statistics, in seconds",
+                            "type": "integer"
+                        },
+                        "major_fault": {
+                            "description": "The number of major page faults",
+                            "type": "integer"
+                        },
+                        "minor_fault": {
+                            "description": "The number of minor page faults",
+                            "type": "integer"
+                        },
+                        "rss": {
+                            "description": "The Resident Set Size of the process running the domain in kB",
+                            "type": "integer"
+                        },
+                        "swap_in": {
+                            "description": "The amount of swapped in data in kB",
+                            "type": "integer"
+                        },
+                        "swap_out": {
+                            "description": "The amount of swapped out data in kB",
+                            "type": "integer"
+                        },
+                        "unused": {
+                            "description": "The amount of memory left completely unused by the system in kB",
+                            "type": "integer"
+                        },
+                        "usable": {
+                            "description": "How much the balloon can be inflated without pushing the guest system to swap in kB",
+                            "type": "integer"
+                        }
+                    },
+                    "type": "object"
+                },
                 "migrated": {
                     "description": "Whether the VM has been migrated, either \"no\" or \"from <last_node>\"",
                     "type": "string"
@@ -922,6 +1174,22 @@
                                 "description": "The virtual network device model",
                                 "type": "string"
                             },
+                            "rd_bytes": {
+                                "description": "The number of read bytes on the interface",
+                                "type": "integer"
+                            },
+                            "rd_drops": {
+                                "description": "The number of read drops on the interface",
+                                "type": "integer"
+                            },
+                            "rd_errors": {
+                                "description": "The number of read errors on the interface",
+                                "type": "integer"
+                            },
+                            "rd_packets": {
+                                "description": "The number of read packets on the interface",
+                                "type": "integer"
+                            },
                             "source": {
                                 "description": "The parent network bridge on the node",
                                 "type": "string"
@@ -929,6 +1197,22 @@
                             "type": {
                                 "description": "The PVC network type",
                                 "type": "string"
+                            },
+                            "wr_bytes": {
+                                "description": "The number of write bytes on the interface",
+                                "type": "integer"
+                            },
+                            "wr_drops": {
+                                "description": "The number of write drops on the interface",
+                                "type": "integer"
+                            },
+                            "wr_errors": {
+                                "description": "The number of write errors on the interface",
+                                "type": "integer"
+                            },
+                            "wr_packets": {
+                                "description": "The number of write packets on the interface",
+                                "type": "integer"
                             }
                         },
                         "type": "object"
@@ -974,6 +1258,23 @@
                     "description": "The assigned vCPUs of the VM",
                     "type": "integer"
                 },
+                "vcpu_stats": {
+                    "properties": {
+                        "cpu_time": {
+                            "description": "The active CPU time for all vCPUs",
+                            "type": "integer"
+                        },
+                        "system_time": {
+                            "description": "vCPU system time",
+                            "type": "integer"
+                        },
+                        "user_time": {
+                            "description": "vCPU user time",
+                            "type": "integer"
+                        }
+                    },
+                    "type": "object"
+                },
                 "vcpu_topology": {
                     "description": "The topology of the assigned vCPUs in Sockets/Cores/Threads format",
                     "type": "string"
@@ -1946,6 +2247,27 @@
                         "name": "limit",
                         "required": false,
                         "type": "string"
+                    },
+                    {
+                        "description": "Limit results to nodes in the specified daemon state",
+                        "in": "query",
+                        "name": "daemon_state",
+                        "required": false,
+                        "type": "string"
+                    },
+                    {
+                        "description": "Limit results to nodes in the specified coordinator state",
+                        "in": "query",
+                        "name": "coordinator_state",
+                        "required": false,
+                        "type": "string"
+                    },
+                    {
+                        "description": "Limit results to nodes in the specified domain state",
+                        "in": "query",
+                        "name": "domain_state",
+                        "required": false,
+                        "type": "string"
                     }
                 ],
                 "responses": {
@@ -2162,6 +2484,12 @@
                         "name": "start_vm",
                         "required": false,
                         "type": "boolean"
+                    },
+                    {
+                        "description": "Script install() function keywork argument in \"arg=data\" format; may be specified multiple times to add multiple arguments",
+                        "in": "query",
+                        "name": "arg",
+                        "type": "string"
                     }
                 ],
                 "responses": {
@@ -2190,6 +2518,160 @@
                 ]
             }
         },
+        "/api/v1/provisioner/ova": {
+            "get": {
+                "description": "",
+                "parameters": [
+                    {
+                        "description": "An OVA name search limit; fuzzy by default, use ^/$ to force exact matches",
+                        "in": "query",
+                        "name": "limit",
+                        "required": false,
+                        "type": "string"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "items": {
+                                "$ref": "#/definitions/ova"
+                            },
+                            "type": "list"
+                        }
+                    }
+                },
+                "summary": "Return a list of OVA sources",
+                "tags": [
+                    "provisioner"
+                ]
+            },
+            "post": {
+                "description": "<br/>The API client is responsible for determining and setting the ova_size value, as this value cannot be determined dynamically before the upload proceeds.",
+                "parameters": [
+                    {
+                        "description": "Storage pool name",
+                        "in": "query",
+                        "name": "pool",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "description": "OVA name on the cluster (usually identical to the OVA file name)",
+                        "in": "query",
+                        "name": "name",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "description": "Size of the OVA file in bytes",
+                        "in": "query",
+                        "name": "ova_size",
+                        "required": true,
+                        "type": "string"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/Message"
+                        }
+                    },
+                    "400": {
+                        "description": "Bad request",
+                        "schema": {
+                            "$ref": "#/definitions/Message"
+                        }
+                    }
+                },
+                "summary": "Upload an OVA image to the cluster",
+                "tags": [
+                    "provisioner"
+                ]
+            }
+        },
+        "/api/v1/provisioner/ova/{ova}": {
+            "delete": {
+                "description": "",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/Message"
+                        }
+                    },
+                    "404": {
+                        "description": "Not found",
+                        "schema": {
+                            "$ref": "#/definitions/Message"
+                        }
+                    }
+                },
+                "summary": "Remove ova {ova}",
+                "tags": [
+                    "provisioner"
+                ]
+            },
+            "get": {
+                "description": "",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/ova"
+                        }
+                    },
+                    "404": {
+                        "description": "Not found",
+                        "schema": {
+                            "$ref": "#/definitions/Message"
+                        }
+                    }
+                },
+                "summary": "Return information about OVA image {ova}",
+                "tags": [
+                    "provisioner"
+                ]
+            },
+            "post": {
+                "description": "<br/>The API client is responsible for determining and setting the ova_size value, as this value cannot be determined dynamically before the upload proceeds.",
+                "parameters": [
+                    {
+                        "description": "Storage pool name",
+                        "in": "query",
+                        "name": "pool",
+                        "required": true,
+                        "type": "string"
+                    },
+                    {
+                        "description": "Size of the OVA file in bytes",
+                        "in": "query",
+                        "name": "ova_size",
+                        "required": true,
+                        "type": "string"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/Message"
+                        }
+                    },
+                    "400": {
+                        "description": "Bad request",
+                        "schema": {
+                            "$ref": "#/definitions/Message"
+                        }
+                    }
+                },
+                "summary": "Upload an OVA image to the cluster",
+                "tags": [
+                    "provisioner"
+                ]
+            }
+        },
         "/api/v1/provisioner/profile": {
             "get": {
                 "description": "",
@@ -2228,39 +2710,57 @@
                         "required": true,
                         "type": "string"
                     },
+                    {
+                        "description": "Profile type",
+                        "enum": [
+                            "provisioner",
+                            "ova"
+                        ],
+                        "in": "query",
+                        "name": "profile_type",
+                        "required": true,
+                        "type": "string"
+                    },
                     {
                         "description": "Script name",
                         "in": "query",
                         "name": "script",
-                        "required": true,
+                        "required": false,
                         "type": "string"
                     },
                     {
                         "description": "System template name",
                         "in": "query",
                         "name": "system_template",
-                        "required": true,
+                        "required": false,
                         "type": "string"
                     },
                     {
                         "description": "Network template name",
                         "in": "query",
                         "name": "network_template",
-                        "required": true,
+                        "required": false,
                         "type": "string"
                     },
                     {
                         "description": "Storage template name",
                         "in": "query",
                         "name": "storage_template",
-                        "required": true,
+                        "required": false,
                         "type": "string"
                     },
                     {
                         "description": "Userdata template name",
                         "in": "query",
                         "name": "userdata",
-                        "required": true,
+                        "required": false,
+                        "type": "string"
+                    },
+                    {
+                        "description": "OVA image source",
+                        "in": "query",
+                        "name": "ova",
+                        "required": false,
                         "type": "string"
                     },
                     {
@@ -2336,6 +2836,17 @@
             "post": {
                 "description": "",
                 "parameters": [
+                    {
+                        "description": "Profile type",
+                        "enum": [
+                            "provisioner",
+                            "ova"
+                        ],
+                        "in": "query",
+                        "name": "profile_type",
+                        "required": true,
+                        "type": "string"
+                    },
                     {
                         "description": "Script name",
                         "in": "query",
@@ -2371,6 +2882,13 @@
                         "required": true,
                         "type": "string"
                     },
+                    {
+                        "description": "OVA image source",
+                        "in": "query",
+                        "name": "ova",
+                        "required": false,
+                        "type": "string"
+                    },
                     {
                         "description": "Script install() function keywork argument in \"arg=data\" format; may be specified multiple times to add multiple arguments",
                         "in": "query",
@@ -3558,6 +4076,77 @@
                 "tags": [
                     "provisioner / template"
                 ]
+            },
+            "put": {
+                "description": "",
+                "parameters": [
+                    {
+                        "description": "vCPU count for VM",
+                        "in": "query",
+                        "name": "vcpus",
+                        "type": "integer"
+                    },
+                    {
+                        "description": "vRAM size in MB for VM",
+                        "in": "query",
+                        "name": "vram",
+                        "type": "integer"
+                    },
+                    {
+                        "description": "Whether to enable serial console for VM",
+                        "in": "query",
+                        "name": "serial",
+                        "type": "boolean"
+                    },
+                    {
+                        "description": "Whether to enable VNC console for VM",
+                        "in": "query",
+                        "name": "vnc",
+                        "type": "boolean"
+                    },
+                    {
+                        "description": "VNC bind address when VNC console is enabled",
+                        "in": "query",
+                        "name": "vnc_bind",
+                        "type": "string"
+                    },
+                    {
+                        "description": "CSV list of node(s) to limit VM assignment to",
+                        "in": "query",
+                        "name": "node_limit",
+                        "type": "string"
+                    },
+                    {
+                        "description": "Selector to use for VM node assignment on migration/move",
+                        "in": "query",
+                        "name": "node_selector",
+                        "type": "string"
+                    },
+                    {
+                        "description": "Whether to start VM with node ready state (one-time)",
+                        "in": "query",
+                        "name": "node_autostart",
+                        "type": "boolean"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/Message"
+                        }
+                    },
+                    "400": {
+                        "description": "Bad request",
+                        "schema": {
+                            "$ref": "#/definitions/Message"
+                        }
+                    }
+                },
+                "summary": "Modify an existing system template {template}",
+                "tags": [
+                    "provisioner / template"
+                ]
             }
         },
         "/api/v1/provisioner/userdata": {
@@ -3780,6 +4369,57 @@
                 ]
             }
         },
+        "/api/v1/storage/ceph/benchmark": {
+            "get": {
+                "description": "",
+                "parameters": [
+                    {
+                        "description": "A single job name to limit results to",
+                        "in": "query",
+                        "name": "job",
+                        "required": false,
+                        "type": "string"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/storagebenchmark"
+                        }
+                    }
+                },
+                "summary": "List results from benchmark jobs",
+                "tags": [
+                    "storage / ceph"
+                ]
+            },
+            "post": {
+                "description": "",
+                "parameters": [
+                    {
+                        "description": "The PVC storage pool to benchmark",
+                        "in": "query",
+                        "name": "pool",
+                        "required": true,
+                        "type": "string"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "description": "The Celery job ID of the benchmark (unused elsewhere)",
+                            "type": "string"
+                        }
+                    }
+                },
+                "summary": "Execute a storage benchmark against a storage pool",
+                "tags": [
+                    "storage / ceph"
+                ]
+            }
+        },
         "/api/v1/storage/ceph/option": {
             "post": {
                 "description": "",
@@ -4691,6 +5331,52 @@
                 ]
             }
         },
+        "/api/v1/storage/ceph/volume/{pool}/{volume}/upload": {
+            "post": {
+                "description": "<br/>The body must be a form body containing a file that is the binary contents of the image.",
+                "parameters": [
+                    {
+                        "description": "The type of source image file",
+                        "enum": [
+                            "raw",
+                            "vmdk",
+                            "qcow2",
+                            "qed",
+                            "vdi",
+                            "vpc"
+                        ],
+                        "in": "query",
+                        "name": "image_format",
+                        "required": true,
+                        "type": "string"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/Message"
+                        }
+                    },
+                    "400": {
+                        "description": "Bad request",
+                        "schema": {
+                            "$ref": "#/definitions/Message"
+                        }
+                    },
+                    "404": {
+                        "description": "Not found",
+                        "schema": {
+                            "$ref": "#/definitions/Message"
+                        }
+                    }
+                },
+                "summary": "Upload a disk image to Ceph volume {volume} in pool {pool}",
+                "tags": [
+                    "storage / ceph"
+                ]
+            }
+        },
         "/api/v1/vm": {
             "get": {
                 "description": "",
@@ -5142,6 +5828,18 @@
                         "in": "query",
                         "name": "force",
                         "type": "boolean"
+                    },
+                    {
+                        "description": "Whether to block waiting for the migration to complete",
+                        "in": "query",
+                        "name": "wait",
+                        "type": "boolean"
+                    },
+                    {
+                        "description": "Whether to enforce live migration and disable shutdown-based fallback migration",
+                        "in": "query",
+                        "name": "force_live",
+                        "type": "boolean"
                     }
                 ],
                 "responses": {
@@ -5202,6 +5900,12 @@
                         "name": "state",
                         "required": true,
                         "type": "string"
+                    },
+                    {
+                        "description": "Whether to block waiting for the state change to complete",
+                        "in": "query",
+                        "name": "wait",
+                        "type": "boolean"
                     }
                 ],
                 "responses": {

gen-api-doc

@@ -8,14 +8,13 @@ import os
 import sys
 import json
 
-os.environ['PVC_CONFIG_FILE'] = "./client-api/pvc-api.sample.yaml"
+os.environ['PVC_CONFIG_FILE'] = "./api-daemon/pvcapid.sample.yaml"
 
-sys.path.append('client-api')
+sys.path.append('api-daemon')
 
-pvc_api = __import__('pvc-api')
+import pvcapid.flaskapi as pvc_api
 
 swagger_file = "docs/manuals/swagger.json"
-
 swagger_data = swagger(pvc_api.app)
 swagger_data['info']['version'] = "1.0"
 swagger_data['info']['title'] = "PVC Client and Provisioner API"

gen-api-migrations

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Generate the database migration files
+
+VERSION="$( head -1 debian/changelog | awk -F'[()-]' '{ print $2 }' )"
+
+pushd api-daemon
+export PVC_CONFIG_FILE="./pvcapid.sample.yaml"
+./pvcapid-manage.py db migrate -m "PVC version ${VERSION}"
+./pvcapid-manage.py db upgrade
+popd

lint

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+if ! which flake8 &>/dev/null; then
+    echo "Flake8 is required to lint this project"
+    exit 1
+fi
+
+flake8 \
+    --ignore=E501 \
+    --exclude=api-daemon/migrations/versions,api-daemon/provisioner/examples
+ret=$?
+if [[ $ret -eq 0 ]]; then
+    echo "No linting issues found!"
+fi
+exit $ret