Rename API and common Debian packages

Closes #79

api-daemon/api_lib/libvirt_schema.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+# libvirt_schema.py - Libvirt schema elements
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+# File header, containing default values for various non-device components
+# Variables:
+#  * vm_name
+#  * vm_uuid
+#  * vm_description
+#  * vm_memory
+#  * vm_vcpus
+#  * vm_architecture
+libvirt_header = """<domain type='kvm'>
+  <name>{vm_name}</name>
+  <uuid>{vm_uuid}</uuid>
+  <description>{vm_description}</description>
+  <memory unit='MiB'>{vm_memory}</memory>
+  <vcpu>{vm_vcpus}</vcpu>
+  <cpu>
+    <topology sockets='1' cores='{vm_vcpus}' threads='1'/>
+  </cpu>
+  <os>
+    <type arch='{vm_architecture}' machine='pc-i440fx-2.7'>hvm</type>
+    <bootmenu enable='yes'/>
+    <boot dev='cdrom'/>
+    <boot dev='hd'/>
+  </os>
+  <features>
+    <acpi/>
+    <apic/>
+    <pae/>
+  </features>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>restart</on_crash>
+  <devices>
+"""
+
+# File footer, closing devices and domain elements
+libvirt_footer = """  </devices>
+</domain>"""
+
+# Default devices for all VMs
+devices_default = """    <emulator>/usr/bin/kvm</emulator>
+    <controller type='usb' index='0'/>
+    <controller type='pci' index='0' model='pci-root'/>
+    <rng model='virtio'>
+      <rate period="1000" bytes="2048"/>
+      <backend model='random'>/dev/random</backend>
+    </rng>
+"""
+
+# Serial device
+# Variables:
+#  * vm_name
+devices_serial = """    <serial type='pty'>
+      <log file='/var/log/libvirt/{vm_name}.log' append='on'/>
+    </serial>
+    <console type='pty'/>
+"""
+
+# VNC device
+# Variables:
+#  * vm_vncport
+#  * vm_vnc_autoport
+#  * vm_vnc_bind
+devices_vnc = """    <graphics type='vnc' port='{vm_vncport}' autoport='{vm_vnc_autoport}' listen='{vm_vnc_bind}'/>
+"""
+
+# VirtIO SCSI device
+devices_scsi_controller = """    <controller type='scsi' index='0' model='virtio-scsi'/>
+"""
+
+# Disk device header
+# Variables:
+#  * ceph_storage_secret
+#  * disk_pool
+#  * vm_name
+#  * disk_id
+devices_disk_header = """    <disk type='network' device='disk'>
+      <driver name='qemu' discard='unmap'/>
+      <target dev='{disk_id}' bus='scsi'/>
+      <auth username='libvirt'>
+        <secret type='ceph' uuid='{ceph_storage_secret}'/>
+      </auth>
+      <source protocol='rbd' name='{disk_pool}/{vm_name}_{disk_id}'>
+"""
+
+# Disk device coordinator element
+# Variables:
+#  * coordinator_name
+#  * coordinator_ceph_mon_port
+devices_disk_coordinator = """        <host name='{coordinator_name}' port='{coordinator_ceph_mon_port}'/>
+"""
+
+# Disk device footer
+devices_disk_footer = """      </source>
+    </disk>
+"""
+
+# vhostmd virtualization passthrough device
+devices_vhostmd = """    <disk type='file' device='disk'>
+      <drive name='qemu' type='raw'/>
+      <source file='/dev/shm/vhostmd0'/>
+      <target dev='sdz' bus='usb'/>
+      <readonly/>
+    </disk>
+"""
+
+# Network interface device
+# Variables:
+#  * eth_macaddr
+#  * eth_bridge
+devices_net_interface = """    <interface type='bridge'>
+      <mac address='{eth_macaddr}'/>
+      <source bridge='{eth_bridge}'/>
+      <model type='virtio'/>
+    </interface>
+"""

api-daemon/daemon_lib

@@ -0,0 +1 @@
+../daemon-common

api-daemon/provisioner/examples/debootstrap_script.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+
+# debootstrap_script.py - PVC Provisioner example script for Debootstrap
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+# This script provides an example of a PVC provisioner script. It will install
+# a Debian system, of the release specified in the keyword argument `deb_release`
+# and from the mirror specified in the keyword argument `deb_mirror`, and
+# including the packages specified in the keyword argument `deb_packages` (a list
+# of strings, which is then joined together as a CSV and passed to debootstrap),
+# to the configured disks, configure fstab, and install GRUB. Any later config
+# should be done within the VM, for instance via cloud-init.
+
+# This script can thus be used as an example or reference implementation of a
+# PVC provisioner script and expanded upon as required.
+
+# This script will run under root privileges as the provisioner does. Be careful
+# with that.
+
+import os
+
+# Installation function - performs a debootstrap install of a Debian system
+# Note that the only arguments are keyword arguments.
+def install(**kwargs):
+    # The provisioner has already mounted the disks on kwargs['temporary_directory'].
+    # by this point, so we can get right to running the debootstrap after setting
+    # some nicer variable names; you don't necessarily have to do this.
+    vm_name = kwargs['vm_name']
+    temporary_directory = kwargs['temporary_directory']
+    disks = kwargs['disks']
+    networks = kwargs['networks']
+    # Our own required arguments. We should, though are not required to, handle
+    # failures of these gracefully, should administrators forget to specify them.
+    try:
+        deb_release = kwargs['deb_release']
+    except:
+        deb_release = "stable"
+    try:
+        deb_mirror = kwargs['deb_mirror']
+    except:
+        deb_mirror = "http://ftp.debian.org/debian"
+    try:
+        deb_packages = kwargs['deb_packages'].split(',')
+    except:
+        deb_packages = ["linux-image-amd64", "grub-pc", "cloud-init", "python3-cffi-backend", "wget"]
+
+    # We need to know our root disk
+    root_disk = None
+    for disk in disks:
+        if disk['mountpoint'] == '/':
+            root_disk = disk
+    if not root_disk:
+        return
+
+    # Ensure we have debootstrap intalled on the provisioner system; this is a
+    # good idea to include if you plan to use anything that is not part of the
+    # base Debian host system, just in case the provisioner host is not properly
+    # configured already.
+    os.system(
+        "apt-get install -y debootstrap"
+    )
+
+    # Perform a deboostrap installation
+    os.system(
+        "debootstrap --include={pkgs} {suite} {target} {mirror}".format(
+            suite=deb_release,
+            target=temporary_directory,
+            mirror=deb_mirror,
+            pkgs=','.join(deb_packages)
+        )
+    )
+
+    # Bind mount the devfs
+    os.system(
+        "mount --bind /dev {}/dev".format(
+            temporary_directory
+        )
+    )
+
+    # Create an fstab entry for each disk
+    fstab_file = "{}/etc/fstab".format(temporary_directory)
+    # The disk ID starts at zero and increments by one for each disk in the fixed-order
+    # disk list. This lets us work around the insanity of Libvirt IDs not matching guest IDs,
+    # while still letting us have some semblance of control here without enforcing things
+    # like labels. It increments in the for loop below at the end of each iteration, and is
+    # used to craft a /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0-0-0-X device ID
+    # which will always match the correct order from Libvirt (unlike sdX/vdX names).
+    disk_id = 0
+    for disk in disks:
+        # We assume SSD-based/-like storage, and dislike atimes
+        options = "defaults,discard,noatime,nodiratime"
+
+        # The root, var, and log volumes have specific values
+        if disk['mountpoint'] == "/":
+            dump = 0
+            cpass = 1
+        elif disk['mountpoint'] == '/var' or disk['mountpoint'] == '/var/log':
+            dump = 0
+            cpass = 2
+        else:
+            dump = 0
+            cpass = 0
+
+        # Append the fstab line
+        with open(fstab_file, 'a') as fh:
+            data = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0-0-0-{disk} {mountpoint} {filesystem} {options} {dump} {cpass}\n".format(
+                disk=disk_id,
+                mountpoint=disk['mountpoint'],
+                filesystem=disk['filesystem'],
+                options=options,
+                dump=dump,
+                cpass=cpass
+            )
+            fh.write(data)
+
+        # Increment the disk_id
+        disk_id += 1
+
+    # Write the hostname
+    hostname_file = "{}/etc/hostname".format(temporary_directory)
+    with open(hostname_file, 'w') as fh:
+        fh.write("{}".format(vm_name))
+
+    # Fix the cloud-init.target since it's broken
+    cloudinit_target_file = "{}/etc/systemd/system/cloud-init.target".format(temporary_directory)
+    with open(cloudinit_target_file, 'w') as fh:
+        data = """[Install]
+WantedBy=multi-user.target
+[Unit]
+Description=Cloud-init target
+After=multi-user.target
+"""
+        fh.write(data)
+
+    # NOTE: Due to device ordering within the Libvirt XML configuration, the first Ethernet interface
+    #       will always be on PCI bus ID 2, hence the name "ens2".
+    # Write a DHCP stanza for ens2
+    ens2_network_file = "{}/etc/network/interfaces.d/ens2".format(temporary_directory)
+    with open(ens2_network_file, 'w') as fh:
+        data = """auto ens2
+iface ens2 inet dhcp
+"""
+        fh.write(data)
+
+    # Write the DHCP config for ens2
+    dhclient_file = "{}/etc/dhcp/dhclient.conf".format(temporary_directory)
+    with open(dhclient_file, 'w') as fh:
+        data = """# DHCP client configuration
+# Written by the PVC provisioner
+option rfc3442-classless-static-routes code 121 = array of unsigned integer 8;
+interface "ens2" {
+""" + """        send fqdn.fqdn = "{hostname}";
+        send host-name = "{hostname}";
+""".format(hostname=vm_name) + """        request subnet-mask, broadcast-address, time-offset, routers,
+                domain-name, domain-name-servers, domain-search, host-name,
+                dhcp6.name-servers, dhcp6.domain-search, dhcp6.fqdn, dhcp6.sntp-servers,
+                netbios-name-servers, netbios-scope, interface-mtu,
+                rfc3442-classless-static-routes, ntp-servers;
+}
+"""
+        fh.write(data)
+
+    # Write the GRUB configuration
+    grubcfg_file = "{}/etc/default/grub".format(temporary_directory)
+    with open(grubcfg_file, 'w') as fh:
+        data = """# Written by the PVC provisioner
+GRUB_DEFAULT=0
+GRUB_TIMEOUT=1
+GRUB_DISTRIBUTOR="PVC Virtual Machine"
+GRUB_CMDLINE_LINUX_DEFAULT="root=/dev/{root_disk} console=tty0 console=ttyS0,115200n8"
+GRUB_CMDLINE_LINUX=""
+GRUB_TERMINAL=console
+GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
+GRUB_DISABLE_LINUX_UUID=false
+""".format(root_disk=root_disk['disk_id'])
+        fh.write(data)
+
+    # Chroot, do some in-root tasks, then exit the chroot
+    # EXITING THE CHROOT IS VERY IMPORTANT OR THE FOLLOWING STAGES OF THE PROVISIONER
+    # WILL FAIL IN UNEXPECTED WAYS! Keep this in mind when using chroot in your scripts.
+    real_root = os.open("/", os.O_RDONLY)
+    os.chroot(temporary_directory)
+    fake_root = os.open("/", os.O_RDONLY)
+    os.fchdir(fake_root)
+
+    # Install and update GRUB
+    os.system(
+        "grub-install --force /dev/rbd/{}/{}_{}".format(root_disk['pool'], vm_name, root_disk['disk_id'])
+    )
+    os.system( 
+        "update-grub"
+    )
+    # Set a really dumb root password [TEMPORARY]
+    os.system(
+        "echo root:test123 | chpasswd"
+    )
+    # Enable cloud-init target on (first) boot
+    # NOTE: Your user-data should handle this and disable it once done, or things get messy.
+    #       That cloud-init won't run without this hack seems like a bug... but even the official
+    #       Debian cloud images are affected, so who knows.
+    os.system(
+        "systemctl enable cloud-init.target"
+    )
+
+    # Restore our original root/exit the chroot
+    # EXITING THE CHROOT IS VERY IMPORTANT OR THE FOLLOWING STAGES OF THE PROVISIONER
+    # WILL FAIL IN UNEXPECTED WAYS! Keep this in mind when using chroot in your scripts.
+    os.fchdir(real_root)
+    os.chroot(".")
+    os.fchdir(real_root)
+    os.close(fake_root)
+    os.close(real_root)
+
+    # Unmount the bound devfs
+    os.system(
+        "umount {}/dev".format(
+            temporary_directory
+        )
+    )
+
+    # Clean up file handles so paths can be unmounted
+    del fake_root
+    del real_root
+
+    # Everything else is done via cloud-init user-data

api-daemon/provisioner/examples/dummy_script.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+# dummy_script.py - PVC Provisioner example script for noop
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2020 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+# This script provides an example of a PVC provisioner script. It will do
+# nothing and return back to the provisioner without taking any action, and
+# expecting no special arguments.
+
+# This script can thus be used as an example or reference implementation of a
+# PVC provisioner script and expanded upon as required.
+
+# This script will run under root privileges as the provisioner does. Be careful
+# with that.
+
+import os
+
+# Installation function - performs a debootstrap install of a Debian system
+# Note that the only arguments are keyword arguments.
+def install(**kwargs):
+    # The provisioner has already mounted the disks on kwargs['temporary_directory'].
+    # by this point, so we can get right to running the debootstrap after setting
+    # some nicer variable names; you don't necessarily have to do this.
+    vm_name = kwargs['vm_name']
+    temporary_directory = kwargs['temporary_directory']
+    disks = kwargs['disks']
+    networks = kwargs['networks']
+    # No operation - this script just returns
+    pass

api-daemon/provisioner/examples/multipart-userdata.yaml

@@ -0,0 +1,16 @@
+Content-Type: multipart/mixed; boundary="==BOUNDARY=="
+MIME-Version: 1.0
+
+--==BOUNDARY==
+Content-Type: text/cloud-config; charset="us-ascii"
+
+users:
+ - blah
+
+--==BOUNDARY==
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash
+echo "pvc is grand" >> /etc/motd
+
+--==BOUNDARY==--

api-daemon/provisioner/examples/userdata.yaml

@@ -0,0 +1,19 @@
+Content-Type: text/cloud-config; charset="us-ascii"
+MIME-Version: 1.0
+
+#cloud-config
+# Example user-data file. It will:
+#  1. Generate locales for us
+#  2. Update packages and install OpenSSH and sudo
+#  3. Disable the manually-enabled cloud-init target (see debootstrap_script.py)
+#  4. Reboot the system
+# You can, of course, do anything you want in here which cloud-init normally supports.
+bootcmd:
+  - "locale-gen"
+package_update: true
+packages:
+  - openssh-server
+  - sudo
+runcmd:
+  - "systemctl disable cloud-init.target"
+  - "reboot"

api-daemon/provisioner/schema.sql

@@ -0,0 +1,11 @@
+CREATE TABLE system_template (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, vcpu_count INT NOT NULL, vram_mb INT NOT NULL, serial BOOL NOT NULL, vnc BOOL NOT NULL, vnc_bind TEXT, node_limit TEXT, node_selector TEXT, node_autostart BOOL NOT NULL);
+CREATE TABLE network_template (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, mac_template TEXT);
+CREATE TABLE network (id SERIAL PRIMARY KEY, network_template INT REFERENCES network_template(id), vni INT NOT NULL);
+CREATE TABLE storage_template (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE);
+CREATE TABLE storage (id SERIAL PRIMARY KEY, storage_template INT REFERENCES storage_template(id), pool TEXT NOT NULL, disk_id TEXT NOT NULL, source_volume TEXT, disk_size_gb INT, mountpoint TEXT, filesystem TEXT, filesystem_args TEXT);
+CREATE TABLE userdata (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, userdata TEXT NOT NULL);
+CREATE TABLE script (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, script TEXT NOT NULL);
+CREATE TABLE profile (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, system_template INT REFERENCES system_template(id), network_template INT REFERENCES network_template(id), storage_template INT REFERENCES storage_template(id), userdata INT REFERENCES userdata(id), script INT REFERENCES script(id), arguments text);
+
+INSERT INTO userdata (name, userdata) VALUES ('empty', '');
+INSERT INTO script (name, script) VALUES ('empty', '');

api-daemon/pvc-api.sample.yaml

@@ -0,0 +1,80 @@
+---
+# pvc-api client configuration file example
+#
+# This configuration file specifies details for the PVC API client running on
+# this machine. Default values are not supported; the values in this sample
+# configuration are considered defaults and can be used as-is.
+#
+# Copy this example to /etc/pvc/pvc-api.conf and edit to your needs
+
+pvc:
+    # debug: Enable/disable API debug mode
+    debug: True
+    # coordinators: The list of cluster coordinator hostnames
+    coordinators:
+      - pvchv1
+      - pvchv2
+      - pvchv3
+    # api: Configuration of the API listener
+    api:
+        # listen_address: IP address(es) to listen on; use 0.0.0.0 for all interfaces
+        listen_address: "127.0.0.1"
+        # listen_port: TCP port to listen on, usually 7370
+        listen_port: "7370"
+        # authentication: Authentication and security settings
+        authentication:
+            # enabled: Enable or disable authentication (True/False)
+            enabled: False
+            # secret_key: Per-cluster secret key for API cookies; generate with uuidgen or pwgen
+            secret_key: ""
+            # tokens: a list of authentication tokens; leave as an empty list to disable authentication
+            tokens:
+                  # description: token description for management
+                - description: "testing"
+                  # token: random token for authentication; generate with uuidgen or pwgen
+                  token: ""
+        # ssl: SSL configuration
+        ssl:
+            # enabled: Enabled or disable SSL operation (True/False)
+            enabled: False
+            # cert_file: SSL certificate file
+            cert_file: ""
+            # key_file: SSL certificate key file
+            key_file: ""
+    # provisioner: Configuration of the Provisioner API listener
+    provisioner:
+        # database: Backend database configuration
+        database:
+            # host: PostgreSQL hostname, usually 'localhost'
+            host: localhost
+            # port: PostgreSQL port, invariably '5432'
+            port: 5432
+            # name: PostgreSQL database name, invariably 'pvcprov'
+            name: pvcprov
+            # user: PostgreSQL username, invariable 'pvcprov'
+            user: pvcprov
+            # pass: PostgreSQL user password, randomly generated
+            pass: pvcprov
+        # queue: Celery backend queue using the PVC Zookeeper cluster
+        queue:
+            # host: Redis hostname, usually 'localhost'
+            host: localhost
+            # port: Redis port, invariably '6279'
+            port: 6379
+            # path: Redis queue path, invariably '/0'
+            path: /0
+        # ceph_cluster: Information about the Ceph storage cluster
+        ceph_cluster:
+            # storage_hosts: The list of hosts that the Ceph monitors are valid on; if empty (the default),
+            #                uses the list of coordinators
+            storage_hosts:
+              - pvchv1
+              - pvchv2
+              - pvchv2
+            # storage_domain: The storage domain name, concatenated with the coordinators list names
+            #                 to form monitor access strings
+            storage_domain: "pvc.storage"
+            # ceph_monitor_port: The port that the Ceph monitor on each coordinator listens on
+            ceph_monitor_port: 6789
+            # ceph_storage_secret_uuid: Libvirt secret UUID for Ceph storage access
+            ceph_storage_secret_uuid: ""

api-daemon/pvc-api.service

@@ -0,0 +1,16 @@
+# Parallel Virtual Cluster API client daemon unit file
+
+[Unit]
+Description = Parallel Virtual Cluster API client daemon
+After = network-online.target
+
+[Service]
+Type = simple
+WorkingDirectory = /usr/share/pvc
+Environment = PYTHONUNBUFFERED=true
+Environment = PVC_CONFIG_FILE=/etc/pvc/pvc-api.yaml
+ExecStart = /usr/share/pvc/pvc-api.py
+Restart = on-failure
+
+[Install]
+WantedBy = multi-user.target

api-daemon/pvc-provisioner-worker.service

@@ -0,0 +1,16 @@
+# Parallel Virtual Cluster Provisioner client worker unit file
+
+[Unit]
+Description = Parallel Virtual Cluster Provisioner worker
+After = network-online.target
+
+[Service]
+Type = simple
+WorkingDirectory = /usr/share/pvc
+Environment = PYTHONUNBUFFERED=true
+Environment = PVC_CONFIG_FILE=/etc/pvc/pvc-api.yaml
+ExecStart = /usr/bin/celery worker -A pvc-api.celery --concurrency 1 --loglevel INFO
+Restart = on-failure
+
+[Install]
+WantedBy = multi-user.target

api-daemon/swagger.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>PVC Client API Documentation</title>
+    <meta charset="utf-8"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <style> body { margin: 0; padding: 0; } </style>
+  </head>
+  <body>
+    <redoc spec-url='./swagger.json' hide-loading></redoc>
+    <script src="https://rebilly.github.io/ReDoc/releases/latest/redoc.min.js"> </script>
+  </body>
+</html>