Add OVA script support

1. Ensure that system_template and script are not nullable in the DB.
2. Ensure that the CLI and API enforce the above and clean up CLI
arguments for profile add.
3. Ensure that, before uploading OVAs, a 'default_ova' provisioning
script is present.
4. Use the 'default_ova' script for new OVA uploads.
5. Ensure that OVA details are properly added to the vm_data dict in the
provisioner vmbuilder.

api-daemon/migrations/versions/88fa0d88a9f8_pvc_version_0_9_55.py

@@ -0,0 +1,38 @@
+"""PVC version 0.9.55
+
+Revision ID: 88fa0d88a9f8
+Revises: 5c2109dbbeae
+Create Date: 2022-10-06 10:33:38.784497
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '88fa0d88a9f8'
+down_revision = '5c2109dbbeae'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('profile', 'script',
+               existing_type=sa.INTEGER(),
+               nullable=False)
+    op.alter_column('profile', 'system_template',
+               existing_type=sa.INTEGER(),
+               nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('profile', 'system_template',
+               existing_type=sa.INTEGER(),
+               nullable=True)
+    op.alter_column('profile', 'script',
+               existing_type=sa.INTEGER(),
+               nullable=True)
+    # ### end Alembic commands ###

api-daemon/provisioner/examples/script/2-ova.py

@@ -0,0 +1,394 @@
+#!/usr/bin/env python3
+
+# 1-ova.py - PVC Provisioner example script for OVA profile install
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+# This script provides an example of a PVC provisioner script. It will create a
+# standard VM config suitable for the OVA profile, and prepare the disks, but do
+# no additional install tasks (noop).
+
+# This script can thus be used as an example or reference implementation of a
+# PVC provisioner script and expanded upon as required.
+# *** READ THIS SCRIPT THOROUGHLY BEFORE USING TO UNDERSTAND HOW IT WORKS. ***
+
+# A script must implement the class "VMBuilderScript" which extends "VMBuilder",
+# providing the 5 functions indicated. Detailed explanation of the role of each
+# function is provided in context of the example; see the other examples for
+# more potential uses.
+
+# Within the VMBuilderScript class, several common variables are exposed through
+# the parent VMBuilder class:
+#  self.vm_name: The name of the VM from PVC's perspective
+#  self.vm_id: The VM ID (numerical component of the vm_name) from PVC's perspective
+#  self.vm_uuid: An automatically-generated UUID for the VM
+#  self.vm_profile: The PVC provisioner profile name used for the VM
+#  self.vm_data: A dictionary of VM data collected by the provisioner; as an example:
+#    {
+#      "ceph_monitor_list": [
+#        "hv1.pvcstorage.tld",
+#        "hv2.pvcstorage.tld",
+#        "hv3.pvcstorage.tld"
+#      ],
+#      "ceph_monitor_port": "6789",
+#      "ceph_monitor_secret": "96721723-8650-4a72-b8f6-a93cd1a20f0c",
+#      "mac_template": null,
+#      "networks": [
+#        {
+#          "eth_bridge": "vmbr1001",
+#          "id": 72,
+#          "network_template": 69,
+#          "vni": "1001"
+#        },
+#        {
+#          "eth_bridge": "vmbr101",
+#          "id": 73,
+#          "network_template": 69,
+#          "vni": "101"
+#        }
+#      ],
+#      "script": [contents of this file]
+#      "script_arguments": {
+#        "deb_mirror": "http://ftp.debian.org/debian",
+#        "deb_release": "bullseye"
+#      },
+#      "system_architecture": "x86_64",
+#      "system_details": {
+#        "id": 78,
+#        "migration_method": "live",
+#        "name": "small",
+#        "node_autostart": false,
+#        "node_limit": null,
+#        "node_selector": null,
+#        "ova": null,
+#        "serial": true,
+#        "vcpu_count": 2,
+#        "vnc": false,
+#        "vnc_bind": null,
+#        "vram_mb": 2048
+#      },
+#      "volumes": [
+#        {
+#          "disk_id": "sda",
+#          "disk_size_gb": 4,
+#          "filesystem": "ext4",
+#          "filesystem_args": "-L=root",
+#          "id": 9,
+#          "mountpoint": "/",
+#          "pool": "vms",
+#          "source_volume": null,
+#          "storage_template": 67
+#        },
+#        {
+#          "disk_id": "sdb",
+#          "disk_size_gb": 4,
+#          "filesystem": "ext4",
+#          "filesystem_args": "-L=var",
+#          "id": 10,
+#          "mountpoint": "/var",
+#          "pool": "vms",
+#          "source_volume": null,
+#          "storage_template": 67
+#        },
+#        {
+#          "disk_id": "sdc",
+#          "disk_size_gb": 4,
+#          "filesystem": "ext4",
+#          "filesystem_args": "-L=log",
+#          "id": 11,
+#          "mountpoint": "/var/log",
+#          "pool": "vms",
+#          "source_volume": null,
+#          "storage_template": 67
+#        }
+#      ]
+#    }
+#
+# Any other information you may require must be obtained manually.
+
+# WARNING:
+#
+# For safety reasons, the script runs in a modified chroot. It will have full access to
+# the entire / (root partition) of the hypervisor, but read-only. In addition it has
+# access to /dev, /sys, /run, and a fresh /tmp to write to; use /tmp/target (as
+# convention) as the destination for any mounting of volumes and installation.
+# Of course, in addition to this safety, it is VERY IMPORTANT to be aware that this
+# script runs AS ROOT ON THE HYPERVISOR SYSTEM. You should never allow arbitrary,
+# untrusted users the ability to add provisioning scripts even with this safeguard,
+# since they could still do destructive things to /dev and the like!
+
+
+# This import is always required here, as VMBuilder is used by the VMBuilderScript class
+# and ProvisioningError is the primary exception that should be raised within the class.
+from pvcapid.vmbuilder import VMBuilder, ProvisioningError
+
+
+# The VMBuilderScript class must be named as such, and extend VMBuilder.
+class VMBuilderScript(VMBuilder):
+    def setup(self):
+        """
+        setup(): Perform special setup steps or validation before proceeding
+
+        Validate that we're actually an OVA profile.
+        """
+
+        if self.vm_data.get("ova_details") is None:
+            raise ProvisioningError(
+                "Attempting to provision non-OVA profile with OVA script."
+            )
+
+    def create(self):
+        """
+        create(): Create the VM libvirt schema definition
+
+        This step *must* return a fully-formed Libvirt XML document as a string or the
+        provisioning task will fail.
+
+        This example leverages the built-in libvirt_schema objects provided by PVC; these
+        can be used as-is, or replaced with your own schema(s) on a per-script basis.
+        """
+
+        # Run any imports first
+        import pvcapid.libvirt_schema as libvirt_schema
+        import datetime
+        import random
+
+        # Create the empty schema document that we will append to and return at the end
+        schema = ""
+
+        # Prepare a description based on the VM profile
+        description = f"PVC provisioner @ {datetime.datetime.now()}, profile '{self.vm_profile}', OVA '{self.vm_data['ova_details']['name']}'"
+
+        # Format the header
+        schema += libvirt_schema.libvirt_header.format(
+            vm_name=self.vm_name,
+            vm_uuid=self.vm_uuid,
+            vm_description=description,
+            vm_memory=self.vm_data["system_details"]["vram_mb"],
+            vm_vcpus=self.vm_data["system_details"]["vcpu_count"],
+            vm_architecture=self.vm_data["system_architecture"],
+        )
+
+        # Add the disk devices
+        monitor_list = self.vm_data["ceph_monitor_list"]
+        monitor_port = self.vm_data["ceph_monitor_port"]
+        monitor_secret = self.vm_data["ceph_monitor_secret"]
+
+        for volume in self.vm_data["volumes"]:
+            schema += libvirt_schema.devices_disk_header.format(
+                ceph_storage_secret=monitor_secret,
+                disk_pool=volume["pool"],
+                vm_name=self.vm_name,
+                disk_id=volume["disk_id"],
+            )
+            for monitor in monitor_list:
+                schema += libvirt_schema.devices_disk_coordinator.format(
+                    coordinator_name=monitor,
+                    coordinator_ceph_mon_port=monitor_port,
+                )
+            schema += libvirt_schema.devices_disk_footer
+
+        # Add the special vhostmd device for hypervisor information inside the VM
+        schema += libvirt_schema.devices_vhostmd
+
+        # Add the network devices
+        network_id = 0
+        for network in self.vm_data["networks"]:
+            vm_id_hex = "{:x}".format(int(self.vm_id % 16))
+            net_id_hex = "{:x}".format(int(network_id % 16))
+
+            if self.vm_data.get("mac_template") is not None:
+                mac_prefix = "52:54:01"
+                macgen_template = self.vm_data["mac_template"]
+                eth_macaddr = macgen_template.format(
+                    prefix=mac_prefix, vmid=vm_id_hex, netid=net_id_hex
+                )
+            else:
+                mac_prefix = "52:54:00"
+                random_octet_A = "{:x}".format(random.randint(16, 238))
+                random_octet_B = "{:x}".format(random.randint(16, 238))
+                random_octet_C = "{:x}".format(random.randint(16, 238))
+
+                macgen_template = "{prefix}:{octetA}:{octetB}:{octetC}"
+                eth_macaddr = macgen_template.format(
+                    prefix=mac_prefix,
+                    octetA=random_octet_A,
+                    octetB=random_octet_B,
+                    octetC=random_octet_C,
+                )
+
+            schema += libvirt_schema.devices_net_interface.format(
+                eth_macaddr=eth_macaddr,
+                eth_bridge=network["eth_bridge"],
+            )
+
+            network_id += 1
+
+        # Add default devices
+        schema += libvirt_schema.devices_default
+
+        # Add serial device
+        if self.vm_data["system_details"]["serial"]:
+            schema += libvirt_schema.devices_serial.format(vm_name=self.vm_name)
+
+        # Add VNC device
+        if self.vm_data["system_details"]["vnc"]:
+            if self.vm_data["system_details"]["vnc_bind"]:
+                vm_vnc_bind = self.vm_data["system_details"]["vnc_bind"]
+            else:
+                vm_vnc_bind = "127.0.0.1"
+
+            vm_vncport = 5900
+            vm_vnc_autoport = "yes"
+
+            schema += libvirt_schema.devices_vnc.format(
+                vm_vncport=vm_vncport,
+                vm_vnc_autoport=vm_vnc_autoport,
+                vm_vnc_bind=vm_vnc_bind,
+            )
+
+        # Add SCSI controller
+        schema += libvirt_schema.devices_scsi_controller
+
+        # Add footer
+        schema += libvirt_schema.libvirt_footer
+
+        return schema
+
+    def prepare(self):
+        """
+        prepare(): Prepare any disks/volumes for the install() step
+        """
+
+        # Run any imports first
+        from pvcapid.vmbuilder import open_zk
+        from pvcapid.Daemon import config
+        import daemon_lib.common as pvc_common
+        import daemon_lib.ceph as pvc_ceph
+
+        # First loop: Create the destination disks
+        for volume in self.vm_data["volumes"]:
+            with open_zk(config) as zkhandler:
+                success, message = pvc_ceph.add_volume(
+                    zkhandler,
+                    volume["pool"],
+                    f"{self.vm_name}_{volume['disk_id']}",
+                    f"{volume['disk_size_gb']}G",
+                )
+                print(message)
+                if not success:
+                    raise ProvisioningError(
+                        f"Failed to create volume '{volume['disk_id']}'."
+                    )
+
+        # Second loop: Map the destination disks
+        for volume in self.vm_data["volumes"]:
+            dst_volume_name = f"{self.vm_name}_{volume['disk_id']}"
+            dst_volume = f"{volume['pool']}/{dst_volume_name}"
+
+            with open_zk(config) as zkhandler:
+                success, message = pvc_ceph.map_volume(
+                    zkhandler,
+                    volume["pool"],
+                    dst_volume_name,
+                )
+                print(message)
+                if not success:
+                    raise ProvisioningError(f"Failed to map volume '{dst_volume}'.")
+
+        # Third loop: Map the source disks
+        for volume in self.vm_data["volumes"]:
+            src_volume_name = volume["volume_name"]
+            src_volume = f"{volume['pool']}/{src_volume_name}"
+
+            with open_zk(config) as zkhandler:
+                success, message = pvc_ceph.map_volume(
+                    zkhandler,
+                    volume["pool"],
+                    src_volume_name,
+                )
+                print(message)
+                if not success:
+                    raise ProvisioningError(f"Failed to map volume '{src_volume}'.")
+
+        # Fourth loop: Convert the source (usually VMDK) volume to the raw destination volume
+        for volume in self.vm_data["volumes"]:
+            src_volume_name = volume["volume_name"]
+            src_volume = f"{volume['pool']}/{src_volume_name}"
+            src_devpath = f"/dev/rbd/{src_volume}"
+            dst_volume_name = f"{self.vm_name}_{volume['disk_id']}"
+            dst_volume = f"{volume['pool']}/{dst_volume_name}"
+            dst_devpath = f"/dev/rbd/{dst_volume}"
+
+            retcode, stdout, stderr = pvc_common.run_os_command(
+                f"qemu-img convert -C -f {volume['volume_format']} -O raw {src_devpath} {dst_devpath}"
+            )
+            if retcode:
+                raise ProvisioningError(
+                    f"Failed to convert {volume['volume_format']} volume '{src_volume}' to raw volume '{dst_volume}' with qemu-img: {stderr}"
+                )
+
+    def install(self):
+        """
+        install(): Perform the installation
+
+        Noop for OVA deploys as no further tasks are performed.
+        """
+
+        pass
+
+    def cleanup(self):
+        """
+        cleanup(): Perform any cleanup required due to prepare()/install()
+
+        This function is also called if there is ANY exception raised in the prepare()
+        or install() steps. While this doesn't mean you shouldn't or can't raise exceptions
+        here, be warned that doing so might cause loops. Do this only if you really need to.
+        """
+
+        for volume in list(reversed(self.vm_data["volumes"])):
+            src_volume_name = volume["volume_name"]
+            src_volume = f"{volume['pool']}/{src_volume_name}"
+            src_devpath = f"/dev/rbd/{src_volume}"
+
+            with open_zk(config) as zkhandler:
+                success, message = pvc_ceph.unmap_volume(
+                    zkhandler,
+                    volume["pool"],
+                    src_volume_name,
+                )
+                if not success:
+                    raise ProvisioningError(
+                        f"Failed to unmap source volume '{src_volume_name}': {message}"
+                    )
+
+        for volume in list(reversed(self.vm_data["volumes"])):
+            dst_volume_name = f"{self.vm_name}_{volume['disk_id']}"
+            dst_volume = f"{volume['pool']}/{dst_volume_name}"
+            dst_devpath = f"/dev/rbd/{dst_volume}"
+
+            with open_zk(config) as zkhandler:
+                success, message = pvc_ceph.unmap_volume(
+                    zkhandler,
+                    volume["pool"],
+                    dst_volume_name,
+                )
+                if not success:
+                    raise ProvisioningError(
+                        f"Failed to unmap destination volume '{dst_volume_name}': {message}"
+                    )

api-daemon/provisioner/examples/script/3-ova.py

@@ -1,604 +0,0 @@
-#!/usr/bin/env python3
-
-# 3-ova.py - PVC Provisioner example script for OVA install
-# Part of the Parallel Virtual Cluster (PVC) system
-#
-#    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, version 3.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-#
-###############################################################################
-
-# This script provides an example of a PVC provisioner script. It will create a
-# custom VM config based on an OVA profile.
-
-# This script can thus be used as an example or reference implementation of a
-# PVC provisioner script and expanded upon as required; this specific script is
-# also hard-coded into a normal PVC provisioner system.
-
-# The script must implement the class "VMBuilderScript" which extens "VMBuilder",
-# providing the 5 functions indicated. Detailed explanation of the role of each
-# function is provided.
-
-# Within the VMBuilderScript class, several common variables are exposed:
-#  self.vm_name: The name of the VM from PVC's perspective
-#  self.vm_id: The VM ID (numerical component of the vm_name) from PVC's perspective
-#  self.vm_uuid: An automatically-generated UUID for the VM
-#  self.vm_profile: The PVC provisioner profile name used for the VM
-#  self.vm-data: A dictionary of VM data collected by the provisioner; an example:
-#    {
-#      "ceph_monitor_list": [
-#        "hv1.pvcstorage.tld",
-#        "hv2.pvcstorage.tld",
-#        "hv3.pvcstorage.tld"
-#      ],
-#      "ceph_monitor_port": "6789",
-#      "ceph_monitor_secret": "96721723-8650-4a72-b8f6-a93cd1a20f0c",
-#      "mac_template": null,
-#      "networks": [
-#        {
-#          "eth_bridge": "vmbr1001",
-#          "id": 72,
-#          "network_template": 69,
-#          "vni": "1001"
-#        },
-#        {
-#          "eth_bridge": "vmbr101",
-#          "id": 73,
-#          "network_template": 69,
-#          "vni": "101"
-#        }
-#      ],
-#      "script": [contents of this file]
-#      "script_arguments": {
-#        "deb_mirror": "http://ftp.debian.org/debian",
-#        "deb_release": "bullseye"
-#      },
-#      "system_architecture": "x86_64",
-#      "system_details": {
-#        "id": 78,
-#        "migration_method": "live",
-#        "name": "small",
-#        "node_autostart": false,
-#        "node_limit": null,
-#        "node_selector": null,
-#        "ova": null,
-#        "serial": true,
-#        "vcpu_count": 2,
-#        "vnc": false,
-#        "vnc_bind": null,
-#        "vram_mb": 2048
-#      },
-#      "volumes": [
-#        {
-#          "disk_id": "sda",
-#          "disk_size_gb": 4,
-#          "filesystem": "ext4",
-#          "filesystem_args": "-L=root",
-#          "id": 9,
-#          "mountpoint": "/",
-#          "pool": "vms",
-#          "source_volume": null,
-#          "storage_template": 67
-#        },
-#        {
-#          "disk_id": "sdb",
-#          "disk_size_gb": 4,
-#          "filesystem": "ext4",
-#          "filesystem_args": "-L=var",
-#          "id": 10,
-#          "mountpoint": "/var",
-#          "pool": "vms",
-#          "source_volume": null,
-#          "storage_template": 67
-#        },
-#        {
-#          "disk_id": "sdc",
-#          "disk_size_gb": 4,
-#          "filesystem": "ext4",
-#          "filesystem_args": "-L=log",
-#          "id": 11,
-#          "mountpoint": "/var/log",
-#          "pool": "vms",
-#          "source_volume": null,
-#          "storage_template": 67
-#        }
-#      ]
-#    }
-
-
-# Run any imports first
-import datetime
-import random
-import pvcapid.libvirt_schema as libvirt_schema
-
-
-class VMBuilderScript(VMBuilder):
-    def setup(self):
-        """
-        setup(): Perform special setup steps or validation before proceeding
-        """
-
-        # Ensure we have debootstrap intalled on the provisioner system
-        retcode, stdout, stderr = pvc_common.run_os_command(f"which debootstrap")
-        if retcode:
-            raise ProvisioningError("Failed to find critical dependency: debootstrap")
-
-    def create(self):
-        """
-        create(): Create the VM libvirt schema definition
-
-        This step *must* return a fully-formed Libvirt XML document as a string.
-
-        This example leverages the built-in libvirt_schema objects provided by PVC; these
-        can be used as-is, or replaced with your own schema(s) on a per-script basis.
-        """
-
-        schema = ""
-
-        # Prepare a description based on the VM profile
-        description = (
-            f"PVC provisioner @ {datetime.datetime.now()}, profile '{self.vm_profile}'"
-        )
-
-        # Format the header
-        schema += libvirt_schema.libvirt_header.format(
-            vm_name=self.vm_name,
-            vm_uuid=self.vm_uuid,
-            vm_description=description,
-            vm_memory=self.vm_data["system_details"]["vram_mb"],
-            vm_vcpus=self.vm_data["system_details"]["vcpu_count"],
-            vm_architecture=self.vm_data["system_architecture"],
-        )
-
-        # Add the disk devices
-        monitor_list = self.vm_data["ceph_monitor_list"]
-        monitor_port = self.vm_data["ceph_monitor_port"]
-        monitor_secret = self.vm_data["ceph_monitor_secret"]
-
-        for volume in self.vm_data["volumes"]:
-            schema += libvirt_schema.devices_disk_header.format(
-                ceph_storage_secret=monitor_secret,
-                disk_pool=volume["pool"],
-                vm_name=self.vm_name,
-                disk_id=volume["disk_id"],
-            )
-            for monitor in monitor_list:
-                schema += libvirt_schema.devices_disk_coordinator.format(
-                    coordinator_name=monitor,
-                    coordinator_ceph_mon_port=monitor_port,
-                )
-            schema += libvirt_schema.devices_disk_footer
-
-        # Add the special vhostmd device for hypervisor information inside the VM
-        schema += libvirt_schema.devices_vhostmd
-
-        # Add the network devices
-        network_id = 0
-        for network in self.vm_data["networks"]:
-            vm_id_hex = "{:x}".format(int(vm_id % 16))
-            net_id_hex = "{:x}".format(int(network_id % 16))
-
-            if self.vm_data.get("mac_template") is not None:
-                mac_prefix = "52:54:01"
-                macgen_template = self.vm_data["mac_template"]
-                eth_macaddr = macgen_template.format(
-                    prefix=mac_prefix, vmid=vm_id_hex, netid=net_id_hex
-                )
-            else:
-                mac_prefix = "52:54:00"
-                random_octet_A = "{:x}".format(random.randint(16, 238))
-                random_octet_B = "{:x}".format(random.randint(16, 238))
-                random_octet_C = "{:x}".format(random.randint(16, 238))
-
-                macgen_template = "{prefix}:{octetA}:{octetB}:{octetC}"
-                eth_macaddr = macgen_template.format(
-                    prefix=mac_prefix,
-                    octetA=random_octet_A,
-                    octetB=random_octet_B,
-                    octetC=random_octet_C,
-                )
-
-            schema += libvirt_schema.devices_net_interface.format(
-                eth_macaddr=eth_macaddr,
-                eth_bridge=eth_bridge,
-            )
-
-            network_id += 1
-
-        # Add default devices
-        schema += libvirt_schema.devices_default
-
-        # Add serial device
-        if self.vm_data["system_details"]["serial"]:
-            schema += libvirt_schema.devices_serial.format(vm_name=self.vm_name)
-
-        # Add VNC device
-        if self.vm_data["system_details"]["vnc"]:
-            if self.vm_data["system_details"]["vnc_bind"]:
-                vm_vnc_bind = self.vm_data["system_details"]["vnc_bind"]
-            else:
-                vm_vnc_bind = "127.0.0.1"
-
-            vm_vncport = 5900
-            vm_vnc_autoport = "yes"
-
-            schema += libvirt_schema.devices_vnc.format(
-                vm_vncport=vm_vncport,
-                vm_vnc_autoport=vm_vnc_autoport,
-                vm_vnc_bind=vm_vnc_bind,
-            )
-
-        # Add SCSI controller
-        schema += libvirt_schema.devices_scsi_controller
-
-        # Add footer
-        schema += libvirt_schema.libvirt_footer
-
-        return schema
-
-    def prepare(self):
-        """
-        prepare(): Prepare any disks/volumes for the install() step
-
-        This function should use the various exposed PVC commands as indicated to create
-        block devices and map them to the host.
-        """
-
-        # First loop: Create the disks, either by cloning (pvc_ceph.clone_volume), or by
-        # new creation (pvc_ceph.add_volume).
-        for volume in self.vm_data["volumes"]:
-            if volume.get("source_volume") is not None:
-                with open_zk(config) as zkhandler:
-                    success, message = pvc_ceph.clone_volume(
-                        zkhandler,
-                        volume["pool"],
-                        volume["source_volume"],
-                        f"{self.vm_name}_{volume['disk_id']}",
-                    )
-                    print(message)
-                    if not success:
-                        raise ProvisioningError(
-                            f"Failed to clone volume '{volume['source_volume']}' to '{volume['disk_id']}'."
-                        )
-            else:
-                with open_zk(config) as zkhandler:
-                    success, message = pvc_ceph.add_volume(
-                        zkhandler,
-                        volume["pool"],
-                        f"{self.vm_name}_{volume['disk_id']}",
-                        f"{volume['disk_size_gb']}G",
-                    )
-                    print(message)
-                    if not success:
-                        raise ProvisioningError(
-                            f"Failed to create volume '{volume['disk_id']}'."
-                        )
-
-        # Second loop: Map the disks to the local system
-        for volume in self.vm_data["volumes"]:
-            dst_volume_name = f"{self.vm_name}_{volume['disk_id']}"
-            dst_volume = f"{volume['pool']}/{dst_volume_name}"
-
-            with open_zk(config) as zkhandler:
-                success, message = pvc_ceph.map_volume(
-                    zkhandler,
-                    volume["pool"],
-                    dst_volume_name,
-                )
-                print(message)
-                if not retcode:
-                    raise ProvisioningError(f"Failed to map volume '{dst_volume}'.")
-
-        # Third loop: Create filesystems on the volumes
-        for volume in self.vm_data["volumes"]:
-            dst_volume_name = f"{self.vm_name}_{volume['disk_id']}"
-            dst_volume = f"{volume['pool']}/{dst_volume_name}"
-
-            if volume.get("source_volume") is not None:
-                continue
-
-            if volume.get("filesystem") is None:
-                continue
-
-            filesystem_args_list = list()
-            for arg in volume["filesystem_args"].split():
-                arg_entry, *arg_data = arg.split("=")
-                arg_data = "=".join(arg_data)
-                filesystem_args_list.append(arg_entry)
-                filesystem_args_list.append(arg_data)
-            filesystem_args = " ".join(filesystem_args_list)
-
-            if volume["filesystem"] == "swap":
-                retcode, stdout, stderr = pvc_common.run_os_command(
-                    f"mkswap -f /dev/rbd/{dst_volume}"
-                )
-                if retcode:
-                    raise ProvisioningError(
-                        f"Failed to create swap on '{dst_volume}': {stderr}"
-                    )
-            else:
-                retcode, stdout, stderr = pvc_common.run_os_command(
-                    f"mkfs.{volume['filesystem']} {filesystem_args} /dev/rbd/{dst_volume}"
-                )
-                if retcode:
-                    raise ProvisioningError(
-                        f"Faield to create {volume['filesystem']} file on '{dst_volume}': {stderr}"
-                    )
-
-            print(stdout)
-
-        # Create a temporary directory to use during install
-        temp_dir = "/tmp/target"
-        if not os.exists(temp_dir):
-            os.mkdir(temp_dir)
-
-        # Fourth loop: Mount the volumes to a set of temporary directories
-        for volume in self.vm_data["volumes"]:
-            dst_volume_name = f"{self.vm_name}_{volume['disk_id']}"
-            dst_volume = f"{volume['pool']}/{dst_volume_name}"
-
-            if volume.get("source_volume") is not None:
-                continue
-
-            if volume.get("filesystem") is None:
-                continue
-
-            mapped_dst_volume = f"/dev/rbd/{dst_volume}"
-
-            mount_path = f"{temp_dir}/{volume['mountpoint']}"
-
-            if not os.exists(mount_path):
-                os.mkdir(mount_path)
-
-            # Mount filesystem
-            retcode, stdout, stderr = pvc_common.run_os_command(
-                f"mount {mapped_dst_volume} {mount_path}"
-            )
-            if retcode:
-                raise ProvisioningError(
-                    f"Failed to mount '{mapped_dst_volume}' on '{mount_path}': {stderr}"
-                )
-
-    def install(self):
-        """
-        install(): Perform the installation
-
-        Since this is a noop example, this step does nothing, aside from getting some
-        arguments for demonstration.
-        """
-
-        # The directory we mounted things on earlier during prepare()
-        temporary_directory = "/tmp/target"
-
-        # Use these convenient aliases for later (avoiding lots of "self.vm_data" everywhere)
-        vm_name = self.vm_name
-        disks = self.vm_data["disks"]
-        networks = self.vm_data["networks"]
-
-        # Parse these arguments out of self.vm_data["script_arguments"]
-        if self.vm_data["script_arguments"].get("deb_release") is not None:
-            deb_release = self.vm_data["script_arguments"].get("deb_release")
-        else:
-            deb_release = "stable"
-
-        if self.vm_data["script_arguments"].get("deb_mirror") is not None:
-            deb_mirror = self.vm_data["script_arguments"].get("deb_mirror")
-        else:
-            deb_mirror = "http://ftp.debian.org/debian"
-
-        if self.vm_data["script_arguments"].get("deb_packages") is not None:
-            deb_packages = (
-                self.vm_data["script_arguments"].get("deb_packages").split(",")
-            )
-        else:
-            deb_packages = [
-                "linux-image-amd64",
-                "grub-pc",
-                "cloud-init",
-                "python3-cffi-backend",
-                "wget",
-            ]
-
-        # We need to know our root disk
-        root_disk = None
-        for disk in disks:
-            if disk["mountpoint"] == "/":
-                root_disk = disk
-        if not root_disk:
-            raise ProvisioningError("Failed to find root disk in disks list")
-
-        # Perform a deboostrap installation
-        os.system(
-            "debootstrap --include={pkgs} {suite} {target} {mirror}".format(
-                suite=deb_release,
-                target=temporary_directory,
-                mirror=deb_mirror,
-                pkgs=",".join(deb_packages),
-            )
-        )
-
-        # Bind mount the devfs
-        os.system("mount --bind /dev {}/dev".format(temporary_directory))
-
-        # Create an fstab entry for each disk
-        fstab_file = "{}/etc/fstab".format(temporary_directory)
-        # The disk ID starts at zero and increments by one for each disk in the fixed-order
-        # disk list. This lets us work around the insanity of Libvirt IDs not matching guest IDs,
-        # while still letting us have some semblance of control here without enforcing things
-        # like labels. It increments in the for loop below at the end of each iteration, and is
-        # used to craft a /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0-0-0-X device ID
-        # which will always match the correct order from Libvirt (unlike sdX/vdX names).
-        disk_id = 0
-        for disk in disks:
-            # We assume SSD-based/-like storage, and dislike atimes
-            options = "defaults,discard,noatime,nodiratime"
-
-            # The root, var, and log volumes have specific values
-            if disk["mountpoint"] == "/":
-                root_disk["scsi_id"] = disk_id
-                dump = 0
-                cpass = 1
-            elif disk["mountpoint"] == "/var" or disk["mountpoint"] == "/var/log":
-                dump = 0
-                cpass = 2
-            else:
-                dump = 0
-                cpass = 0
-
-            # Append the fstab line
-            with open(fstab_file, "a") as fh:
-                data = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0-0-0-{disk} {mountpoint} {filesystem} {options} {dump} {cpass}\n".format(
-                    disk=disk_id,
-                    mountpoint=disk["mountpoint"],
-                    filesystem=disk["filesystem"],
-                    options=options,
-                    dump=dump,
-                    cpass=cpass,
-                )
-                fh.write(data)
-
-            # Increment the disk_id
-            disk_id += 1
-
-        # Write the hostname
-        hostname_file = "{}/etc/hostname".format(temporary_directory)
-        with open(hostname_file, "w") as fh:
-            fh.write("{}".format(vm_name))
-
-        # Fix the cloud-init.target since it's broken
-        cloudinit_target_file = "{}/etc/systemd/system/cloud-init.target".format(
-            temporary_directory
-        )
-        with open(cloudinit_target_file, "w") as fh:
-            data = """[Install]
-WantedBy=multi-user.target
-[Unit]
-Description=Cloud-init target
-After=multi-user.target
-"""
-            fh.write(data)
-
-        # NOTE: Due to device ordering within the Libvirt XML configuration, the first Ethernet interface
-        #       will always be on PCI bus ID 2, hence the name "ens2".
-        # Write a DHCP stanza for ens2
-        ens2_network_file = "{}/etc/network/interfaces.d/ens2".format(
-            temporary_directory
-        )
-        with open(ens2_network_file, "w") as fh:
-            data = """auto ens2
-iface ens2 inet dhcp
-"""
-            fh.write(data)
-
-        # Write the DHCP config for ens2
-        dhclient_file = "{}/etc/dhcp/dhclient.conf".format(temporary_directory)
-        with open(dhclient_file, "w") as fh:
-            data = (
-                """# DHCP client configuration
-# Written by the PVC provisioner
-option rfc3442-classless-static-routes code 121 = array of unsigned integer 8;
-interface "ens2" {
-"""
-                + """        send fqdn.fqdn = "{hostname}";
-        send host-name = "{hostname}";
-    """.format(
-                    hostname=vm_name
-                )
-                + """        request subnet-mask, broadcast-address, time-offset, routers,
-                domain-name, domain-name-servers, domain-search, host-name,
-                dhcp6.name-servers, dhcp6.domain-search, dhcp6.fqdn, dhcp6.sntp-servers,
-                netbios-name-servers, netbios-scope, interface-mtu,
-                rfc3442-classless-static-routes, ntp-servers;
-}
-"""
-            )
-            fh.write(data)
-
-        # Write the GRUB configuration
-        grubcfg_file = "{}/etc/default/grub".format(temporary_directory)
-        with open(grubcfg_file, "w") as fh:
-            data = """# Written by the PVC provisioner
-GRUB_DEFAULT=0
-GRUB_TIMEOUT=1
-GRUB_DISTRIBUTOR="PVC Virtual Machine"
-GRUB_CMDLINE_LINUX_DEFAULT="root=/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0-0-0-{root_disk} console=tty0 console=ttyS0,115200n8"
-GRUB_CMDLINE_LINUX=""
-GRUB_TERMINAL=console
-GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
-GRUB_DISABLE_LINUX_UUID=false
-""".format(
-                root_disk=root_disk["scsi_id"]
-            )
-            fh.write(data)
-
-        # Chroot, do some in-root tasks, then exit the chroot
-        with chroot_target(temporary_directory):
-            # Install and update GRUB
-            os.system(
-                "grub-install --force /dev/rbd/{}/{}_{}".format(
-                    root_disk["pool"], vm_name, root_disk["disk_id"]
-                )
-            )
-            os.system("update-grub")
-            # Set a really dumb root password [TEMPORARY]
-            os.system("echo root:test123 | chpasswd")
-            # Enable cloud-init target on (first) boot
-            # NOTE: Your user-data should handle this and disable it once done, or things get messy.
-            #       That cloud-init won't run without this hack seems like a bug... but even the official
-            #       Debian cloud images are affected, so who knows.
-            os.system("systemctl enable cloud-init.target")
-
-        # Unmount the bound devfs
-        os.system("umount {}/dev".format(temporary_directory))
-
-    def cleanup(self):
-        """
-        cleanup(): Perform any cleanup required due to prepare()/install()
-
-        It is important to now reverse *all* steps taken in those functions that might
-        need cleanup before teardown of the overlay chroot environment.
-        """
-
-        temp_dir = "/tmp/target"
-
-        for volume in list(reversed(self.vm_data["volumes"])):
-            dst_volume_name = f"{self.vm_name}_{volume['disk_id']}"
-            dst_volume = f"{volume['pool']}/{dst_volume_name}"
-            mapped_dst_volume = f"/dev/rbd/{dst_volume}"
-            mount_path = f"{temp_dir}/{volume['mountpoint']}"
-
-            if (
-                volume.get("source_volume") is None
-                and volume.get("filesystem") is not None
-            ):
-                # Unmount filesystem
-                retcode, stdout, stderr = pvc_common.run_os_command(
-                    f"umount {mount_path}"
-                )
-                if retcode:
-                    raise ProvisioningError(
-                        f"Failed to unmount '{mapped_dst_volume}' on '{mount_path}': {stderr}"
-                    )
-
-                # Unmap volume
-                with open_zk(config) as zkhandler:
-                    success, message = pvc_ceph.unmap_volume(
-                        zkhandler,
-                        volume["pool"],
-                        dst_volume_name,
-                    )
-                    if not success:
-                        raise ProvisioningError(
-                            f"Failed to unmap '{mapped_dst_volume}': {stderr}"
-                        )

api-daemon/pvcapid/flaskapi.py

@@ -7352,11 +7352,19 @@ class API_Provisioner_Profile_Root(Resource):
                 "required": True,
                 "helptext": "A profile type must be specified.",
             },
-            {"name": "system_template"},
+            {
+                "name": "system_template",
+                "required": True,
+                "helptext": "A system_template must be specified.",
+            },
             {"name": "network_template"},
             {"name": "storage_template"},
             {"name": "userdata"},
-            {"name": "script"},
+            {
+                "name": "script",
+                "required": True,
+                "helptext": "A script must be specified.",
+            },
             {"name": "ova"},
             {"name": "arg", "action": "append"},
         ]
@@ -7385,12 +7393,12 @@ class API_Provisioner_Profile_Root(Resource):
           - in: query
             name: script
             type: string
-            required: false
+            required: true
             description: Script name
           - in: query
             name: system_template
             type: string
-            required: false
+            required: true
             description: System template name
           - in: query
             name: network_template
@@ -7473,11 +7481,19 @@ class API_Provisioner_Profile_Element(Resource):
                 "required": True,
                 "helptext": "A profile type must be specified.",
             },
-            {"name": "system_template"},
+            {
+                "name": "system_template",
+                "required": True,
+                "helptext": "A system_template must be specified.",
+            },
             {"name": "network_template"},
             {"name": "storage_template"},
             {"name": "userdata"},
-            {"name": "script"},
+            {
+                "name": "script",
+                "required": True,
+                "helptext": "A script must be specified.",
+            },
             {"name": "ova"},
             {"name": "arg", "action": "append"},
         ]
@@ -7511,17 +7527,17 @@ class API_Provisioner_Profile_Element(Resource):
           - in: query
             name: network_template
             type: string
-            required: true
+            required: false
             description: Network template name
           - in: query
             name: storage_template
             type: string
-            required: true
+            required: false
             description: Storage template name
           - in: query
             name: userdata
             type: string
-            required: true
+            required: false
             description: Userdata template name
           - in: query
             name: ova

api-daemon/pvcapid/models.py

@@ -230,11 +230,13 @@ class DBProfile(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     name = db.Column(db.Text, nullable=False, unique=True)
     profile_type = db.Column(db.Text, nullable=False)
-    system_template = db.Column(db.Integer, db.ForeignKey("system_template.id"))
+    system_template = db.Column(
+        db.Integer, db.ForeignKey("system_template.id"), nullable=False
+    )
     network_template = db.Column(db.Integer, db.ForeignKey("network_template.id"))
     storage_template = db.Column(db.Integer, db.ForeignKey("storage_template.id"))
     userdata = db.Column(db.Integer, db.ForeignKey("userdata.id"))
-    script = db.Column(db.Integer, db.ForeignKey("script.id"))
+    script = db.Column(db.Integer, db.ForeignKey("script.id"), nullable=False)
     ova = db.Column(db.Integer, db.ForeignKey("ova.id"))
     arguments = db.Column(db.Text)
 

api-daemon/pvcapid/ova.py

@@ -168,6 +168,15 @@ def delete_ova(zkhandler, name):
 
 @ZKConnection(config)
 def upload_ova(zkhandler, pool, name, ova_size):
+    # Check that we have a default_ova provisioning script
+    _, retcode = provisioner.list_script("default_ova", is_fuzzy=False)
+    if retcode != "200":
+        output = {
+            "message": "Did not find a 'default_ova' provisioning script. Please add one with that name, either the example from '/usr/share/pvc/provisioner/examples/script/2-ova.py' or a custom one, before uploading OVAs."
+        }
+        retcode = 400
+        return output, retcode
+
     ova_archive = None
 
     # Cleanup function
@@ -402,7 +411,7 @@ def upload_ova(zkhandler, pool, name, ova_size):
         None,
         None,
         userdata=None,
-        script=None,
+        script="default_ova",
         ova=name,
         arguments=None,
     )

api-daemon/pvcapid/vmbuilder.py

@@ -227,11 +227,6 @@ def create_vm(
         else:
             vm_data["script_arguments"] = []
 
-        if profile_data.get("profile_type") == "ova":
-            is_ova_install = True
-        else:
-            is_ova_install = False
-
         # Get the system details
         query = "SELECT * FROM system_template WHERE id = %s"
         args = (profile_data["system_template"],)
@@ -282,6 +277,8 @@ def create_vm(
             vm_data["script"] = db_row.get("script")
         else:
             vm_data["script"] = None
+
+        if profile_data.get("profile_type") == "ova":
             query = "SELECT * FROM ova WHERE id = %s"
             args = (profile_data["ova"],)
             db_cur.execute(query, args)
@@ -290,6 +287,7 @@ def create_vm(
             query = "SELECT * FROM ova_volume WHERE ova = %s"
             args = (profile_data["ova"],)
             db_cur.execute(query, args)
+            # Replace the existing volumes list with our OVA volume list
             vm_data["volumes"] = db_cur.fetchall()
 
     retcode, stdout, stderr = pvc_common.run_os_command("uname -m")
@@ -446,32 +444,29 @@ def create_vm(
 
         print("There is enough space on cluster to store VM volumes")
 
-    if not is_ova_install:
-        # Verify that every specified filesystem is valid
-        used_filesystems = list()
-        for volume in vm_data["volumes"]:
-            if volume["source_volume"] is not None:
-                continue
-            if volume["filesystem"] and volume["filesystem"] not in used_filesystems:
-                used_filesystems.append(volume["filesystem"])
+    # Verify that every specified filesystem is valid
+    used_filesystems = list()
+    for volume in vm_data["volumes"]:
+        if volume["source_volume"] is not None:
+            continue
+        if volume["filesystem"] and volume["filesystem"] not in used_filesystems:
+            used_filesystems.append(volume["filesystem"])
 
-        for filesystem in used_filesystems:
-            if filesystem == "swap":
-                retcode, stdout, stderr = pvc_common.run_os_command("which mkswap")
-                if retcode:
-                    raise ProvisioningError(
-                        "Failed to find binary for mkswap: {}".format(stderr)
-                    )
-            else:
-                retcode, stdout, stderr = pvc_common.run_os_command(
-                    "which mkfs.{}".format(filesystem)
+    for filesystem in used_filesystems:
+        if filesystem == "swap":
+            retcode, stdout, stderr = pvc_common.run_os_command("which mkswap")
+            if retcode:
+                raise ProvisioningError(
+                    "Failed to find binary for mkswap: {}".format(stderr)
+                )
+        else:
+            retcode, stdout, stderr = pvc_common.run_os_command(
+                "which mkfs.{}".format(filesystem)
+            )
+            if retcode:
+                raise ProvisioningError(
+                    "Failed to find binary for mkfs.{}: {}".format(filesystem, stderr)
                 )
-                if retcode:
-                    raise ProvisioningError(
-                        "Failed to find binary for mkfs.{}: {}".format(
-                            filesystem, stderr
-                        )
-                    )
 
         print("All selected filesystems are valid")
 

client-cli/pvc/pvc.py

@@ -5265,7 +5265,8 @@ def provisioner_profile_list(limit):
     "-s",
     "--system-template",
     "system_template",
-    help="The system template for the profile.",
+    required=True,
+    help="The system template for the profile (required).",
 )
 @click.option(
     "-n",
@@ -5280,10 +5281,24 @@ def provisioner_profile_list(limit):
     help="The storage template for the profile.",
 )
 @click.option(
-    "-u", "--userdata", "userdata", help="The userdata document for the profile."
+    "-u",
+    "--userdata",
+    "userdata",
+    help="The userdata document for the profile.",
+)
+@click.option(
+    "-x",
+    "--script",
+    "script",
+    required=True,
+    help="The script for the profile (required).",
+)
+@click.option(
+    "-o",
+    "--ova",
+    "ova",
+    help="The OVA image for the profile; set automatically with 'provisioner ova upload'.",
 )
-@click.option("-x", "--script", "script", help="The script for the profile.")
-@click.option("-o", "--ova", "ova", help="The OVA image for the profile.")
 @click.option(
     "-a",
     "--script-arg",

docs/manuals/swagger.json

@@ -3025,14 +3025,14 @@
                         "description": "Script name",
                         "in": "query",
                         "name": "script",
-                        "required": false,
+                        "required": true,
                         "type": "string"
                     },
                     {
                         "description": "System template name",
                         "in": "query",
                         "name": "system_template",
-                        "required": false,
+                        "required": true,
                         "type": "string"
                     },
                     {
@@ -3165,21 +3165,21 @@
                         "description": "Network template name",
                         "in": "query",
                         "name": "network_template",
-                        "required": true,
+                        "required": false,
                         "type": "string"
                     },
                     {
                         "description": "Storage template name",
                         "in": "query",
                         "name": "storage_template",
-                        "required": true,
+                        "required": false,
                         "type": "string"
                     },
                     {
                         "description": "Userdata template name",
                         "in": "query",
                         "name": "userdata",
-                        "required": true,
+                        "required": false,
                         "type": "string"
                     },
                     {