pvc/api-daemon/provisioner/examples/2-debootstrap.py

612 lines
22 KiB
Python

#!/usr/bin/env python3
# 2-debootstrap.py - PVC Provisioner example script for debootstrap install
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
# This script provides an example of a PVC provisioner script. It will create a
# standard VM config and install a Debian-like OS using debootstrap.
# This script can thus be used as an example or reference implementation of a
# PVC provisioner script and expanded upon as required.
# The script must implement the class "VMBuilderScript" which extens "VMBuilder",
# providing the 5 functions indicated. Detailed explanation of the role of each
# function is provided.
# Within the VMBuilderScript class, several common variables are exposed:
# self.vm_name: The name of the VM from PVC's perspective
# self.vm_id: The VM ID (numerical component of the vm_name) from PVC's perspective
# self.vm_uuid: An automatically-generated UUID for the VM
# self.vm_profile: The PVC provisioner profile name used for the VM
# self.vm-data: A dictionary of VM data collected by the provisioner; an example:
# {
# "ceph_monitor_list": [
# "hv1.pvcstorage.tld",
# "hv2.pvcstorage.tld",
# "hv3.pvcstorage.tld"
# ],
# "ceph_monitor_port": "6789",
# "ceph_monitor_secret": "96721723-8650-4a72-b8f6-a93cd1a20f0c",
# "mac_template": null,
# "networks": [
# {
# "eth_bridge": "vmbr1001",
# "id": 72,
# "network_template": 69,
# "vni": "1001"
# },
# {
# "eth_bridge": "vmbr101",
# "id": 73,
# "network_template": 69,
# "vni": "101"
# }
# ],
# "script": [contents of this file]
# "script_arguments": {
# "deb_mirror": "http://ftp.debian.org/debian",
# "deb_release": "bullseye"
# },
# "system_architecture": "x86_64",
# "system_details": {
# "id": 78,
# "migration_method": "live",
# "name": "small",
# "node_autostart": false,
# "node_limit": null,
# "node_selector": null,
# "ova": null,
# "serial": true,
# "vcpu_count": 2,
# "vnc": false,
# "vnc_bind": null,
# "vram_mb": 2048
# },
# "volumes": [
# {
# "disk_id": "sda",
# "disk_size_gb": 4,
# "filesystem": "ext4",
# "filesystem_args": "-L=root",
# "id": 9,
# "mountpoint": "/",
# "pool": "vms",
# "source_volume": null,
# "storage_template": 67
# },
# {
# "disk_id": "sdb",
# "disk_size_gb": 4,
# "filesystem": "ext4",
# "filesystem_args": "-L=var",
# "id": 10,
# "mountpoint": "/var",
# "pool": "vms",
# "source_volume": null,
# "storage_template": 67
# },
# {
# "disk_id": "sdc",
# "disk_size_gb": 4,
# "filesystem": "ext4",
# "filesystem_args": "-L=log",
# "id": 11,
# "mountpoint": "/var/log",
# "pool": "vms",
# "source_volume": null,
# "storage_template": 67
# }
# ]
# }
from pvcapid.vmbuilder import VMBuilder
class VMBuilderScript(VMBuilder):
def setup(self):
"""
setup(): Perform special setup steps or validation before proceeding
"""
# Run any imports first
import daemon_lib.common as pvc_common
# Ensure we have debootstrap intalled on the provisioner system
retcode, stdout, stderr = pvc_common.run_os_command(f"which debootstrap")
if retcode:
raise ProvisioningError("Failed to find critical dependency: debootstrap")
def create(self):
"""
create(): Create the VM libvirt schema definition
This step *must* return a fully-formed Libvirt XML document as a string.
This example leverages the built-in libvirt_schema objects provided by PVC; these
can be used as-is, or replaced with your own schema(s) on a per-script basis.
"""
# Run any imports first
import datetime
import random
import pvcapid.libvirt_schema as libvirt_schema
schema = ""
# Prepare a description based on the VM profile
description = (
f"PVC provisioner @ {datetime.datetime.now()}, profile '{self.vm_profile}'"
)
# Format the header
schema += libvirt_schema.libvirt_header.format(
vm_name=self.vm_name,
vm_uuid=self.vm_uuid,
vm_description=description,
vm_memory=self.vm_data["system_details"]["vram_mb"],
vm_vcpus=self.vm_data["system_details"]["vcpu_count"],
vm_architecture=self.vm_data["system_architecture"],
)
# Add the disk devices
monitor_list = self.vm_data["ceph_monitor_list"]
monitor_port = self.vm_data["ceph_monitor_port"]
monitor_secret = self.vm_data["ceph_monitor_secret"]
for volume in self.vm_data["volumes"]:
schema += libvirt_schema.devices_disk_header.format(
ceph_storage_secret=monitor_secret,
disk_pool=volume["pool"],
vm_name=self.vm_name,
disk_id=volume["disk_id"],
)
for monitor in monitor_list:
schema += libvirt_schema.devices_disk_coordinator.format(
coordinator_name=monitor,
coordinator_ceph_mon_port=monitor_port,
)
schema += libvirt_schema.devices_disk_footer
# Add the special vhostmd device for hypervisor information inside the VM
schema += libvirt_schema.devices_vhostmd
# Add the network devices
network_id = 0
for network in self.vm_data["networks"]:
vm_id_hex = "{:x}".format(int(self.vm_id % 16))
net_id_hex = "{:x}".format(int(network_id % 16))
if self.vm_data.get("mac_template") is not None:
mac_prefix = "52:54:01"
macgen_template = self.vm_data["mac_template"]
eth_macaddr = macgen_template.format(
prefix=mac_prefix, vmid=vm_id_hex, netid=net_id_hex
)
else:
mac_prefix = "52:54:00"
random_octet_A = "{:x}".format(random.randint(16, 238))
random_octet_B = "{:x}".format(random.randint(16, 238))
random_octet_C = "{:x}".format(random.randint(16, 238))
macgen_template = "{prefix}:{octetA}:{octetB}:{octetC}"
eth_macaddr = macgen_template.format(
prefix=mac_prefix,
octetA=random_octet_A,
octetB=random_octet_B,
octetC=random_octet_C,
)
schema += libvirt_schema.devices_net_interface.format(
eth_macaddr=eth_macaddr,
eth_bridge=network["eth_bridge"],
)
network_id += 1
# Add default devices
schema += libvirt_schema.devices_default
# Add serial device
if self.vm_data["system_details"]["serial"]:
schema += libvirt_schema.devices_serial.format(vm_name=self.vm_name)
# Add VNC device
if self.vm_data["system_details"]["vnc"]:
if self.vm_data["system_details"]["vnc_bind"]:
vm_vnc_bind = self.vm_data["system_details"]["vnc_bind"]
else:
vm_vnc_bind = "127.0.0.1"
vm_vncport = 5900
vm_vnc_autoport = "yes"
schema += libvirt_schema.devices_vnc.format(
vm_vncport=vm_vncport,
vm_vnc_autoport=vm_vnc_autoport,
vm_vnc_bind=vm_vnc_bind,
)
# Add SCSI controller
schema += libvirt_schema.devices_scsi_controller
# Add footer
schema += libvirt_schema.libvirt_footer
return schema
def prepare(self):
"""
prepare(): Prepare any disks/volumes for the install() step
This function should use the various exposed PVC commands as indicated to create
block devices and map them to the host.
"""
# First loop: Create the disks, either by cloning (pvc_ceph.clone_volume), or by
# new creation (pvc_ceph.add_volume).
for volume in self.vm_data["volumes"]:
if volume.get("source_volume") is not None:
with open_zk(config) as zkhandler:
success, message = pvc_ceph.clone_volume(
zkhandler,
volume["pool"],
volume["source_volume"],
f"{self.vm_name}_{volume['disk_id']}",
)
print(message)
if not success:
raise ProvisioningError(
f"Failed to clone volume '{volume['source_volume']}' to '{volume['disk_id']}'."
)
else:
with open_zk(config) as zkhandler:
success, message = pvc_ceph.add_volume(
zkhandler,
volume["pool"],
f"{self.vm_name}_{volume['disk_id']}",
f"{volume['disk_size_gb']}G",
)
print(message)
if not success:
raise ProvisioningError(
f"Failed to create volume '{volume['disk_id']}'."
)
# Second loop: Map the disks to the local system
for volume in self.vm_data["volumes"]:
dst_volume_name = f"{self.vm_name}_{volume['disk_id']}"
dst_volume = f"{volume['pool']}/{dst_volume_name}"
with open_zk(config) as zkhandler:
success, message = pvc_ceph.map_volume(
zkhandler,
volume["pool"],
dst_volume_name,
)
print(message)
if not retcode:
raise ProvisioningError(f"Failed to map volume '{dst_volume}'.")
# Third loop: Create filesystems on the volumes
for volume in self.vm_data["volumes"]:
dst_volume_name = f"{self.vm_name}_{volume['disk_id']}"
dst_volume = f"{volume['pool']}/{dst_volume_name}"
if volume.get("source_volume") is not None:
continue
if volume.get("filesystem") is None:
continue
filesystem_args_list = list()
for arg in volume["filesystem_args"].split():
arg_entry, *arg_data = arg.split("=")
arg_data = "=".join(arg_data)
filesystem_args_list.append(arg_entry)
filesystem_args_list.append(arg_data)
filesystem_args = " ".join(filesystem_args_list)
if volume["filesystem"] == "swap":
retcode, stdout, stderr = pvc_common.run_os_command(
f"mkswap -f /dev/rbd/{dst_volume}"
)
if retcode:
raise ProvisioningError(
f"Failed to create swap on '{dst_volume}': {stderr}"
)
else:
retcode, stdout, stderr = pvc_common.run_os_command(
f"mkfs.{volume['filesystem']} {filesystem_args} /dev/rbd/{dst_volume}"
)
if retcode:
raise ProvisioningError(
f"Faield to create {volume['filesystem']} file on '{dst_volume}': {stderr}"
)
print(stdout)
# Create a temporary directory to use during install
temp_dir = "/tmp/target"
if not os.exists(temp_dir):
os.mkdir(temp_dir)
# Fourth loop: Mount the volumes to a set of temporary directories
for volume in self.vm_data["volumes"]:
dst_volume_name = f"{self.vm_name}_{volume['disk_id']}"
dst_volume = f"{volume['pool']}/{dst_volume_name}"
if volume.get("source_volume") is not None:
continue
if volume.get("filesystem") is None:
continue
mapped_dst_volume = f"/dev/rbd/{dst_volume}"
mount_path = f"{temp_dir}/{volume['mountpoint']}"
if not os.exists(mount_path):
os.mkdir(mount_path)
# Mount filesystem
retcode, stdout, stderr = pvc_common.run_os_command(
f"mount {mapped_dst_volume} {mount_path}"
)
if retcode:
raise ProvisioningError(
f"Failed to mount '{mapped_dst_volume}' on '{mount_path}': {stderr}"
)
def install(self):
"""
install(): Perform the installation
Since this is a noop example, this step does nothing, aside from getting some
arguments for demonstration.
"""
# Run any imports first
from pvcapid.vmbuilder import chroot
# The directory we mounted things on earlier during prepare()
temporary_directory = "/tmp/target"
# Use these convenient aliases for later (avoiding lots of "self.vm_data" everywhere)
vm_name = self.vm_name
disks = self.vm_data["disks"]
networks = self.vm_data["networks"]
# Parse these arguments out of self.vm_data["script_arguments"]
if self.vm_data["script_arguments"].get("deb_release") is not None:
deb_release = self.vm_data["script_arguments"].get("deb_release")
else:
deb_release = "stable"
if self.vm_data["script_arguments"].get("deb_mirror") is not None:
deb_mirror = self.vm_data["script_arguments"].get("deb_mirror")
else:
deb_mirror = "http://ftp.debian.org/debian"
if self.vm_data["script_arguments"].get("deb_packages") is not None:
deb_packages = (
self.vm_data["script_arguments"].get("deb_packages").split(",")
)
else:
deb_packages = [
"linux-image-amd64",
"grub-pc",
"cloud-init",
"python3-cffi-backend",
"wget",
]
# We need to know our root disk
root_disk = None
for disk in disks:
if disk["mountpoint"] == "/":
root_disk = disk
if not root_disk:
raise ProvisioningError("Failed to find root disk in disks list")
# Perform a deboostrap installation
os.system(
"debootstrap --include={pkgs} {suite} {target} {mirror}".format(
suite=deb_release,
target=temporary_directory,
mirror=deb_mirror,
pkgs=",".join(deb_packages),
)
)
# Bind mount the devfs
os.system("mount --bind /dev {}/dev".format(temporary_directory))
# Create an fstab entry for each disk
fstab_file = "{}/etc/fstab".format(temporary_directory)
# The disk ID starts at zero and increments by one for each disk in the fixed-order
# disk list. This lets us work around the insanity of Libvirt IDs not matching guest IDs,
# while still letting us have some semblance of control here without enforcing things
# like labels. It increments in the for loop below at the end of each iteration, and is
# used to craft a /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0-0-0-X device ID
# which will always match the correct order from Libvirt (unlike sdX/vdX names).
disk_id = 0
for disk in disks:
# We assume SSD-based/-like storage, and dislike atimes
options = "defaults,discard,noatime,nodiratime"
# The root, var, and log volumes have specific values
if disk["mountpoint"] == "/":
root_disk["scsi_id"] = disk_id
dump = 0
cpass = 1
elif disk["mountpoint"] == "/var" or disk["mountpoint"] == "/var/log":
dump = 0
cpass = 2
else:
dump = 0
cpass = 0
# Append the fstab line
with open(fstab_file, "a") as fh:
data = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0-0-0-{disk} {mountpoint} {filesystem} {options} {dump} {cpass}\n".format(
disk=disk_id,
mountpoint=disk["mountpoint"],
filesystem=disk["filesystem"],
options=options,
dump=dump,
cpass=cpass,
)
fh.write(data)
# Increment the disk_id
disk_id += 1
# Write the hostname
hostname_file = "{}/etc/hostname".format(temporary_directory)
with open(hostname_file, "w") as fh:
fh.write("{}".format(vm_name))
# Fix the cloud-init.target since it's broken
cloudinit_target_file = "{}/etc/systemd/system/cloud-init.target".format(
temporary_directory
)
with open(cloudinit_target_file, "w") as fh:
data = """[Install]
WantedBy=multi-user.target
[Unit]
Description=Cloud-init target
After=multi-user.target
"""
fh.write(data)
# NOTE: Due to device ordering within the Libvirt XML configuration, the first Ethernet interface
# will always be on PCI bus ID 2, hence the name "ens2".
# Write a DHCP stanza for ens2
ens2_network_file = "{}/etc/network/interfaces.d/ens2".format(
temporary_directory
)
with open(ens2_network_file, "w") as fh:
data = """auto ens2
iface ens2 inet dhcp
"""
fh.write(data)
# Write the DHCP config for ens2
dhclient_file = "{}/etc/dhcp/dhclient.conf".format(temporary_directory)
with open(dhclient_file, "w") as fh:
data = (
"""# DHCP client configuration
# Written by the PVC provisioner
option rfc3442-classless-static-routes code 121 = array of unsigned integer 8;
interface "ens2" {
"""
+ """ send fqdn.fqdn = "{hostname}";
send host-name = "{hostname}";
""".format(
hostname=vm_name
)
+ """ request subnet-mask, broadcast-address, time-offset, routers,
domain-name, domain-name-servers, domain-search, host-name,
dhcp6.name-servers, dhcp6.domain-search, dhcp6.fqdn, dhcp6.sntp-servers,
netbios-name-servers, netbios-scope, interface-mtu,
rfc3442-classless-static-routes, ntp-servers;
}
"""
)
fh.write(data)
# Write the GRUB configuration
grubcfg_file = "{}/etc/default/grub".format(temporary_directory)
with open(grubcfg_file, "w") as fh:
data = """# Written by the PVC provisioner
GRUB_DEFAULT=0
GRUB_TIMEOUT=1
GRUB_DISTRIBUTOR="PVC Virtual Machine"
GRUB_CMDLINE_LINUX_DEFAULT="root=/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0-0-0-{root_disk} console=tty0 console=ttyS0,115200n8"
GRUB_CMDLINE_LINUX=""
GRUB_TERMINAL=console
GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
GRUB_DISABLE_LINUX_UUID=false
""".format(
root_disk=root_disk["scsi_id"]
)
fh.write(data)
# Chroot, do some in-root tasks, then exit the chroot
with chroot(temporary_directory):
# Install and update GRUB
os.system(
"grub-install --force /dev/rbd/{}/{}_{}".format(
root_disk["pool"], vm_name, root_disk["disk_id"]
)
)
os.system("update-grub")
# Set a really dumb root password [TEMPORARY]
os.system("echo root:test123 | chpasswd")
# Enable cloud-init target on (first) boot
# NOTE: Your user-data should handle this and disable it once done, or things get messy.
# That cloud-init won't run without this hack seems like a bug... but even the official
# Debian cloud images are affected, so who knows.
os.system("systemctl enable cloud-init.target")
# Unmount the bound devfs
os.system("umount {}/dev".format(temporary_directory))
def cleanup(self):
"""
cleanup(): Perform any cleanup required due to prepare()/install()
It is important to now reverse *all* steps taken in those functions that might
need cleanup before teardown of the overlay chroot environment.
"""
temp_dir = "/tmp/target"
for volume in list(reversed(self.vm_data["volumes"])):
dst_volume_name = f"{self.vm_name}_{volume['disk_id']}"
dst_volume = f"{volume['pool']}/{dst_volume_name}"
mapped_dst_volume = f"/dev/rbd/{dst_volume}"
mount_path = f"{temp_dir}/{volume['mountpoint']}"
if (
volume.get("source_volume") is None
and volume.get("filesystem") is not None
):
# Unmount filesystem
retcode, stdout, stderr = pvc_common.run_os_command(
f"umount {mount_path}"
)
if retcode:
raise ProvisioningError(
f"Failed to unmount '{mapped_dst_volume}' on '{mount_path}': {stderr}"
)
# Unmap volume
with open_zk(config) as zkhandler:
success, message = pvc_ceph.unmap_volume(
zkhandler,
volume["pool"],
dst_volume_name,
)
if not success:
raise ProvisioningError(
f"Failed to unmap '{mapped_dst_volume}': {stderr}"
)