parallelvirtualcluster
/
pvc
1
0
Fork 0
Code Issues 15 Pull Requests Projects Releases 106 Wiki Activity

Move all host provisioner steps to a try block 

Make the provisioner a bit more robust. This way, even if a provisioning
step fails, cleanup is still performed this preventing the system from
being left in an undefined state requiring manual correction.

Addresses #91
This commit is contained in:
Joshua Boniface 2020-08-06 12:24:04 -04:00
parent ccee124c8b
commit 5526e13da9
1 changed files with 206 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
api-daemon/pvcapid/provisioner.py
View File

@ -1389,6 +1389,8 @@ def create_vm(self, vm_name, vm_profile, define_vm=True, start_vm=True, script_r
print("Final VM schema:\n{}\n".format(vm_schema)) print("Final VM schema:\n{}\n".format(vm_schema))
# All the following steps may require cleanup later on, so catch them here and do cleanup in a Finally block
try:
# Phase 5 - definition # Phase 5 - definition
# * Create the VM in the PVC cluster # * Create the VM in the PVC cluster
self.update_state(state='RUNNING', meta={'current': 5, 'total': 10, 'status': 'Defining VM on the cluster'}) self.update_state(state='RUNNING', meta={'current': 5, 'total': 10, 'status': 'Defining VM on the cluster'})
@ -1416,12 +1418,12 @@ def create_vm(self, vm_name, vm_profile, define_vm=True, start_vm=True, script_r
success, message = pvc_ceph.clone_volume(zk_conn, volume['pool'], "{}_{}".format(vm_name, volume['disk_id']), volume['source_volume']) success, message = pvc_ceph.clone_volume(zk_conn, volume['pool'], "{}_{}".format(vm_name, volume['disk_id']), volume['source_volume'])
print(message) print(message)
if not success: if not success:
raise ClusterError('Failed to clone volume "{}" to "{}".'.format(volume['source_volume'], volume['disk_id'])) raise ProvisioningError('Failed to clone volume "{}" to "{}".'.format(volume['source_volume'], volume['disk_id']))
else: else:
success, message = pvc_ceph.add_volume(zk_conn, volume['pool'], "{}_{}".format(vm_name, volume['disk_id']), "{}G".format(volume['disk_size_gb'])) success, message = pvc_ceph.add_volume(zk_conn, volume['pool'], "{}_{}".format(vm_name, volume['disk_id']), "{}G".format(volume['disk_size_gb']))
print(message) print(message)
if not success: if not success:
raise ClusterError('Failed to create volume "{}".'.format(volume['disk_id'])) raise ProvisioningError('Failed to create volume "{}".'.format(volume['disk_id']))
# Phase 7 - disk mapping # Phase 7 - disk mapping
# * Map each volume to the local host in order # * Map each volume to the local host in order
@ -1560,9 +1562,15 @@ def create_vm(self, vm_name, vm_profile, define_vm=True, start_vm=True, script_r
networks=vm_data['networks'], networks=vm_data['networks'],
**script_arguments **script_arguments
) )
except: except Exception as e:
pass raise ProvisioningError('Failed to run install script: {}'.format(e))
except Exception as e:
start_vm = False
raise e
# Always perform the cleanup steps
finally:
# Phase 9 - install cleanup # Phase 9 - install cleanup
# * Unmount any mounted volumes # * Unmount any mounted volumes
# * Remove any temporary directories # * Remove any temporary directories
@ -1580,9 +1588,14 @@ def create_vm(self, vm_name, vm_profile, define_vm=True, start_vm=True, script_r
print("Cleaning up mount {}{}".format(temp_dir, volume['mountpoint'])) print("Cleaning up mount {}{}".format(temp_dir, volume['mountpoint']))
mount_path = "{}{}".format(temp_dir, volume['mountpoint']) mount_path = "{}{}".format(temp_dir, volume['mountpoint'])
# Make sure any bind mounts or submounts are unmounted first
if volume['mountpoint'] == '/':
retcode, stdout, stderr = pvc_common.run_os_command("umount {}/**{/**,}".format(mount_path))
retcode, stdout, stderr = pvc_common.run_os_command("umount {}".format(mount_path)) retcode, stdout, stderr = pvc_common.run_os_command("umount {}".format(mount_path))
if retcode: if retcode:
raise ProvisioningError('Failed to unmount "{}": {}'.format(mount_path, stderr)) print('Failed to unmount "{}": {}'.format(mount_path, stderr))
# Unmap the RBD device # Unmap the RBD device
if volume['filesystem']: if volume['filesystem']:
@ -1591,7 +1604,7 @@ def create_vm(self, vm_name, vm_profile, define_vm=True, start_vm=True, script_r
rbd_volume = "/dev/rbd/{}/{}_{}".format(volume['pool'], vm_name, volume['disk_id']) rbd_volume = "/dev/rbd/{}/{}_{}".format(volume['pool'], vm_name, volume['disk_id'])
retcode, stdout, stderr = pvc_common.run_os_command("rbd unmap {}".format(rbd_volume)) retcode, stdout, stderr = pvc_common.run_os_command("rbd unmap {}".format(rbd_volume))
if retcode: if retcode:
raise ProvisioningError('Failed to unmap volume "{}": {}'.format(rbd_volume, stderr)) print('Failed to unmap volume "{}": {}'.format(rbd_volume, stderr))
print("Cleaning up temporary directories and files") print("Cleaning up temporary directories and files")
@ -1608,10 +1621,9 @@ def create_vm(self, vm_name, vm_profile, define_vm=True, start_vm=True, script_r
# Phase 10 - startup # Phase 10 - startup
# * Start the VM in the PVC cluster # * Start the VM in the PVC cluster
if start_vm:
self.update_state(state='RUNNING', meta={'current': 10, 'total': 10, 'status': 'Starting VM'}) self.update_state(state='RUNNING', meta={'current': 10, 'total': 10, 'status': 'Starting VM'})
time.sleep(1) time.sleep(1)
if start_vm:
retcode, retmsg = pvc_vm.start_vm(zk_conn, vm_name) retcode, retmsg = pvc_vm.start_vm(zk_conn, vm_name)
print(retmsg) print(retmsg)