2018-05-31 20:26:44 -04:00
#!/usr/bin/env python3
2018-06-06 01:47:53 -04:00
# VMInstance.py - Class implementing a PVC virtual machine and run by pvcd
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
2018-06-06 02:14:41 -04:00
import os , sys , uuid , socket , time , threading , libvirt , kazoo . client
2018-05-31 20:26:44 -04:00
2018-06-06 22:56:03 -04:00
# ANSII colours for output
class bcolours :
PURPLE = ' \033 [95m '
BLUE = ' \033 [94m '
GREEN = ' \033 [92m '
YELLOW = ' \033 [93m '
RED = ' \033 [91m '
ENDC = ' \033 [0m '
BOLD = ' \033 [1m '
UNDERLINE = ' \033 [4m '
2018-05-31 20:26:44 -04:00
class VMInstance :
def __init__ ( self , domuuid , zk , thishypervisor ) :
# Passed-in variables on creation
self . domuuid = domuuid
self . zk = zk
self . thishypervisor = thishypervisor
# These will all be set later
self . hypervisor = None
self . state = None
2018-06-02 18:34:48 -04:00
self . instart = False
self . instop = False
2018-06-02 16:28:18 -04:00
self . inshutdown = False
2018-06-02 18:34:48 -04:00
self . inmigrate = False
self . inreceive = False
2018-06-02 16:09:25 -04:00
2018-06-06 02:13:14 -04:00
self . dom = lookupByUUID ( self . domuuid )
2018-05-31 20:26:44 -04:00
# Watch for changes to the hypervisor field in Zookeeper
2018-06-06 21:57:58 -04:00
@zk.DataWatch ( ' /domains/ {} /hypervisor ' . format ( self . domuuid ) )
2018-06-01 12:21:58 -04:00
def watch_hypervisor ( data , stat , event = " " ) :
2018-06-06 11:41:57 -04:00
try :
self . hypervisor = data . decode ( ' ascii ' )
except :
return
2018-06-04 12:22:21 -04:00
self . manage_vm_state ( )
2018-05-31 20:26:44 -04:00
# Watch for changes to the state field in Zookeeper
2018-06-06 21:57:58 -04:00
@zk.DataWatch ( ' /domains/ {} /state ' . format ( self . domuuid ) )
2018-06-01 12:21:58 -04:00
def watch_state ( data , stat , event = " " ) :
2018-06-06 11:41:57 -04:00
try :
self . state = data . decode ( ' ascii ' )
except :
return
2018-06-04 12:22:21 -04:00
self . manage_vm_state ( )
2018-06-02 15:03:44 -04:00
# Get data functions
2018-06-02 15:19:29 -04:00
def getstate ( self ) :
2018-06-02 15:03:44 -04:00
return self . state
2018-06-02 15:19:29 -04:00
def gethypervisor ( self ) :
return self . hypervisor
2018-06-02 15:03:44 -04:00
2018-05-31 20:26:44 -04:00
# Start up the VM
2018-06-04 01:22:18 -04:00
def start_vm ( self , xmlconfig ) :
2018-06-06 22:56:03 -04:00
print ( bcolours . BLUE + ' >>> ' + bcolours . ENDC + ' {} - Starting VM. ' . format ( self . domuuid ) )
2018-06-02 18:34:48 -04:00
self . instart = True
2018-06-04 01:22:18 -04:00
# Start up a new Libvirt connection
libvirt_name = " qemu:///system "
conn = libvirt . open ( libvirt_name )
if conn == None :
2018-06-06 22:56:03 -04:00
print ( bcolours . RED + ' >>> ' + bcolours . ENDC + ' {} - Failed to open local libvirt connection. ' . format ( self . domuuid ) )
2018-06-04 01:22:18 -04:00
self . instart = False
return
2018-06-02 00:30:25 -04:00
try :
dom = conn . createXML ( xmlconfig , 0 )
2018-06-02 16:09:25 -04:00
except libvirt . libvirtError as e :
2018-06-06 22:56:03 -04:00
print ( bcolours . RED + ' >>> ' + bcolours . ENDC + ' {} - Failed to create VM. ' . format ( self . domuuid ) )
2018-06-06 21:57:58 -04:00
self . zk . set ( ' /domains/ {} /state ' . format ( self . domuuid ) , ' stop ' . encode ( ' ascii ' ) )
2018-06-02 00:33:54 -04:00
2018-06-01 12:21:58 -04:00
if not self . domuuid in self . thishypervisor . domain_list :
self . thishypervisor . domain_list . append ( self . domuuid )
2018-06-02 00:33:54 -04:00
2018-06-04 01:22:18 -04:00
conn . close ( )
2018-06-06 22:56:03 -04:00
print ( bcolours . GREEN + ' >>> ' + bcolours . ENDC + ' {} - Successfully started VM. ' . format ( self . domuuid ) )
2018-06-02 15:52:50 -04:00
self . dom = dom
2018-06-02 18:34:48 -04:00
self . instart = False
2018-05-31 20:26:44 -04:00
2018-06-06 21:47:06 -04:00
# Stop the VM forcibly without updating state
def terminate_vm ( self ) :
2018-06-06 22:56:03 -04:00
print ( bcolours . BLUE + ' >>> ' + bcolours . ENDC + ' {} - Terminating VM. ' . format ( self . domuuid ) )
2018-06-06 21:47:06 -04:00
self . instop = True
try :
self . dom . destroy ( )
except AttributeError :
2018-06-06 22:56:03 -04:00
print ( bcolours . RED + ' >>> ' + bcolours . ENDC + ' {} - Failed to terminate VM. ' . format ( self . domuuid ) )
2018-06-06 21:47:06 -04:00
if self . domuuid in self . thishypervisor . domain_list :
try :
self . thishypervisor . domain_list . remove ( self . domuuid )
except ValueError :
pass
2018-06-06 22:56:03 -04:00
print ( bcolours . GREEN + ' >>> ' + bcolours . ENDC + ' {} - Successfully terminated VM. ' . format ( self . domuuid ) )
2018-06-06 21:47:06 -04:00
2018-05-31 20:26:44 -04:00
# Stop the VM forcibly
def stop_vm ( self ) :
2018-06-06 22:56:03 -04:00
print ( bcolours . BLUE + ' >>> ' + bcolours . ENDC + ' {} - Forcibly stopping VM. ' . format ( self . domuuid ) )
2018-06-02 18:34:48 -04:00
self . instop = True
2018-06-06 11:48:28 -04:00
try :
self . dom . destroy ( )
except AttributeError :
2018-06-06 22:56:03 -04:00
print ( bcolours . RED + ' >>> ' + bcolours . ENDC + ' {} - Failed to stop VM. ' . format ( self . domuuid ) )
2018-06-01 12:21:58 -04:00
if self . domuuid in self . thishypervisor . domain_list :
2018-06-02 15:43:02 -04:00
try :
self . thishypervisor . domain_list . remove ( self . domuuid )
except ValueError :
pass
2018-06-02 15:52:50 -04:00
2018-06-06 21:57:58 -04:00
self . zk . set ( ' /domains/ {} /state ' . format ( self . domuuid ) , ' stop ' . encode ( ' ascii ' ) )
2018-06-06 22:56:03 -04:00
print ( bcolours . GREEN + ' >>> ' + bcolours . ENDC + ' {} - Successfully stopped VM. ' . format ( self . domuuid ) )
2018-06-02 15:52:50 -04:00
self . dom = None
2018-06-02 18:34:48 -04:00
self . instop = False
2018-05-31 20:26:44 -04:00
# Shutdown the VM gracefully
def shutdown_vm ( self ) :
2018-06-06 22:56:03 -04:00
print ( bcolours . BLUE + ' >>> ' + bcolours . ENDC + ' {} - Gracefully stopping VM. ' . format ( self . domuuid ) )
2018-06-02 18:34:48 -04:00
self . inshutdown = True
2018-05-31 20:26:44 -04:00
self . dom . shutdown ( )
2018-06-02 16:24:11 -04:00
try :
2018-06-02 16:28:18 -04:00
tick = 0
while self . dom . state ( ) [ 0 ] == libvirt . VIR_DOMAIN_RUNNING and tick < 60 :
tick + = 1
2018-06-02 16:24:11 -04:00
time . sleep ( 0.5 )
2018-06-02 16:28:18 -04:00
if tick > = 60 :
2018-06-06 22:56:03 -04:00
print ( bcolours . RED + ' >>> ' + bcolours . ENDC + ' {} - Shutdown timeout expired. ' . format ( self . domuuid ) )
2018-06-02 16:28:18 -04:00
self . stop_vm ( )
2018-06-02 18:34:48 -04:00
self . inshutdown = False
2018-06-02 16:28:18 -04:00
return
2018-06-02 16:24:11 -04:00
except :
pass
2018-06-02 16:19:51 -04:00
2018-06-01 12:21:58 -04:00
if self . domuuid in self . thishypervisor . domain_list :
2018-06-02 15:43:02 -04:00
try :
self . thishypervisor . domain_list . remove ( self . domuuid )
except ValueError :
pass
2018-05-31 20:26:44 -04:00
2018-06-06 21:57:58 -04:00
self . zk . set ( ' /domains/ {} /state ' . format ( self . domuuid ) , ' stop ' . encode ( ' ascii ' ) )
2018-06-06 22:56:03 -04:00
print ( bcolours . GREEN + ' >>> ' + bcolours . ENDC + ' {} - Successfully shutdown VM. ' . format ( self . domuuid ) )
2018-06-02 15:52:50 -04:00
self . dom = None
2018-06-02 18:34:48 -04:00
self . inshutdown = False
2018-06-02 15:52:50 -04:00
2018-06-04 12:15:37 -04:00
def live_migrate_vm ( self , dest_hypervisor ) :
2018-06-02 15:39:17 -04:00
try :
2018-06-06 21:54:38 -04:00
dest_conn = libvirt . open ( ' qemu+tcp:// {} /system ' . format ( self . hypervisor ) )
2018-06-02 15:39:17 -04:00
if dest_conn == None :
raise
except :
2018-06-06 22:56:03 -04:00
print ( bcolours . RED + ' >>> ' + bcolours . ENDC + ' {} - Failed to open connection to qemu+tcp:// {} /system; aborting migration. ' . format ( self . dom_uuid , self . hypervisor ) )
2018-06-04 12:15:37 -04:00
return 1
2018-05-31 20:26:44 -04:00
2018-06-02 15:39:17 -04:00
try :
target_dom = self . dom . migrate ( dest_conn , libvirt . VIR_MIGRATE_LIVE , None , None , 0 )
if target_dom == None :
raise
2018-06-06 22:56:03 -04:00
print ( bcolours . GREEN + ' >>> ' + bcolours . ENDC + ' {} - Migrated successfully. ' . format ( self . domuuid ) )
2018-06-02 15:39:17 -04:00
except :
2018-06-04 12:15:37 -04:00
dest_conn . close ( )
return 1
dest_conn . close ( )
return 0
# Migrate the VM to a target host
def migrate_vm ( self ) :
self . inmigrate = True
2018-06-06 22:56:03 -04:00
print ( bcolours . BLUE + ' >>> ' + bcolours . ENDC + ' {} - Migrating VM to hypervisor " {} " . ' . format ( self . domuuid , self . hypervisor ) )
2018-06-04 16:30:09 -04:00
migrate_ret = self . live_migrate_vm ( self . hypervisor )
2018-06-04 12:15:37 -04:00
if migrate_ret != 0 :
2018-06-06 22:56:03 -04:00
print ( bcolours . RED + ' >>> ' + bcolours . ENDC + ' {} - Could not live migrate VM; shutting down to migrate instead. ' . format ( self . domuuid ) )
2018-06-04 16:30:09 -04:00
self . shutdown_vm ( )
2018-06-06 21:54:38 -04:00
time . sleep ( 1 )
2018-06-06 21:57:58 -04:00
self . zk . set ( ' /domains/ {} /state ' . format ( self . domuuid ) , ' start ' . encode ( ' ascii ' ) )
2018-06-04 12:15:37 -04:00
else :
try :
self . thishypervisor . domain_list . remove ( self . domuuid )
except ValueError :
pass
2018-06-02 15:58:21 -04:00
2018-06-02 18:34:48 -04:00
self . inmigrate = False
2018-06-04 11:49:39 -04:00
2018-05-31 20:26:44 -04:00
# Receive the migration from another host (wait until VM is running)
2018-06-04 01:22:18 -04:00
def receive_migrate ( self ) :
2018-06-06 22:56:03 -04:00
print ( bcolours . BLUE + ' >>> ' + bcolours . ENDC + ' {} - Receiving migration. ' . format ( self . domuuid ) )
2018-06-02 18:34:48 -04:00
self . inreceive = True
2018-05-31 20:26:44 -04:00
while True :
2018-06-06 02:13:14 -04:00
self . dom = lookupByUUID ( self . domuuid )
2018-06-04 01:13:48 -04:00
if self . dom == None :
2018-06-02 19:06:59 -04:00
time . sleep ( 0.2 )
continue
2018-06-02 18:38:59 -04:00
2018-06-02 19:06:59 -04:00
if self . dom . state ( ) [ 0 ] == libvirt . VIR_DOMAIN_RUNNING :
break
2018-06-02 18:38:59 -04:00
2018-06-06 21:57:58 -04:00
self . zk . set ( ' /domains/ {} /state ' . format ( self . domuuid ) , ' start ' . encode ( ' ascii ' ) )
2018-06-02 18:38:59 -04:00
if not self . domuuid in self . thishypervisor . domain_list :
self . thishypervisor . domain_list . append ( self . domuuid )
2018-06-04 03:00:17 -04:00
2018-06-06 22:56:03 -04:00
print ( bcolours . GREEN + ' >>> ' + bcolours . ENDC + ' {} - Migrated successfully. ' . format ( self . domuuid ) )
2018-06-02 18:34:48 -04:00
self . inreceive = False
2018-05-31 23:28:26 -04:00
2018-05-31 20:26:44 -04:00
#
# Main function to manage a VM (taking only self)
#
def manage_vm_state ( self ) :
# Check the current state of the VM
try :
2018-05-31 23:28:26 -04:00
if self . dom != None :
2018-06-06 11:42:49 -04:00
running , reason = self . dom . state ( )
2018-05-31 23:28:26 -04:00
else :
2018-06-02 15:52:50 -04:00
raise
2018-05-31 20:26:44 -04:00
except :
2018-06-01 12:21:58 -04:00
running = libvirt . VIR_DOMAIN_NOSTATE
2018-05-31 20:26:44 -04:00
2018-06-02 15:26:37 -04:00
# VM should be stopped
2018-06-02 18:34:48 -04:00
if running == libvirt . VIR_DOMAIN_RUNNING and self . state == " stop " and self . hypervisor == self . thishypervisor . name and self . instop == False :
2018-05-31 20:26:44 -04:00
self . stop_vm ( )
2018-06-02 15:26:37 -04:00
# VM should be shut down
2018-06-02 18:34:48 -04:00
elif running == libvirt . VIR_DOMAIN_RUNNING and self . state == " shutdown " and self . hypervisor == self . thishypervisor . name and self . inshutdown == False :
2018-05-31 20:26:44 -04:00
self . shutdown_vm ( )
2018-06-02 15:26:37 -04:00
# VM should be migrated to this hypervisor
2018-06-02 18:34:48 -04:00
elif running != libvirt . VIR_DOMAIN_RUNNING and self . state == " migrate " and self . hypervisor == self . thishypervisor . name and self . inreceive == False :
2018-06-04 01:22:18 -04:00
self . receive_migrate ( )
2018-06-02 15:26:37 -04:00
# VM should be migrated away from this hypervisor
2018-06-02 18:34:48 -04:00
elif running == libvirt . VIR_DOMAIN_RUNNING and self . state == " migrate " and self . hypervisor != self . thishypervisor . name and self . inmigrate == False :
2018-05-31 20:26:44 -04:00
self . migrate_vm ( )
2018-06-06 21:45:03 -04:00
# VM should be running but not on this hypervisor
elif running == libvirt . VIR_DOMAIN_RUNNING and self . state == " start " and self . hypervisor != self . thishypervisor . name :
2018-06-06 21:47:06 -04:00
self . terminate_vm ( )
2018-06-06 21:45:03 -04:00
2018-06-02 15:54:43 -04:00
# VM is already running and should be
elif running == libvirt . VIR_DOMAIN_RUNNING and self . state == " start " and self . hypervisor == self . thishypervisor . name :
if not self . domuuid in self . thishypervisor . domain_list :
self . thishypervisor . domain_list . append ( self . domuuid )
2018-06-02 15:26:37 -04:00
# VM should be started
2018-06-02 18:34:48 -04:00
elif running != libvirt . VIR_DOMAIN_RUNNING and self . state == " start " and self . hypervisor == self . thishypervisor . name and self . instart == False :
2018-05-31 20:26:44 -04:00
# Grab the domain information from Zookeeper
2018-06-06 21:57:58 -04:00
domxml , domxmlstat = self . zk . get ( ' /domains/ {} /xml ' . format ( self . domuuid ) )
2018-05-31 20:26:44 -04:00
domxml = str ( domxml . decode ( ' ascii ' ) )
2018-06-04 01:22:18 -04:00
self . start_vm ( domxml )
2018-06-06 01:47:53 -04:00
2018-06-06 11:51:12 -04:00
2018-06-06 01:47:53 -04:00
# This function is a wrapper for libvirt.lookupByUUID which fixes some problems
# 1. Takes a text UUID and handles converting it to bytes
# 2. Try's it and returns a sensible value if not
def lookupByUUID ( tuuid ) :
conn = None
dom = None
libvirt_name = " qemu:///system "
# Convert the text UUID to bytes
buuid = uuid . UUID ( tuuid ) . bytes
# Try
try :
# Open a libvirt connection
conn = libvirt . open ( libvirt_name )
if conn == None :
2018-06-06 22:56:03 -04:00
print ( bcolours . RED + ' >>> ' + bcolours . ENDC + ' {} - Failed to open local libvirt connection. ' . format ( self . domuuid ) )
2018-06-06 01:47:53 -04:00
return dom
# Lookup the UUID
dom = conn . lookupByUUID ( buuid )
# Fail
except :
pass
# After everything
finally :
# Close the libvirt connection
if conn != None :
conn . close ( )
# Return the dom object (or None)
return dom