2018-05-31 20:26:44 -04:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
2018-05-31 22:31:20 -04:00
|
|
|
import os, time, uuid, threading, libvirt, kazoo.client
|
2018-05-31 20:26:44 -04:00
|
|
|
|
|
|
|
class VMInstance:
|
|
|
|
def __init__(self, domuuid, zk, thishypervisor):
|
|
|
|
# Passed-in variables on creation
|
|
|
|
self.domuuid = domuuid
|
|
|
|
self.zkey = '/domains/%s' % domuuid
|
|
|
|
self.zk = zk
|
|
|
|
self.thishypervisor = thishypervisor
|
|
|
|
|
|
|
|
# These will all be set later
|
|
|
|
self.hypervisor = None
|
|
|
|
self.state = None
|
2018-06-02 16:09:25 -04:00
|
|
|
|
|
|
|
# Start up a new Libvirt connection
|
|
|
|
libvirt_name = "qemu:///system"
|
|
|
|
conn = libvirt.open(libvirt_name)
|
|
|
|
if conn == None:
|
2018-06-02 16:22:05 -04:00
|
|
|
print('>>> Failed to open local libvirt connection.')
|
2018-06-02 16:09:25 -04:00
|
|
|
exit(1)
|
|
|
|
|
|
|
|
try:
|
|
|
|
self.dom = conn.lookupByUUID(uuid.UUID(self.domuuid).bytes)
|
|
|
|
conn.close()
|
|
|
|
except libvirt.libvirtError:
|
|
|
|
self.dom = None
|
2018-05-31 20:26:44 -04:00
|
|
|
|
|
|
|
# Watch for changes to the hypervisor field in Zookeeper
|
|
|
|
@zk.DataWatch(self.zkey + '/hypervisor')
|
2018-06-01 12:21:58 -04:00
|
|
|
def watch_hypervisor(data, stat, event=""):
|
2018-05-31 23:28:26 -04:00
|
|
|
if self.hypervisor != data.decode('ascii'):
|
|
|
|
self.hypervisor = data.decode('ascii')
|
|
|
|
self.manage_vm_state()
|
2018-05-31 20:26:44 -04:00
|
|
|
|
|
|
|
# Watch for changes to the state field in Zookeeper
|
|
|
|
@zk.DataWatch(self.zkey + '/state')
|
2018-06-01 12:21:58 -04:00
|
|
|
def watch_state(data, stat, event=""):
|
2018-05-31 23:28:26 -04:00
|
|
|
if self.state != data.decode('ascii'):
|
|
|
|
self.state = data.decode('ascii')
|
|
|
|
self.manage_vm_state()
|
2018-06-02 15:03:44 -04:00
|
|
|
|
|
|
|
# Get data functions
|
2018-06-02 15:19:29 -04:00
|
|
|
def getstate(self):
|
2018-06-02 15:03:44 -04:00
|
|
|
return self.state
|
|
|
|
|
2018-06-02 15:19:29 -04:00
|
|
|
def gethypervisor(self):
|
|
|
|
return self.hypervisor
|
2018-06-02 15:03:44 -04:00
|
|
|
|
2018-05-31 20:26:44 -04:00
|
|
|
# Start up the VM
|
|
|
|
def start_vm(self, conn, xmlconfig):
|
2018-06-02 16:22:05 -04:00
|
|
|
print(">>> Starting VM %s" % self.domuuid)
|
2018-06-02 00:30:25 -04:00
|
|
|
try:
|
|
|
|
dom = conn.createXML(xmlconfig, 0)
|
2018-06-02 16:09:25 -04:00
|
|
|
except libvirt.libvirtError as e:
|
2018-06-02 16:22:05 -04:00
|
|
|
print('>>> Failed to create domain %s' % self.domuuid)
|
2018-06-02 00:50:57 -04:00
|
|
|
self.zk.set(self.zkey + '/state', 'stop'.encode('ascii'))
|
2018-06-02 00:33:54 -04:00
|
|
|
|
2018-06-01 12:21:58 -04:00
|
|
|
if not self.domuuid in self.thishypervisor.domain_list:
|
|
|
|
self.thishypervisor.domain_list.append(self.domuuid)
|
2018-06-02 00:33:54 -04:00
|
|
|
|
2018-06-02 15:52:50 -04:00
|
|
|
self.dom = dom
|
2018-05-31 20:26:44 -04:00
|
|
|
|
|
|
|
# Stop the VM forcibly
|
|
|
|
def stop_vm(self):
|
2018-06-02 16:22:05 -04:00
|
|
|
print(">>> Forcibly stopping VM %s" % self.domuuid)
|
2018-05-31 20:26:44 -04:00
|
|
|
self.dom.destroy()
|
2018-06-01 12:21:58 -04:00
|
|
|
if self.domuuid in self.thishypervisor.domain_list:
|
2018-06-02 15:43:02 -04:00
|
|
|
try:
|
|
|
|
self.thishypervisor.domain_list.remove(self.domuuid)
|
|
|
|
except ValueError:
|
|
|
|
pass
|
2018-06-02 15:52:50 -04:00
|
|
|
|
|
|
|
self.dom = None
|
2018-05-31 20:26:44 -04:00
|
|
|
|
|
|
|
# Shutdown the VM gracefully
|
|
|
|
def shutdown_vm(self):
|
2018-06-02 16:22:05 -04:00
|
|
|
print(">>> Stopping VM %s" % self.domuuid)
|
2018-05-31 20:26:44 -04:00
|
|
|
self.dom.shutdown()
|
2018-06-02 16:19:51 -04:00
|
|
|
while self.dom.state()[0] == libvirt.VIR_DOMAIN_RUNNING:
|
|
|
|
time.sleep(0.5)
|
|
|
|
|
2018-06-01 12:21:58 -04:00
|
|
|
if self.domuuid in self.thishypervisor.domain_list:
|
2018-06-02 15:43:02 -04:00
|
|
|
try:
|
|
|
|
self.thishypervisor.domain_list.remove(self.domuuid)
|
|
|
|
except ValueError:
|
|
|
|
pass
|
2018-05-31 20:26:44 -04:00
|
|
|
|
2018-06-02 15:52:50 -04:00
|
|
|
self.dom = None
|
|
|
|
|
2018-05-31 20:26:44 -04:00
|
|
|
# Migrate the VM to a target host
|
|
|
|
def migrate_vm(self):
|
2018-06-02 15:34:25 -04:00
|
|
|
self.zk.set(self.zkey + '/state', 'migrate'.encode('ascii'))
|
|
|
|
|
2018-06-02 15:39:17 -04:00
|
|
|
try:
|
2018-06-02 16:15:37 -04:00
|
|
|
dest_conn = libvirt.open('qemu+tcp://%s/system' % self.hypervisor)
|
2018-06-02 15:39:17 -04:00
|
|
|
if dest_conn == None:
|
|
|
|
raise
|
|
|
|
except:
|
2018-06-02 16:22:05 -04:00
|
|
|
print('>>> Failed to open connection to qemu+ssh://%s/system' % target)
|
2018-06-02 15:46:20 -04:00
|
|
|
self.zk.set(self.zkey + '/state', 'start'.encode('ascii'))
|
2018-06-02 15:34:25 -04:00
|
|
|
return
|
2018-05-31 20:26:44 -04:00
|
|
|
|
2018-06-02 15:39:17 -04:00
|
|
|
try:
|
|
|
|
target_dom = self.dom.migrate(dest_conn, libvirt.VIR_MIGRATE_LIVE, None, None, 0)
|
|
|
|
if target_dom == None:
|
|
|
|
raise
|
2018-06-02 16:22:05 -04:00
|
|
|
print('>>> Migrated successfully')
|
2018-06-02 15:39:17 -04:00
|
|
|
except:
|
2018-06-02 16:22:05 -04:00
|
|
|
print('>>> Could not migrate to the new domain; forcing away uncleanly')
|
2018-06-02 15:29:12 -04:00
|
|
|
self.stop_vm()
|
2018-06-02 16:10:59 -04:00
|
|
|
time.sleep(0.5)
|
|
|
|
self.zk.set(self.zkey + '/state', 'start'.encode('ascii'))
|
2018-05-31 20:26:44 -04:00
|
|
|
|
2018-06-02 15:58:21 -04:00
|
|
|
try:
|
|
|
|
self.thishypervisor.domain_list.remove(self.domuuid)
|
|
|
|
except ValueError:
|
|
|
|
pass
|
|
|
|
|
2018-05-31 20:26:44 -04:00
|
|
|
dest_conn.close()
|
|
|
|
|
|
|
|
# Receive the migration from another host (wait until VM is running)
|
|
|
|
def receive_migrate(self):
|
|
|
|
while True:
|
2018-06-02 14:32:59 -04:00
|
|
|
if self.dom == None or self.dom.state() != libvirt.VIR_DOMAIN_RUNNING:
|
2018-06-02 15:52:50 -04:00
|
|
|
try:
|
|
|
|
self.dom = conn.lookupByUUID(uuid.UUID(self.domuuid).bytes)
|
|
|
|
except:
|
|
|
|
pass
|
2018-05-31 20:26:44 -04:00
|
|
|
else:
|
|
|
|
self.zk.set(self.zkey + '/status', b'start')
|
2018-06-01 12:21:58 -04:00
|
|
|
if not self.domuuid in self.thishypervisor.domain_list:
|
|
|
|
self.thishypervisor.domain_list.append(self.domuuid)
|
2018-05-31 20:26:44 -04:00
|
|
|
break
|
2018-05-31 23:28:26 -04:00
|
|
|
|
2018-05-31 20:26:44 -04:00
|
|
|
#
|
|
|
|
# Main function to manage a VM (taking only self)
|
|
|
|
#
|
|
|
|
def manage_vm_state(self):
|
|
|
|
# Start up a new Libvirt connection
|
|
|
|
libvirt_name = "qemu:///system"
|
|
|
|
conn = libvirt.open(libvirt_name)
|
|
|
|
if conn == None:
|
2018-06-02 16:22:05 -04:00
|
|
|
print('>>> Failed to open local libvirt connection.')
|
2018-05-31 20:26:44 -04:00
|
|
|
exit(1)
|
|
|
|
|
|
|
|
# Check the current state of the VM
|
|
|
|
try:
|
2018-05-31 23:28:26 -04:00
|
|
|
if self.dom != None:
|
2018-06-01 12:21:58 -04:00
|
|
|
running, reason = self.dom.state()
|
2018-05-31 23:28:26 -04:00
|
|
|
else:
|
2018-06-02 15:52:50 -04:00
|
|
|
raise
|
2018-05-31 20:26:44 -04:00
|
|
|
except:
|
2018-06-01 12:21:58 -04:00
|
|
|
running = libvirt.VIR_DOMAIN_NOSTATE
|
2018-05-31 20:26:44 -04:00
|
|
|
|
2018-06-02 15:26:37 -04:00
|
|
|
# VM should be stopped
|
2018-06-01 12:21:58 -04:00
|
|
|
if running == libvirt.VIR_DOMAIN_RUNNING and self.state == "stop" and self.hypervisor == self.thishypervisor.name:
|
2018-05-31 20:26:44 -04:00
|
|
|
self.stop_vm()
|
|
|
|
|
2018-06-02 15:26:37 -04:00
|
|
|
# VM should be shut down
|
|
|
|
elif running == libvirt.VIR_DOMAIN_RUNNING and self.state == "shutdown" and self.hypervisor == self.thishypervisor.name:
|
2018-05-31 20:26:44 -04:00
|
|
|
self.shutdown_vm()
|
|
|
|
|
2018-06-02 15:26:37 -04:00
|
|
|
# VM should be migrated to this hypervisor
|
2018-06-02 14:18:09 -04:00
|
|
|
elif running != libvirt.VIR_DOMAIN_RUNNING and self.state == "migrate" and self.hypervisor == self.thishypervisor.name:
|
2018-05-31 20:26:44 -04:00
|
|
|
self.receive_migrate()
|
2018-06-02 15:26:37 -04:00
|
|
|
|
|
|
|
# VM should be migrated away from this hypervisor
|
2018-06-02 15:29:12 -04:00
|
|
|
elif running == libvirt.VIR_DOMAIN_RUNNING and self.state == "start" and self.hypervisor != self.thishypervisor.name:
|
2018-05-31 20:26:44 -04:00
|
|
|
self.migrate_vm()
|
|
|
|
|
2018-06-02 15:54:43 -04:00
|
|
|
# VM is already running and should be
|
|
|
|
elif running == libvirt.VIR_DOMAIN_RUNNING and self.state == "start" and self.hypervisor == self.thishypervisor.name:
|
|
|
|
if not self.domuuid in self.thishypervisor.domain_list:
|
|
|
|
self.thishypervisor.domain_list.append(self.domuuid)
|
|
|
|
|
2018-06-02 15:26:37 -04:00
|
|
|
# VM should be started
|
2018-06-01 12:21:58 -04:00
|
|
|
elif running != libvirt.VIR_DOMAIN_RUNNING and self.state == "start" and self.hypervisor == self.thishypervisor.name:
|
2018-05-31 20:26:44 -04:00
|
|
|
# Grab the domain information from Zookeeper
|
|
|
|
domxml, domxmlstat = self.zk.get(self.zkey + '/xml')
|
|
|
|
domxml = str(domxml.decode('ascii'))
|
2018-06-02 15:52:50 -04:00
|
|
|
self.start_vm(conn, domxml)
|
2018-05-31 22:31:20 -04:00
|
|
|
|
2018-05-31 20:26:44 -04:00
|
|
|
# The VM should now be running so return the domain and active connection
|
2018-06-02 16:09:25 -04:00
|
|
|
conn.close()
|