parallelvirtualcluster
/
pvc
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 109 Wiki Activity

Add ANSII colours to output commands for easy parsing

This commit is contained in:
Joshua Boniface 2018-06-06 22:56:03 -04:00
parent ad1bf35837
commit 640f38c3aa
4 changed files with 68 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
NodeInstance.py
View File

@ -22,6 +22,17 @@
import os, sys, socket, time, libvirt, kazoo.client, threading, fencenode import os, sys, socket, time, libvirt, kazoo.client, threading, fencenode
# ANSII colours for output
class bcolours:
PURPLE = '\033[95m'
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
class NodeInstance(): class NodeInstance():
def __init__(self, name, t_node, s_domain, zk): def __init__(self, name, t_node, s_domain, zk):
# Passed-in variables on creation # Passed-in variables on creation
@ -87,7 +98,7 @@ class NodeInstance():
# Flush all VMs on the host # Flush all VMs on the host
def flush(self): def flush(self):
print('>>> Flushing node {} of running VMs.'.format(self.name)) print(bcolours.BLUE + '>>> ' + bcolours.ENDC + 'Flushing node {} of running VMs.'.format(self.name))
for dom_uuid in self.domain_list: for dom_uuid in self.domain_list:
most_memfree = 0 most_memfree = 0
target_hypervisor = None target_hypervisor = None
@ -104,12 +115,12 @@ class NodeInstance():
target_hypervisor = hypervisor target_hypervisor = hypervisor
if target_hypervisor == None: if target_hypervisor == None:
print('>>> Failed to find migration target for VM "{}"; shutting down.'.format(dom_uuid)) print(bcolours.RED + '>>> ' + bcolours.ENDC + 'Failed to find migration target for VM "{}"; shutting down.'.format(dom_uuid))
transaction = self.zk.transaction() transaction = self.zk.transaction()
transaction.set_data('/domains/{}/state'.format(dom_uuid), 'shutdown'.encode('ascii')) transaction.set_data('/domains/{}/state'.format(dom_uuid), 'shutdown'.encode('ascii'))
transaction.commit() transaction.commit()
else: else:
print('>>> Migrating VM "{}" to hypervisor "{}".'.format(dom_uuid, target_hypervisor)) print(bcolours.BLUE + '>>> ' + bcolours.ENDC + 'Migrating VM "{}" to hypervisor "{}".'.format(dom_uuid, target_hypervisor))
transaction = self.zk.transaction() transaction = self.zk.transaction()
transaction.set_data('/domains/{}/state'.format(dom_uuid), 'migrate'.encode('ascii')) transaction.set_data('/domains/{}/state'.format(dom_uuid), 'migrate'.encode('ascii'))
transaction.set_data('/domains/{}/hypervisor'.format(dom_uuid), target_hypervisor.encode('ascii')) transaction.set_data('/domains/{}/hypervisor'.format(dom_uuid), target_hypervisor.encode('ascii'))
@ -120,14 +131,14 @@ class NodeInstance():
time.sleep(1) time.sleep(1)
def unflush(self): def unflush(self):
print('>>> Restoring node {} to active service.'.format(self.name)) print(bcolours.BLUE + '>>> ' + bcolours.ENDC + 'Restoring node {} to active service.'.format(self.name))
self.zk.set('/nodes/{}/state'.format(self.name), 'start'.encode('ascii')) self.zk.set('/nodes/{}/state'.format(self.name), 'start'.encode('ascii'))
for dom_uuid in self.s_domain: for dom_uuid in self.s_domain:
last_hypervisor = self.zk.get('/domains/{}/lasthypervisor'.format(dom_uuid))[0].decode('ascii') last_hypervisor = self.zk.get('/domains/{}/lasthypervisor'.format(dom_uuid))[0].decode('ascii')
if last_hypervisor != self.name: if last_hypervisor != self.name:
continue continue
print('>>> Setting unmigration for VM "{}".'.format(dom_uuid)) print(bcolours.BLUE + '>>> ' + bcolours.ENDC + 'Setting unmigration for VM "{}".'.format(dom_uuid))
transaction = self.zk.transaction() transaction = self.zk.transaction()
transaction.set_data('/domains/{}/state'.format(dom_uuid), 'migrate'.encode('ascii')) transaction.set_data('/domains/{}/state'.format(dom_uuid), 'migrate'.encode('ascii'))
transaction.set_data('/domains/{}/hypervisor'.format(dom_uuid), self.name.encode('ascii')) transaction.set_data('/domains/{}/hypervisor'.format(dom_uuid), self.name.encode('ascii'))
@ -142,7 +153,7 @@ class NodeInstance():
libvirt_name = "qemu:///system" libvirt_name = "qemu:///system"
conn = libvirt.open(libvirt_name) conn = libvirt.open(libvirt_name)
if conn == None: if conn == None:
print('>>> Failed to open connection to {}'.format(libvirt_name)) print(bcolours.RED + '>>> ' + bcolours.ENDC + 'Failed to open connection to {}'.format(libvirt_name))
return return
# Get past state and update if needed # Get past state and update if needed
@ -193,7 +204,7 @@ class NodeInstance():
conn.close() conn.close()
# Display node information to the terminal # Display node information to the terminal
print('>>> {} - {} keepalive'.format(time.strftime('%d/%m/%Y %H:%M:%S'), self.name)) print(bcolours.PURPLE + '>>> ' + bcolours.ENDC + '{} - {} keepalive'.format(time.strftime('%d/%m/%Y %H:%M:%S'), self.name))
print(' CPUs: {} | Free memory: {} | Load: {}'.format(self.cpucount, self.memfree, self.cpuload)) print(' CPUs: {} | Free memory: {} | Load: {}'.format(self.cpucount, self.memfree, self.cpuload))
print(' Active domains: {}'.format(' '.join(self.domain_list))) print(' Active domains: {}'.format(' '.join(self.domain_list)))
@ -210,7 +221,7 @@ class NodeInstance():
# (A node is considered dead when its keepalive timer is >30s out-of-date while in 'start' state) # (A node is considered dead when its keepalive timer is >30s out-of-date while in 'start' state)
node_deadtime = int(time.time()) - 30 node_deadtime = int(time.time()) - 30
if node_keepalive < node_deadtime and node_state == 'start': if node_keepalive < node_deadtime and node_state == 'start':
print('>>> Node {} is dead! Performing fence operation in 3 seconds.'.format(node_name)) print(bcolours.RED + '>>> ' + bcolours.ENDC + 'Node {} is dead! Performing fence operation in 3 seconds.'.format(node_name))
self.zk.set('/nodes/{}/state'.format(node_name), 'dead'.encode('ascii')) self.zk.set('/nodes/{}/state'.format(node_name), 'dead'.encode('ascii'))
fence_thread = threading.Thread(target=fencenode.fence, args=(node_name, self.zk), kwargs={}) fence_thread = threading.Thread(target=fencenode.fence, args=(node_name, self.zk), kwargs={})
fence_thread.start() fence_thread.start()
@ -248,7 +259,7 @@ class NodeInstance():
pass pass
# Display cluster information to the terminal # Display cluster information to the terminal
print('>>> {} - Cluster status'.format(time.strftime('%d/%m/%Y %H:%M:%S'))) print(bcolours.PURPLE + '>>> ' + bcolours.ENDC + '{} - Cluster status'.format(time.strftime('%d/%m/%Y %H:%M:%S')))
print(' Active nodes: {}'.format(' '.join(self.active_node_list))) print(' Active nodes: {}'.format(' '.join(self.active_node_list)))
print(' Flushed nodes: {}'.format(' '.join(self.flushed_node_list))) print(' Flushed nodes: {}'.format(' '.join(self.flushed_node_list)))
print(' Inactive nodes: {}'.format(' '.join(self.inactive_node_list))) print(' Inactive nodes: {}'.format(' '.join(self.inactive_node_list)))

47
VMInstance.py
View File

@ -22,6 +22,17 @@
import os, sys, uuid, socket, time, threading, libvirt, kazoo.client import os, sys, uuid, socket, time, threading, libvirt, kazoo.client
# ANSII colours for output
class bcolours:
PURPLE = '\033[95m'
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
class VMInstance: class VMInstance:
def __init__(self, domuuid, zk, thishypervisor): def __init__(self, domuuid, zk, thishypervisor):
# Passed-in variables on creation # Passed-in variables on creation
@ -67,52 +78,54 @@ class VMInstance:
# Start up the VM # Start up the VM
def start_vm(self, xmlconfig): def start_vm(self, xmlconfig):
print(">>> {} - Starting VM.".format(self.domuuid)) print(bcolours.BLUE + '>>> ' + bcolours.ENDC + '{} - Starting VM.'.format(self.domuuid))
self.instart = True self.instart = True
# Start up a new Libvirt connection # Start up a new Libvirt connection
libvirt_name = "qemu:///system" libvirt_name = "qemu:///system"
conn = libvirt.open(libvirt_name) conn = libvirt.open(libvirt_name)
if conn == None: if conn == None:
print('>>> {} - Failed to open local libvirt connection.'.format(self.domuuid)) print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Failed to open local libvirt connection.'.format(self.domuuid))
self.instart = False self.instart = False
return return
try: try:
dom = conn.createXML(xmlconfig, 0) dom = conn.createXML(xmlconfig, 0)
except libvirt.libvirtError as e: except libvirt.libvirtError as e:
print('>>> {} - Failed to create VM.'.format(self.domuuid)) print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Failed to create VM.'.format(self.domuuid))
self.zk.set('/domains/{}/state'.format(self.domuuid), 'stop'.encode('ascii')) self.zk.set('/domains/{}/state'.format(self.domuuid), 'stop'.encode('ascii'))
if not self.domuuid in self.thishypervisor.domain_list: if not self.domuuid in self.thishypervisor.domain_list:
self.thishypervisor.domain_list.append(self.domuuid) self.thishypervisor.domain_list.append(self.domuuid)
conn.close() conn.close()
print(bcolours.GREEN + '>>> ' + bcolours.ENDC + '{} - Successfully started VM.'.format(self.domuuid))
self.dom = dom self.dom = dom
self.instart = False self.instart = False
# Stop the VM forcibly without updating state # Stop the VM forcibly without updating state
def terminate_vm(self): def terminate_vm(self):
print(">>> {} - Terminating VM.".format(self.domuuid)) print(bcolours.BLUE + '>>> ' + bcolours.ENDC + '{} - Terminating VM.'.format(self.domuuid))
self.instop = True self.instop = True
try: try:
self.dom.destroy() self.dom.destroy()
except AttributeError: except AttributeError:
pass print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Failed to terminate VM.'.format(self.domuuid))
if self.domuuid in self.thishypervisor.domain_list: if self.domuuid in self.thishypervisor.domain_list:
try: try:
self.thishypervisor.domain_list.remove(self.domuuid) self.thishypervisor.domain_list.remove(self.domuuid)
except ValueError: except ValueError:
pass pass
print(bcolours.GREEN + '>>> ' + bcolours.ENDC + '{} - Successfully terminated VM.'.format(self.domuuid))
# Stop the VM forcibly # Stop the VM forcibly
def stop_vm(self): def stop_vm(self):
print(">>> {} - Forcibly stopping VM.".format(self.domuuid)) print(bcolours.BLUE + '>>> ' + bcolours.ENDC + '{} - Forcibly stopping VM.'.format(self.domuuid))
self.instop = True self.instop = True
try: try:
self.dom.destroy() self.dom.destroy()
except AttributeError: except AttributeError:
pass print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Failed to stop VM.'.format(self.domuuid))
if self.domuuid in self.thishypervisor.domain_list: if self.domuuid in self.thishypervisor.domain_list:
try: try:
self.thishypervisor.domain_list.remove(self.domuuid) self.thishypervisor.domain_list.remove(self.domuuid)
@ -120,12 +133,13 @@ class VMInstance:
pass pass
self.zk.set('/domains/{}/state'.format(self.domuuid), 'stop'.encode('ascii')) self.zk.set('/domains/{}/state'.format(self.domuuid), 'stop'.encode('ascii'))
print(bcolours.GREEN + '>>> ' + bcolours.ENDC + '{} - Successfully stopped VM.'.format(self.domuuid))
self.dom = None self.dom = None
self.instop = False self.instop = False
# Shutdown the VM gracefully # Shutdown the VM gracefully
def shutdown_vm(self): def shutdown_vm(self):
print(">>> {} - Gracefully stopping VM.".format(self.domuuid)) print(bcolours.BLUE + '>>> ' + bcolours.ENDC + '{} - Gracefully stopping VM.'.format(self.domuuid))
self.inshutdown = True self.inshutdown = True
self.dom.shutdown() self.dom.shutdown()
try: try:
@ -135,7 +149,7 @@ class VMInstance:
time.sleep(0.5) time.sleep(0.5)
if tick >= 60: if tick >= 60:
print(">>> {} - Shutdown timeout expired.".format(self.domuuid)) print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Shutdown timeout expired.'.format(self.domuuid))
self.stop_vm() self.stop_vm()
self.inshutdown = False self.inshutdown = False
return return
@ -149,6 +163,7 @@ class VMInstance:
pass pass
self.zk.set('/domains/{}/state'.format(self.domuuid), 'stop'.encode('ascii')) self.zk.set('/domains/{}/state'.format(self.domuuid), 'stop'.encode('ascii'))
print(bcolours.GREEN + '>>> ' + bcolours.ENDC + '{} - Successfully shutdown VM.'.format(self.domuuid))
self.dom = None self.dom = None
self.inshutdown = False self.inshutdown = False
@ -158,14 +173,14 @@ class VMInstance:
if dest_conn == None: if dest_conn == None:
raise raise
except: except:
print('>>> {} - Failed to open connection to qemu+tcp://{}/system; aborting migration.'.format(self.dom_uuid, self.hypervisor)) print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Failed to open connection to qemu+tcp://{}/system; aborting migration.'.format(self.dom_uuid, self.hypervisor))
return 1 return 1
try: try:
target_dom = self.dom.migrate(dest_conn, libvirt.VIR_MIGRATE_LIVE, None, None, 0) target_dom = self.dom.migrate(dest_conn, libvirt.VIR_MIGRATE_LIVE, None, None, 0)
if target_dom == None: if target_dom == None:
raise raise
print('>>> {} - Migrated successfully.'.format(self.domuuid)) print(bcolours.GREEN + '>>> ' + bcolours.ENDC + '{} - Migrated successfully.'.format(self.domuuid))
except: except:
dest_conn.close() dest_conn.close()
return 1 return 1
@ -177,10 +192,10 @@ class VMInstance:
def migrate_vm(self): def migrate_vm(self):
self.inmigrate = True self.inmigrate = True
print('>>> {} - Migrating VM to hypervisor "{}".'.format(self.domuuid, self.hypervisor)) print(bcolours.BLUE + '>>> ' + bcolours.ENDC + '{} - Migrating VM to hypervisor "{}".'.format(self.domuuid, self.hypervisor))
migrate_ret = self.live_migrate_vm(self.hypervisor) migrate_ret = self.live_migrate_vm(self.hypervisor)
if migrate_ret != 0: if migrate_ret != 0:
print('>>> {} - Could not live migrate VM; shutting down to migrate instead.'.format(self.domuuid)) print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Could not live migrate VM; shutting down to migrate instead.'.format(self.domuuid))
self.shutdown_vm() self.shutdown_vm()
time.sleep(1) time.sleep(1)
self.zk.set('/domains/{}/state'.format(self.domuuid), 'start'.encode('ascii')) self.zk.set('/domains/{}/state'.format(self.domuuid), 'start'.encode('ascii'))
@ -194,7 +209,7 @@ class VMInstance:
# Receive the migration from another host (wait until VM is running) # Receive the migration from another host (wait until VM is running)
def receive_migrate(self): def receive_migrate(self):
print('>>> {} - Receiving migration.'.format(self.domuuid)) print(bcolours.BLUE + '>>> ' + bcolours.ENDC + '{} - Receiving migration.'.format(self.domuuid))
self.inreceive = True self.inreceive = True
while True: while True:
self.dom = lookupByUUID(self.domuuid) self.dom = lookupByUUID(self.domuuid)
@ -209,7 +224,7 @@ class VMInstance:
if not self.domuuid in self.thishypervisor.domain_list: if not self.domuuid in self.thishypervisor.domain_list:
self.thishypervisor.domain_list.append(self.domuuid) self.thishypervisor.domain_list.append(self.domuuid)
print('>>> {} - Migrated successfully.'.format(self.domuuid)) print(bcolours.GREEN + '>>> ' + bcolours.ENDC + '{} - Migrated successfully.'.format(self.domuuid))
self.inreceive = False self.inreceive = False
# #
@ -274,7 +289,7 @@ def lookupByUUID(tuuid):
# Open a libvirt connection # Open a libvirt connection
conn = libvirt.open(libvirt_name) conn = libvirt.open(libvirt_name)
if conn == None: if conn == None:
print('>>> {} - Failed to open local libvirt connection.'.format(self.domuuid)) print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Failed to open local libvirt connection.'.format(self.domuuid))
return dom return dom
# Lookup the UUID # Lookup the UUID

17
fencenode.py
View File

@ -22,16 +22,27 @@
import os, sys, libvirt, uuid, kazoo.client, time import os, sys, libvirt, uuid, kazoo.client, time
# ANSII colours for output
class bcolours:
PURPLE = '\033[95m'
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
# #
# Trigger function # Trigger function
# #
def fence(node_name, zk): def fence(node_name, zk):
time.sleep(3) time.sleep(3)
print('>>> Fencing node {} via IPMI reboot signal.'.format(node_name)) print(bcolours.YELLOW + '>>> ' + bcolours.ENDC + 'Fencing node {} via IPMI reboot signal.'.format(node_name))
# DO IPMI FENCE HERE # DO IPMI FENCE HERE
print('>>> Moving VMs from dead hypervisor {} to new hosts.'.format(node_name)) print(bcolours.BLUE + '>>> ' + bcolours.ENDC + 'Moving VMs from dead hypervisor {} to new hosts.'.format(node_name))
dead_node_running_domains = zk.get('/nodes/{}/runningdomains'.format(node_name))[0].decode('ascii').split() dead_node_running_domains = zk.get('/nodes/{}/runningdomains'.format(node_name))[0].decode('ascii').split()
for dom_uuid in dead_node_running_domains: for dom_uuid in dead_node_running_domains:
most_memfree = 0 most_memfree = 0
@ -47,7 +58,7 @@ def fence(node_name, zk):
most_memfree = memfree most_memfree = memfree
target_hypervisor = hypervisor target_hypervisor = hypervisor
print('>>> Moving VM "{}" to hypervisor "{}".'.format(dom_uuid, target_hypervisor)) print(bcolours.BLUE + '>>> ' + bcolours.ENDC + 'Moving VM "{}" to hypervisor "{}".'.format(dom_uuid, target_hypervisor))
transaction = zk.transaction() transaction = zk.transaction()
transaction.set_data('/domains/{}/state'.format(dom_uuid), 'start'.encode('ascii')) transaction.set_data('/domains/{}/state'.format(dom_uuid), 'start'.encode('ascii'))
transaction.set_data('/domains/{}/hypervisor'.format(dom_uuid), target_hypervisor.encode('ascii')) transaction.set_data('/domains/{}/hypervisor'.format(dom_uuid), target_hypervisor.encode('ascii'))

5
pvcd.py
View File

@ -31,8 +31,9 @@ import time
import atexit import atexit
import apscheduler.schedulers.background import apscheduler.schedulers.background
# ANSII colours for output
class bcolours: class bcolours:
HEADER = '\033[95m' PURPLE = '\033[95m'
BLUE = '\033[94m' BLUE = '\033[94m'
GREEN = '\033[92m' GREEN = '\033[92m'
YELLOW = '\033[93m' YELLOW = '\033[93m'
@ -42,7 +43,7 @@ class bcolours:
UNDERLINE = '\033[4m' UNDERLINE = '\033[4m'
def help(): def help():
print(bcolours.HEADER + bcolours.BOLD + bcolours.RED + "pvcd - Parallel Virtual Cluster management daemon" + bcolours.ENDC) print(bcolours.BOLD + "pvcd - Parallel Virtual Cluster management daemon" + bcolours.ENDC)
# exit(0) # exit(0)
help() help()