Changes for pending fencing configurationt

This commit is contained in:
Joshua Boniface 2018-06-06 15:41:06 -04:00
parent 76a431a500
commit 20a45f3d36
3 changed files with 264 additions and 4 deletions

View File

@ -20,7 +20,7 @@
#
###############################################################################
import os, sys, socket, time, libvirt, kazoo.client
import os, sys, socket, time, libvirt, kazoo.client, threading, fencenode
class NodeInstance():
def __init__(self, name, t_node, s_domain, zk):
@ -179,11 +179,13 @@ class NodeInstance():
self.cpucount = conn.getCPUMap()[0]
self.memfree = conn.getFreeMemory()
self.cpuload = os.getloadavg()[0]
keepalive_time = int(time.time())
try:
self.zk.set(self.zkey + '/cpucount', str(self.cpucount).encode('ascii'))
self.zk.set(self.zkey + '/memfree', str(self.memfree).encode('ascii'))
self.zk.set(self.zkey + '/cpuload', str(self.cpuload).encode('ascii'))
self.zk.set(self.zkey + '/runningdomains', ' '.join(self.domain_list).encode('ascii'))
self.zk.set(self.zkey + '/keepalive', keepalive_time.encode('ascii'))
except:
return
@ -198,11 +200,21 @@ class NodeInstance():
# Update our local node lists
for node_name in self.t_node:
try:
state, stat = self.zk.get('/nodes/%s/state' % node_name)
node_state = state.decode('ascii')
node_state = self.zk.get('/nodes/%s/state' % node_name)[0].decode('ascii')
node_keepalive = int(self.zk.get('/nodes/%s/keepalive' % node_name)[0].decode('ascii'))
except:
node_state = 'stop'
node_state = 'unknown'
node_keepalive = 0
# Handle deadtime and fencng if needed (>30 seconds out-of-date keepalive info)
node_deadtime = int(time.time()) - 30
if node_keepalive < node_deadtime and ( node_state != 'dead' and node_state != 'flush' and node_state != 'stop' ):
print('>>> Node {} is dead! Performing fence operation in 3 seconds.'.format(node_name))
self.zk.set('/domains/{}/state'.format(node_name), 'dead'.encode('ascii'))
fence_thread = threading.Thread(target=fencenode.fence, args=(node_name), kwargs={})
fence_thread.start()
# Update the arrays
if node_state == 'start' and node_name not in self.active_node_list:
self.active_node_list.append(node_name)
try:

246
fencenode.py Normal file
View File

@ -0,0 +1,246 @@
#!/usr/bin/env python3
# fencenode.py - Supplemental functions to handle fencing of nodes
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
import os, sys, libvirt, uuid, kazoo.client, lxml.objectify, click
#
# Validate a UUID
#
def validateUUID(dom_uuid):
try:
uuid.UUID(dom_uuid)
return True
except:
return False
#
# Connect and disconnect from Zookeeper
#
def startZKConnection(zk_host):
zk = kazoo.client.KazooClient(hosts=zk_host)
zk.start()
return zk
def stopZKConnection(zk):
zk.stop()
zk.close()
return 0
#
# XML information parsing functions
#
def getInformationFromXML(zk, uuid, long_output):
# Obtain the contents of the XML from Zookeeper
try:
xml = zk.get('/domains/%s/xml' % uuid)[0].decode('ascii')
dstate = zk.get('/domains/%s/state' % uuid)[0].decode('ascii')
dhypervisor = zk.get('/domains/%s/hypervisor' % uuid)[0].decode('ascii')
dlasthypervisor = zk.get('/domains/%s/lasthypervisor' % uuid)[0].decode('ascii')
except:
return None
if dlasthypervisor == '':
dlasthypervisor = 'N/A'
# Parse XML using lxml.objectify
parsed_xml = lxml.objectify.fromstring(xml)
# Get the information we want from it
duuid = parsed_xml.uuid
dname = parsed_xml.name
dmemory = parsed_xml.memory
dmemory_unit = parsed_xml.memory.attrib['unit']
dvcpu = parsed_xml.vcpu
try:
dvcputopo = '{}/{}/{}'.format(parsed_xml.cpu.topology.attrib['sockets'], parsed_xml.cpu.topology.attrib['cores'], parsed_xml.cpu.topology.attrib['threads'])
except:
dvcputopo = 'N/A'
dtype = parsed_xml.os.type
darch = parsed_xml.os.type.attrib['arch']
dmachine = parsed_xml.os.type.attrib['machine']
dfeatures = []
for feature in parsed_xml.features.getchildren():
dfeatures.append(feature.tag)
dconsole = parsed_xml.devices.console.attrib['type']
demulator = parsed_xml.devices.emulator
ddisks = []
dnets = []
dcontrollers = []
for device in parsed_xml.devices.getchildren():
if device.tag == 'disk':
disk_attrib = device.source.attrib
disk_target = device.target.attrib
disk_type = device.attrib['type']
if disk_type == 'network':
disk_obj = { 'type': disk_attrib.get('protocol'), 'name': disk_attrib.get('name'), 'dev': disk_target.get('dev'), 'bus': disk_target.get('bus') }
elif disk_type == 'file':
disk_obj = { 'type': 'file', 'name': disk_attrib.get('file'), 'dev': disk_target.get('dev'), 'bus': disk_target.get('bus') }
else:
disk_obj = {}
ddisks.append(disk_obj)
if device.tag == 'interface':
net_type = device.attrib['type']
net_mac = device.mac.attrib['address']
net_bridge = device.source.attrib[net_type]
net_model = device.model.attrib['type']
net_obj = { 'type': net_type, 'mac': net_mac, 'source': net_bridge, 'model': net_model }
dnets.append(net_obj)
if device.tag == 'controller':
controller_type = device.attrib['type']
try:
controller_model = device.attrib['model']
except KeyError:
controller_model = 'none'
controller_obj = { 'type': controller_type, 'model': controller_model }
dcontrollers.append(controller_obj)
# Format a nice output; do this line-by-line then concat the elements at the end
ainformation = []
ainformation.append('Virtual machine information:')
ainformation.append('')
# Basic information
ainformation.append('UUID: {}'.format(duuid))
ainformation.append('Name: {}'.format(dname))
ainformation.append('Memory: {} {}'.format(dmemory, dmemory_unit))
ainformation.append('vCPUs: {}'.format(dvcpu))
ainformation.append('Topology [S/C/T]: {}'.format(dvcputopo))
if long_output == True:
# Virtualization information
ainformation.append('')
ainformation.append('Emulator: {}'.format(demulator))
ainformation.append('Type: {}'.format(dtype))
ainformation.append('Arch: {}'.format(darch))
ainformation.append('Machine: {}'.format(dmachine))
ainformation.append('Features: {}'.format(' '.join(dfeatures)))
# PVC cluster information
ainformation.append('')
ainformation.append('State: {}'.format(dstate))
ainformation.append('Active Hypervisor: {}'.format(dhypervisor))
ainformation.append('Last Hypervisor: {}'.format(dlasthypervisor))
if long_output == True:
# Disk list
ainformation.append('')
ainformation.append('Disks: ID Type Name Dev Bus')
for disk in ddisks:
ainformation.append(' {0: <3} {1: <5} {2: <20} {3: <4} {4: <5}'.format(ddisks.index(disk), disk['type'], disk['name'], disk['dev'], disk['bus']))
# Network list
ainformation.append('')
ainformation.append('Interfaces: ID Type Source Model MAC')
for net in dnets:
ainformation.append(' {0: <3} {1: <8} {2: <8} {3: <8} {4: <17}'.format(dnets.index(net), net['type'], net['source'], net['model'], net['mac']))
# Controller list
ainformation.append('')
ainformation.append('Controllers: ID Type Model')
for controller in dcontrollers:
ainformation.append(' {0: <3} {1: <8} {2: <8}'.format(dcontrollers.index(controller), controller['type'], controller['model']))
# Join it all together
information = '\n'.join(ainformation)
return information
#
# Cluster search functions
#
def getClusterDomainList(zk):
# Get a list of UUIDs by listing the children of /domains
uuid_list = zk.get_children('/domains')
name_list = []
# For each UUID, get the corresponding name from the data
for uuid in uuid_list:
name_list.append(zk.get('/domains/%s' % uuid)[0].decode('ascii'))
return uuid_list, name_list
def searchClusterByUUID(zk, uuid):
try:
# Get the lists
uuid_list, name_list = getClusterDomainList(zk)
# We're looking for UUID, so find that element ID
index = uuid_list.index(uuid)
# Get the name_list element at that index
name = name_list[index]
except ValueError:
# We didn't find anything
return None
return name
def searchClusterByName(zk, name):
try:
# Get the lists
uuid_list, name_list = getClusterDomainList(zk)
# We're looking for name, so find that element ID
index = name_list.index(name)
# Get the uuid_list element at that index
uuid = uuid_list[index]
except ValueError:
# We didn't find anything
return None
return uuid
#
# Allow mutually exclusive options in Click
#
class MutuallyExclusiveOption(click.Option):
def __init__(self, *args, **kwargs):
meargs = kwargs.pop('mutually_exclusive', [])
_me_arg = []
_me_func = []
for arg in meargs:
_me_arg.append(arg['argument'])
_me_func.append(arg['function'])
self.me_arg = set(_me_arg)
self.me_func = set(_me_func)
help = kwargs.get('help', '')
if self.me_func:
ex_str = ', '.join(self.me_arg)
kwargs['help'] = help + (
' Mutually exclusive with `' + ex_str + '`.'
)
super(MutuallyExclusiveOption, self).__init__(*args, **kwargs)
def handle_parse_result(self, ctx, opts, args):
if self.me_func.intersection(opts) and self.name in opts:
raise click.UsageError(
"Illegal usage: `{}` is mutually exclusive with "
"arguments `{}`.".format(
self.opts[-1],
', '.join(self.me_arg)
)
)
return super(MutuallyExclusiveOption, self).handle_parse_result(
ctx,
opts,
args
)

View File

@ -78,12 +78,14 @@ atexit.register(cleanup)
if zk.exists('%s' % mynodestring):
print("Node is present in Zookeeper")
else:
keepalive_time = int(time.time())
zk.create('%s' % mynodestring, 'hypervisor'.encode('ascii'))
zk.create('%s/state' % mynodestring, 'stop'.encode('ascii'))
zk.create('%s/cpucount' % mynodestring, '0'.encode('ascii'))
zk.create('%s/memfree' % mynodestring, '0'.encode('ascii'))
zk.create('%s/cpuload' % mynodestring, '0.0'.encode('ascii'))
zk.create('%s/runningdomains' % mynodestring, ''.encode('ascii'))
zk.create('%s/keepalive' % mynodestring, keepalive_time.encode('ascii'))
t_node = dict()
s_domain = dict()