Improve handling to use hostname

This commit is contained in:
Joshua Boniface 2018-06-08 12:44:47 -04:00
parent 6cd84f725a
commit dec88f6c1a
3 changed files with 21 additions and 23 deletions

View File

@ -36,7 +36,7 @@ class NodeInstance():
self.inactive_node_list = [] self.inactive_node_list = []
self.s_domain = s_domain self.s_domain = s_domain
self.domain_list = [] self.domain_list = []
self.ipmiaddress = '' self.ipmi_hostname = self.config['ipmi_hostname']
# Zookeeper handlers for changed states # Zookeeper handlers for changed states
@zk.DataWatch('/nodes/{}/state'.format(self.name)) @zk.DataWatch('/nodes/{}/state'.format(self.name))
@ -195,12 +195,9 @@ class NodeInstance():
# Close the Libvirt connection # Close the Libvirt connection
conn.close() conn.close()
# Get IPMI address
self.ipmiaddress = fencenode.getIPMIAddress()
# Display node information to the terminal # Display node information to the terminal
ansiiprint.echo('{}{} keepalive{}'.format(ansiiprint.purple(), self.name, ansiiprint.end()), '', 't') ansiiprint.echo('{}{} keepalive{}'.format(ansiiprint.purple(), self.name, ansiiprint.end()), '', 't')
ansiiprint.echo('{0}CPUs:{1} {2} {0}Free memory:{1} {3} {0}Load:{1} {4} {0}IPMI Address:{1} {5}'.format(ansiiprint.bold(), ansiiprint.end(), self.cpucount, self.memfree, self.cpuload, self.ipmiaddress), '', 'c') ansiiprint.echo('{0}CPUs:{1} {2} {0}Free memory:{1} {3} {0}Load:{1} {4} {0}IPMI Address:{1} {5}'.format(ansiiprint.bold(), ansiiprint.end(), self.cpucount, self.memfree, self.cpuload, self.ipmi_hostname), '', 'c')
ansiiprint.echo('{}Active domains:{} {}'.format(ansiiprint.bold(), ansiiprint.end(), ' '.join(self.domain_list)), '', 'c') ansiiprint.echo('{}Active domains:{} {}'.format(ansiiprint.bold(), ansiiprint.end(), ' '.join(self.domain_list)), '', 'c')
# Update our local node lists # Update our local node lists
@ -213,8 +210,9 @@ class NodeInstance():
node_keepalive = 0 node_keepalive = 0
# Handle deadtime and fencng if needed # Handle deadtime and fencng if needed
# (A node is considered dead when its keepalive timer is >30s out-of-date while in 'start' state) # (A node is considered dead when its keepalive timer is >6*keepalive_interval seconds
node_deadtime = int(time.time()) - 30 # out-of-date while in 'start' state)
node_deadtime = int(time.time()) - ( int(self.config['keepalive_interval']) * 6 )
if node_keepalive < node_deadtime and node_state == 'start': if node_keepalive < node_deadtime and node_state == 'start':
ansiiprint.echo('Node {} is dead - performing fence operation in 3 seconds'.format(node_name), '', 'w') ansiiprint.echo('Node {} is dead - performing fence operation in 3 seconds'.format(node_name), '', 'w')
self.zk.set('/nodes/{}/state'.format(node_name), 'dead'.encode('ascii')) self.zk.set('/nodes/{}/state'.format(node_name), 'dead'.encode('ascii'))

View File

@ -29,10 +29,10 @@ def fence(node_name, zk):
time.sleep(3) time.sleep(3)
ansiiprint.echo('Fencing node "{}" via IPMI reboot signal'.format(node_name), '', 'w') ansiiprint.echo('Fencing node "{}" via IPMI reboot signal'.format(node_name), '', 'w')
ipmi_address = zk.get('/nodes/{}/ipmiaddress'.format(node_name))[0].decode('ascii') ipmi_hostname = zk.get('/nodes/{}/ipmihostname'.format(node_name))[0].decode('ascii')
ipmi_username = zk.get('/nodes/{}/ipmiusername'.format(node_name))[0].decode('ascii') ipmi_username = zk.get('/nodes/{}/ipmiusername'.format(node_name))[0].decode('ascii')
ipmi_password = zk.get('/nodes/{}/ipmipassword'.format(node_name))[0].decode('ascii') ipmi_password = zk.get('/nodes/{}/ipmipassword'.format(node_name))[0].decode('ascii')
rebootViaIPMI(ipmi_address, ipmi_user, ipmi_password) rebootViaIPMI(ipmi_hostname, ipmi_user, ipmi_password)
ansiiprint.echo('Moving VMs from dead hypervisor "{}" to new hosts'.format(node_name), '', 'i') ansiiprint.echo('Moving VMs from dead hypervisor "{}" to new hosts'.format(node_name), '', 'i')
dead_node_running_domains = zk.get('/nodes/{}/runningdomains'.format(node_name))[0].decode('ascii').split() dead_node_running_domains = zk.get('/nodes/{}/runningdomains'.format(node_name))[0].decode('ascii').split()
@ -57,17 +57,17 @@ def fence(node_name, zk):
transaction.set_data('/domains/{}/lasthypervisor'.format(dom_uuid), current_hypervisor.encode('ascii')) transaction.set_data('/domains/{}/lasthypervisor'.format(dom_uuid), current_hypervisor.encode('ascii'))
transaction.commit() transaction.commit()
def getIPMIAddress(): #def getIPMIAddress():
ipmi_command = ['bash', '/fakeipmi.sh'] # ipmi_command = ['bash', '/fakeipmi.sh']
#
# # Get the IPMI address
# ipmi_lan_output = subprocess.run(ipmi_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# ipmi_lan_parsed = ipmi_lan_output.stdout.decode('ascii').split('\n')
# ipmi_lan_address = [s for s in ipmi_lan_parsed if re.search('IP Address[ ]*:', s)][0].split(':')[-1].strip()
# return ipmi_lan_address
# Get the IPMI address def rebootViaIPMI(ipmi_hostname, ipmi_user, ipmi_password):
ipmi_lan_output = subprocess.run(ipmi_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ipmi_command = ['ipmitool', '-H', ipmi_hostname, '-U', ipmi_user, '-P', ipmi_password, 'chassis', 'power', 'reset']
ipmi_lan_parsed = ipmi_lan_output.stdout.decode('ascii').split('\n')
ipmi_lan_address = [s for s in ipmi_lan_parsed if re.search('IP Address[ ]*:', s)][0].split(':')[-1].strip()
return ipmi_lan_address
def rebootViaIPMI(ipmi_address, ipmi_user, ipmi_password):
ipmi_command = ['ipmitool', '-H', ipmi_address, '-U', ipmi_user, '-P', ipmi_password, 'chassis', 'power', 'reset']
ipmi_command_output = subprocess.run(ipmi_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ipmi_command_output = subprocess.run(ipmi_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if ipmi_command_output == 0: if ipmi_command_output == 0:
ansiiprint.echo('Successfully rebooted dead node', '', 'o') ansiiprint.echo('Successfully rebooted dead node', '', 'o')

View File

@ -106,9 +106,9 @@ else:
zk.create('/domains/{}/runningdomains'.format(myhostname), ''.encode('ascii')) zk.create('/domains/{}/runningdomains'.format(myhostname), ''.encode('ascii'))
# Keepalives and fencing information # Keepalives and fencing information
zk.create('/domains/{}/keepalive'.format(myhostname), str(keepalive_time).encode('ascii')) zk.create('/domains/{}/keepalive'.format(myhostname), str(keepalive_time).encode('ascii'))
zk.create('/domains/{}/ipmiaddress'.format(myhostname), ''.encode('ascii')) zk.create('/domains/{}/ipmihostname'.format(config['ipmi_hostname']), ''.encode('ascii'))
zk.create('/domains/{}/ipmiusername'.format(myhostname), ''.encode('ascii')) zk.create('/domains/{}/ipmiusername'.format(config['ipmi_username']), ''.encode('ascii'))
zk.create('/domains/{}/ipmipassword'.format(myhostname), ''.encode('ascii')) zk.create('/domains/{}/ipmipassword'.format(config['ipmi_password']), ''.encode('ascii'))
t_node = dict() t_node = dict()
s_domain = dict() s_domain = dict()
@ -144,7 +144,7 @@ update_zookeeper = this_node.update_zookeeper
# Create timer to update this node in Zookeeper # Create timer to update this node in Zookeeper
update_timer = apscheduler.schedulers.background.BackgroundScheduler() update_timer = apscheduler.schedulers.background.BackgroundScheduler()
update_timer.add_job(update_zookeeper, 'interval', seconds=config['keepalive_interval']) update_timer.add_job(update_zookeeper, 'interval', seconds=int(config['keepalive_interval']))
update_timer.start() update_timer.start()
# Tick loop # Tick loop