From dec88f6c1a27a004a97768100c4c0fb6d80000b2 Mon Sep 17 00:00:00 2001 From: Joshua Boniface Date: Fri, 8 Jun 2018 12:44:47 -0400 Subject: [PATCH] Improve handling to use hostname --- NodeInstance.py | 12 +++++------- fencenode.py | 24 ++++++++++++------------ pvcd.py | 8 ++++---- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/NodeInstance.py b/NodeInstance.py index 0892b340..38aa662a 100644 --- a/NodeInstance.py +++ b/NodeInstance.py @@ -36,7 +36,7 @@ class NodeInstance(): self.inactive_node_list = [] self.s_domain = s_domain self.domain_list = [] - self.ipmiaddress = '' + self.ipmi_hostname = self.config['ipmi_hostname'] # Zookeeper handlers for changed states @zk.DataWatch('/nodes/{}/state'.format(self.name)) @@ -195,12 +195,9 @@ class NodeInstance(): # Close the Libvirt connection conn.close() - # Get IPMI address - self.ipmiaddress = fencenode.getIPMIAddress() - # Display node information to the terminal ansiiprint.echo('{}{} keepalive{}'.format(ansiiprint.purple(), self.name, ansiiprint.end()), '', 't') - ansiiprint.echo('{0}CPUs:{1} {2} {0}Free memory:{1} {3} {0}Load:{1} {4} {0}IPMI Address:{1} {5}'.format(ansiiprint.bold(), ansiiprint.end(), self.cpucount, self.memfree, self.cpuload, self.ipmiaddress), '', 'c') + ansiiprint.echo('{0}CPUs:{1} {2} {0}Free memory:{1} {3} {0}Load:{1} {4} {0}IPMI Address:{1} {5}'.format(ansiiprint.bold(), ansiiprint.end(), self.cpucount, self.memfree, self.cpuload, self.ipmi_hostname), '', 'c') ansiiprint.echo('{}Active domains:{} {}'.format(ansiiprint.bold(), ansiiprint.end(), ' '.join(self.domain_list)), '', 'c') # Update our local node lists @@ -213,8 +210,9 @@ class NodeInstance(): node_keepalive = 0 # Handle deadtime and fencng if needed - # (A node is considered dead when its keepalive timer is >30s out-of-date while in 'start' state) - node_deadtime = int(time.time()) - 30 + # (A node is considered dead when its keepalive timer is >6*keepalive_interval seconds + # out-of-date while in 'start' state) + node_deadtime = int(time.time()) - ( int(self.config['keepalive_interval']) * 6 ) if node_keepalive < node_deadtime and node_state == 'start': ansiiprint.echo('Node {} is dead - performing fence operation in 3 seconds'.format(node_name), '', 'w') self.zk.set('/nodes/{}/state'.format(node_name), 'dead'.encode('ascii')) diff --git a/fencenode.py b/fencenode.py index 03cce0ae..8862885b 100644 --- a/fencenode.py +++ b/fencenode.py @@ -29,10 +29,10 @@ def fence(node_name, zk): time.sleep(3) ansiiprint.echo('Fencing node "{}" via IPMI reboot signal'.format(node_name), '', 'w') - ipmi_address = zk.get('/nodes/{}/ipmiaddress'.format(node_name))[0].decode('ascii') + ipmi_hostname = zk.get('/nodes/{}/ipmihostname'.format(node_name))[0].decode('ascii') ipmi_username = zk.get('/nodes/{}/ipmiusername'.format(node_name))[0].decode('ascii') ipmi_password = zk.get('/nodes/{}/ipmipassword'.format(node_name))[0].decode('ascii') - rebootViaIPMI(ipmi_address, ipmi_user, ipmi_password) + rebootViaIPMI(ipmi_hostname, ipmi_user, ipmi_password) ansiiprint.echo('Moving VMs from dead hypervisor "{}" to new hosts'.format(node_name), '', 'i') dead_node_running_domains = zk.get('/nodes/{}/runningdomains'.format(node_name))[0].decode('ascii').split() @@ -57,17 +57,17 @@ def fence(node_name, zk): transaction.set_data('/domains/{}/lasthypervisor'.format(dom_uuid), current_hypervisor.encode('ascii')) transaction.commit() -def getIPMIAddress(): - ipmi_command = ['bash', '/fakeipmi.sh'] +#def getIPMIAddress(): +# ipmi_command = ['bash', '/fakeipmi.sh'] +# +# # Get the IPMI address +# ipmi_lan_output = subprocess.run(ipmi_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +# ipmi_lan_parsed = ipmi_lan_output.stdout.decode('ascii').split('\n') +# ipmi_lan_address = [s for s in ipmi_lan_parsed if re.search('IP Address[ ]*:', s)][0].split(':')[-1].strip() +# return ipmi_lan_address - # Get the IPMI address - ipmi_lan_output = subprocess.run(ipmi_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - ipmi_lan_parsed = ipmi_lan_output.stdout.decode('ascii').split('\n') - ipmi_lan_address = [s for s in ipmi_lan_parsed if re.search('IP Address[ ]*:', s)][0].split(':')[-1].strip() - return ipmi_lan_address - -def rebootViaIPMI(ipmi_address, ipmi_user, ipmi_password): - ipmi_command = ['ipmitool', '-H', ipmi_address, '-U', ipmi_user, '-P', ipmi_password, 'chassis', 'power', 'reset'] +def rebootViaIPMI(ipmi_hostname, ipmi_user, ipmi_password): + ipmi_command = ['ipmitool', '-H', ipmi_hostname, '-U', ipmi_user, '-P', ipmi_password, 'chassis', 'power', 'reset'] ipmi_command_output = subprocess.run(ipmi_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if ipmi_command_output == 0: ansiiprint.echo('Successfully rebooted dead node', '', 'o') diff --git a/pvcd.py b/pvcd.py index f982911e..53e8c152 100755 --- a/pvcd.py +++ b/pvcd.py @@ -106,9 +106,9 @@ else: zk.create('/domains/{}/runningdomains'.format(myhostname), ''.encode('ascii')) # Keepalives and fencing information zk.create('/domains/{}/keepalive'.format(myhostname), str(keepalive_time).encode('ascii')) - zk.create('/domains/{}/ipmiaddress'.format(myhostname), ''.encode('ascii')) - zk.create('/domains/{}/ipmiusername'.format(myhostname), ''.encode('ascii')) - zk.create('/domains/{}/ipmipassword'.format(myhostname), ''.encode('ascii')) + zk.create('/domains/{}/ipmihostname'.format(config['ipmi_hostname']), ''.encode('ascii')) + zk.create('/domains/{}/ipmiusername'.format(config['ipmi_username']), ''.encode('ascii')) + zk.create('/domains/{}/ipmipassword'.format(config['ipmi_password']), ''.encode('ascii')) t_node = dict() s_domain = dict() @@ -144,7 +144,7 @@ update_zookeeper = this_node.update_zookeeper # Create timer to update this node in Zookeeper update_timer = apscheduler.schedulers.background.BackgroundScheduler() -update_timer.add_job(update_zookeeper, 'interval', seconds=config['keepalive_interval']) +update_timer.add_job(update_zookeeper, 'interval', seconds=int(config['keepalive_interval'])) update_timer.start() # Tick loop