Improve handling to use hostname
This commit is contained in:
parent
6cd84f725a
commit
dec88f6c1a
|
@ -36,7 +36,7 @@ class NodeInstance():
|
||||||
self.inactive_node_list = []
|
self.inactive_node_list = []
|
||||||
self.s_domain = s_domain
|
self.s_domain = s_domain
|
||||||
self.domain_list = []
|
self.domain_list = []
|
||||||
self.ipmiaddress = ''
|
self.ipmi_hostname = self.config['ipmi_hostname']
|
||||||
|
|
||||||
# Zookeeper handlers for changed states
|
# Zookeeper handlers for changed states
|
||||||
@zk.DataWatch('/nodes/{}/state'.format(self.name))
|
@zk.DataWatch('/nodes/{}/state'.format(self.name))
|
||||||
|
@ -195,12 +195,9 @@ class NodeInstance():
|
||||||
# Close the Libvirt connection
|
# Close the Libvirt connection
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
# Get IPMI address
|
|
||||||
self.ipmiaddress = fencenode.getIPMIAddress()
|
|
||||||
|
|
||||||
# Display node information to the terminal
|
# Display node information to the terminal
|
||||||
ansiiprint.echo('{}{} keepalive{}'.format(ansiiprint.purple(), self.name, ansiiprint.end()), '', 't')
|
ansiiprint.echo('{}{} keepalive{}'.format(ansiiprint.purple(), self.name, ansiiprint.end()), '', 't')
|
||||||
ansiiprint.echo('{0}CPUs:{1} {2} {0}Free memory:{1} {3} {0}Load:{1} {4} {0}IPMI Address:{1} {5}'.format(ansiiprint.bold(), ansiiprint.end(), self.cpucount, self.memfree, self.cpuload, self.ipmiaddress), '', 'c')
|
ansiiprint.echo('{0}CPUs:{1} {2} {0}Free memory:{1} {3} {0}Load:{1} {4} {0}IPMI Address:{1} {5}'.format(ansiiprint.bold(), ansiiprint.end(), self.cpucount, self.memfree, self.cpuload, self.ipmi_hostname), '', 'c')
|
||||||
ansiiprint.echo('{}Active domains:{} {}'.format(ansiiprint.bold(), ansiiprint.end(), ' '.join(self.domain_list)), '', 'c')
|
ansiiprint.echo('{}Active domains:{} {}'.format(ansiiprint.bold(), ansiiprint.end(), ' '.join(self.domain_list)), '', 'c')
|
||||||
|
|
||||||
# Update our local node lists
|
# Update our local node lists
|
||||||
|
@ -213,8 +210,9 @@ class NodeInstance():
|
||||||
node_keepalive = 0
|
node_keepalive = 0
|
||||||
|
|
||||||
# Handle deadtime and fencng if needed
|
# Handle deadtime and fencng if needed
|
||||||
# (A node is considered dead when its keepalive timer is >30s out-of-date while in 'start' state)
|
# (A node is considered dead when its keepalive timer is >6*keepalive_interval seconds
|
||||||
node_deadtime = int(time.time()) - 30
|
# out-of-date while in 'start' state)
|
||||||
|
node_deadtime = int(time.time()) - ( int(self.config['keepalive_interval']) * 6 )
|
||||||
if node_keepalive < node_deadtime and node_state == 'start':
|
if node_keepalive < node_deadtime and node_state == 'start':
|
||||||
ansiiprint.echo('Node {} is dead - performing fence operation in 3 seconds'.format(node_name), '', 'w')
|
ansiiprint.echo('Node {} is dead - performing fence operation in 3 seconds'.format(node_name), '', 'w')
|
||||||
self.zk.set('/nodes/{}/state'.format(node_name), 'dead'.encode('ascii'))
|
self.zk.set('/nodes/{}/state'.format(node_name), 'dead'.encode('ascii'))
|
||||||
|
|
24
fencenode.py
24
fencenode.py
|
@ -29,10 +29,10 @@ def fence(node_name, zk):
|
||||||
time.sleep(3)
|
time.sleep(3)
|
||||||
ansiiprint.echo('Fencing node "{}" via IPMI reboot signal'.format(node_name), '', 'w')
|
ansiiprint.echo('Fencing node "{}" via IPMI reboot signal'.format(node_name), '', 'w')
|
||||||
|
|
||||||
ipmi_address = zk.get('/nodes/{}/ipmiaddress'.format(node_name))[0].decode('ascii')
|
ipmi_hostname = zk.get('/nodes/{}/ipmihostname'.format(node_name))[0].decode('ascii')
|
||||||
ipmi_username = zk.get('/nodes/{}/ipmiusername'.format(node_name))[0].decode('ascii')
|
ipmi_username = zk.get('/nodes/{}/ipmiusername'.format(node_name))[0].decode('ascii')
|
||||||
ipmi_password = zk.get('/nodes/{}/ipmipassword'.format(node_name))[0].decode('ascii')
|
ipmi_password = zk.get('/nodes/{}/ipmipassword'.format(node_name))[0].decode('ascii')
|
||||||
rebootViaIPMI(ipmi_address, ipmi_user, ipmi_password)
|
rebootViaIPMI(ipmi_hostname, ipmi_user, ipmi_password)
|
||||||
|
|
||||||
ansiiprint.echo('Moving VMs from dead hypervisor "{}" to new hosts'.format(node_name), '', 'i')
|
ansiiprint.echo('Moving VMs from dead hypervisor "{}" to new hosts'.format(node_name), '', 'i')
|
||||||
dead_node_running_domains = zk.get('/nodes/{}/runningdomains'.format(node_name))[0].decode('ascii').split()
|
dead_node_running_domains = zk.get('/nodes/{}/runningdomains'.format(node_name))[0].decode('ascii').split()
|
||||||
|
@ -57,17 +57,17 @@ def fence(node_name, zk):
|
||||||
transaction.set_data('/domains/{}/lasthypervisor'.format(dom_uuid), current_hypervisor.encode('ascii'))
|
transaction.set_data('/domains/{}/lasthypervisor'.format(dom_uuid), current_hypervisor.encode('ascii'))
|
||||||
transaction.commit()
|
transaction.commit()
|
||||||
|
|
||||||
def getIPMIAddress():
|
#def getIPMIAddress():
|
||||||
ipmi_command = ['bash', '/fakeipmi.sh']
|
# ipmi_command = ['bash', '/fakeipmi.sh']
|
||||||
|
#
|
||||||
|
# # Get the IPMI address
|
||||||
|
# ipmi_lan_output = subprocess.run(ipmi_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
# ipmi_lan_parsed = ipmi_lan_output.stdout.decode('ascii').split('\n')
|
||||||
|
# ipmi_lan_address = [s for s in ipmi_lan_parsed if re.search('IP Address[ ]*:', s)][0].split(':')[-1].strip()
|
||||||
|
# return ipmi_lan_address
|
||||||
|
|
||||||
# Get the IPMI address
|
def rebootViaIPMI(ipmi_hostname, ipmi_user, ipmi_password):
|
||||||
ipmi_lan_output = subprocess.run(ipmi_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
ipmi_command = ['ipmitool', '-H', ipmi_hostname, '-U', ipmi_user, '-P', ipmi_password, 'chassis', 'power', 'reset']
|
||||||
ipmi_lan_parsed = ipmi_lan_output.stdout.decode('ascii').split('\n')
|
|
||||||
ipmi_lan_address = [s for s in ipmi_lan_parsed if re.search('IP Address[ ]*:', s)][0].split(':')[-1].strip()
|
|
||||||
return ipmi_lan_address
|
|
||||||
|
|
||||||
def rebootViaIPMI(ipmi_address, ipmi_user, ipmi_password):
|
|
||||||
ipmi_command = ['ipmitool', '-H', ipmi_address, '-U', ipmi_user, '-P', ipmi_password, 'chassis', 'power', 'reset']
|
|
||||||
ipmi_command_output = subprocess.run(ipmi_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
ipmi_command_output = subprocess.run(ipmi_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
if ipmi_command_output == 0:
|
if ipmi_command_output == 0:
|
||||||
ansiiprint.echo('Successfully rebooted dead node', '', 'o')
|
ansiiprint.echo('Successfully rebooted dead node', '', 'o')
|
||||||
|
|
8
pvcd.py
8
pvcd.py
|
@ -106,9 +106,9 @@ else:
|
||||||
zk.create('/domains/{}/runningdomains'.format(myhostname), ''.encode('ascii'))
|
zk.create('/domains/{}/runningdomains'.format(myhostname), ''.encode('ascii'))
|
||||||
# Keepalives and fencing information
|
# Keepalives and fencing information
|
||||||
zk.create('/domains/{}/keepalive'.format(myhostname), str(keepalive_time).encode('ascii'))
|
zk.create('/domains/{}/keepalive'.format(myhostname), str(keepalive_time).encode('ascii'))
|
||||||
zk.create('/domains/{}/ipmiaddress'.format(myhostname), ''.encode('ascii'))
|
zk.create('/domains/{}/ipmihostname'.format(config['ipmi_hostname']), ''.encode('ascii'))
|
||||||
zk.create('/domains/{}/ipmiusername'.format(myhostname), ''.encode('ascii'))
|
zk.create('/domains/{}/ipmiusername'.format(config['ipmi_username']), ''.encode('ascii'))
|
||||||
zk.create('/domains/{}/ipmipassword'.format(myhostname), ''.encode('ascii'))
|
zk.create('/domains/{}/ipmipassword'.format(config['ipmi_password']), ''.encode('ascii'))
|
||||||
|
|
||||||
t_node = dict()
|
t_node = dict()
|
||||||
s_domain = dict()
|
s_domain = dict()
|
||||||
|
@ -144,7 +144,7 @@ update_zookeeper = this_node.update_zookeeper
|
||||||
|
|
||||||
# Create timer to update this node in Zookeeper
|
# Create timer to update this node in Zookeeper
|
||||||
update_timer = apscheduler.schedulers.background.BackgroundScheduler()
|
update_timer = apscheduler.schedulers.background.BackgroundScheduler()
|
||||||
update_timer.add_job(update_zookeeper, 'interval', seconds=config['keepalive_interval'])
|
update_timer.add_job(update_zookeeper, 'interval', seconds=int(config['keepalive_interval']))
|
||||||
update_timer.start()
|
update_timer.start()
|
||||||
|
|
||||||
# Tick loop
|
# Tick loop
|
||||||
|
|
Loading…
Reference in New Issue