From d0a76b2ecbe33832d77e58e60b9fdb1ef7b72d88 Mon Sep 17 00:00:00 2001 From: Joshua Boniface Date: Tue, 12 Jun 2018 02:12:36 -0400 Subject: [PATCH] Reword some of the stuff and handle updates better --- NodeInstance.py | 22 ++++++++++++---------- fencenode.py | 18 ++++++++++++++++-- pvc.py | 8 ++++++-- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/NodeInstance.py b/NodeInstance.py index 5577ddef..9c4a6965 100644 --- a/NodeInstance.py +++ b/NodeInstance.py @@ -155,7 +155,7 @@ class NodeInstance(): transaction.set_data('/domains/{}/state'.format(dom_uuid), 'migrate'.encode('ascii')) transaction.set_data('/domains/{}/hypervisor'.format(dom_uuid), target_hypervisor.encode('ascii')) transaction.set_data('/domains/{}/lasthypervisor'.format(dom_uuid), current_hypervisor.encode('ascii')) - result = transaction.commit() + transaction.commit() # Wait 2s between migrations time.sleep(2) @@ -177,7 +177,7 @@ class NodeInstance(): transaction.set_data('/domains/{}/state'.format(dom_uuid), 'migrate'.encode('ascii')) transaction.set_data('/domains/{}/hypervisor'.format(dom_uuid), self.name.encode('ascii')) transaction.set_data('/domains/{}/lasthypervisor'.format(dom_uuid), ''.encode('ascii')) - result = transaction.commit() + transaction.commit() # Wait 2s between migrations time.sleep(2) @@ -196,7 +196,7 @@ class NodeInstance(): past_state = self.zk.get('/nodes/{}/daemonstate'.format(self.name))[0].decode('ascii') if past_state != 'start': self.daemon_state = 'start' - self.zk.set('/nodes/{}/daemonstate'.format(self.name), 'start'.encode('ascii')) + self.zk.set('/nodes/{}/daemonstate'.format(self.name), 'run'.encode('ascii')) else: self.daemon_state = 'start' @@ -221,13 +221,15 @@ class NodeInstance(): self.domains_count = len(conn.listDomainsID()) keepalive_time = int(time.time()) try: - self.zk.set('/nodes/{}/cpucount'.format(self.name), str(self.cpucount).encode('ascii')) - self.zk.set('/nodes/{}/memused'.format(self.name), str(self.memused).encode('ascii')) - self.zk.set('/nodes/{}/memfree'.format(self.name), str(self.memfree).encode('ascii')) - self.zk.set('/nodes/{}/cpuload'.format(self.name), str(self.cpuload).encode('ascii')) - self.zk.set('/nodes/{}/runningdomains'.format(self.name), ' '.join(self.domain_list).encode('ascii')) - self.zk.set('/nodes/{}/domainscount'.format(self.name), str(self.domains_count).encode('ascii')) - self.zk.set('/nodes/{}/keepalive'.format(self.name), str(keepalive_time).encode('ascii')) + transaction = self.zk.transaction() + transaction.set_data('/nodes/{}/cpucount'.format(self.name), str(self.cpucount).encode('ascii')) + transaction.set_data('/nodes/{}/memused'.format(self.name), str(self.memused).encode('ascii')) + transaction.set_data('/nodes/{}/memfree'.format(self.name), str(self.memfree).encode('ascii')) + transaction.set_data('/nodes/{}/cpuload'.format(self.name), str(self.cpuload).encode('ascii')) + transaction.set_data('/nodes/{}/runningdomains'.format(self.name), ' '.join(self.domain_list).encode('ascii')) + transaction.set_data('/nodes/{}/domainscount'.format(self.name), str(self.domains_count).encode('ascii')) + transaction.set_data('/nodes/{}/keepalive'.format(self.name), str(keepalive_time).encode('ascii')) + transaction.commit() except: return diff --git a/fencenode.py b/fencenode.py index 8862885b..e0a973fb 100644 --- a/fencenode.py +++ b/fencenode.py @@ -26,8 +26,22 @@ import os, sys, libvirt, uuid, kazoo.client, time, subprocess, re, ansiiprint # Trigger function # def fence(node_name, zk): - time.sleep(3) - ansiiprint.echo('Fencing node "{}" via IPMI reboot signal'.format(node_name), '', 'w') + failcount = 0 + while failcount < 3 + # Wait 5 seconds + time.sleep(5) + # Get the state + node_daemon_state = self.zk.get('/nodes/{}/daemonstate'.format(node_name))[0].decode('ascii') + # Is it still 'dead' + if node_daemon_state == 'dead': + failcount += 1 + ansiiprint.echo('Node "{}" failed {} saving throw'.format(node_name, failcount), '', 'w') + # It changed back to something else so it must be alive + else: + ansiiprint.echo('Node "{}" passed a saving throw; canceling fence'.format(node_name), '', 'o') + return + + ansiiprint.echo('Fencing node "{}" via IPMI reboot signal'.format(node_name), '', 'e') ipmi_hostname = zk.get('/nodes/{}/ipmihostname'.format(node_name))[0].decode('ascii') ipmi_username = zk.get('/nodes/{}/ipmiusername'.format(node_name))[0].decode('ascii') diff --git a/pvc.py b/pvc.py index c434c549..f1bf8168 100755 --- a/pvc.py +++ b/pvc.py @@ -225,12 +225,16 @@ RAM (MiB): {node_mem_total: <6} {node_mem_used: <6} {node_mem_free: <6} {node_me # Format the string (elements) for node_name in node_list: - if node_daemon_state[node_name] == 'start': + if node_daemon_state[node_name] == 'run': daemon_state_colour = ansiiprint.green() elif node_daemon_state[node_name] == 'stop': daemon_state_colour = ansiiprint.red() - else: + elif node_daemon_state[node_name] == 'init': daemon_state_colour = ansiiprint.yellow() + elif node_daemon_state[node_name] == 'dead': + daemon_state_colour = ansiiprint.red() + ansiiprint.bold() + else + daemon_state_colour = ansiiprint.blue() if node_domain_state[node_name] == 'ready': domain_state_colour = ansiiprint.green()