Merge branch 'api'

client-api/client_lib

@@ -0,0 +1 @@
+../client-common

client-api/pvcapi.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+
+# pvcapi.py - PVC HTTP API interface
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018  Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+import flask
+import json
+
+import client_lib.common as pvc_common
+import client_lib.node as pvc_node
+import client_lib.vm as pvc_vm
+import client_lib.network as pvc_network
+import client_lib.ceph as pvc_ceph
+
+zk_host = "hv1:2181,hv2:2181,hv3:2181"
+
+pvcapi = flask.Flask(__name__)
+pvcapi.config["DEBUG"] = True
+
+@pvcapi.route('/api/v1', methods=['GET'])
+def api_root():
+    print(flask.request)
+    print(flask.request.args)
+    return "", 200
+
+@pvcapi.route('/api/v1/node', methods=['GET'])
+def api_node():
+    """
+    Manage the state of a node in the PVC cluster
+    """
+    return "Manage the state of a node in the PVC cluster.\n", 209
+
+@pvcapi.route('/api/v1/node/secondary', methods=['POST'])
+def api_node_secondary():
+    """
+    Take NODE out of primary router mode.
+    """
+    # Get data from flask
+    data = json.loads(flask.request.data.decode('utf8'))
+    # Get node
+    if 'node' in data:
+        node = data['node']
+    else:
+        return "Error: No node provided. Please specify a node.\n", 510
+
+    zk_conn = pvc_common.startZKConnection(zk_host) 
+    retflag, retmsg = pvc_node.secondary_node(zk_conn, node)
+    if retflag:
+        retcode = 200
+    else:
+        retcode = 510
+
+    pvc_common.stopZKConnection(zk_conn)
+    output = {
+        'message': retmsg,
+    }
+    return flask.jsonify(output), retcode
+
+@pvcapi.route('/api/v1/node/primary', methods=['POST'])
+def api_node_primary():
+    """
+    Set NODE to primary router mode.
+    """
+    # Get data from flask
+    data = json.loads(flask.request.data.decode('utf8'))
+    # Get node
+    if 'node' in data:
+        node = data['node']
+    else:
+        return "Error: No node provided. Please specify a node.\n", 510
+
+    zk_conn = pvc_common.startZKConnection(zk_host) 
+    retflag, retmsg = pvc_node.primary_node(zk_conn, node)
+    if retflag:
+        retcode = 200
+    else:
+        retcode = 510
+
+    pvc_common.stopZKConnection(zk_conn)
+    output = {
+        'message': retmsg,
+    }
+    return flask.jsonify(output), retcode
+
+@pvcapi.route('/api/v1/node/flush', methods=['POST'])
+def api_node_flush():
+    """
+    Flush NODE of running VMs.
+    """
+    # Get data from flask
+    data = json.loads(flask.request.data.decode('utf8'))
+    # Get node
+    if 'node' in data:
+        node = data['node']
+    else:
+        return "Error: No node provided. Please specify a node.\n", 510
+
+    zk_conn = pvc_common.startZKConnection(zk_host)
+    retflag, retmsg = pvc_node.flush_node(zk_conn, node, False)
+    if retflag:
+        retcode = 200
+    else:
+        retcode = 510
+
+    pvc_common.stopZKConnection(zk_conn)
+    output = {
+        'message': retmsg,
+    }
+    return flask.jsonify(output), retcode
+
+@pvcapi.route('/api/v1/node/unflush', methods=['POST'])
+@pvcapi.route('/api/v1/node/ready', methods=['POST'])
+def api_node_ready():
+    """
+    Restore NODE to active service.
+    """
+    # Get data from flask
+    data = json.loads(flask.request.data.decode('utf8'))
+    # Get node
+    if 'node' in data:
+        node = data['node']
+    else:
+        return "Error: No node provided. Please specify a node.\n", 510
+
+    zk_conn = pvc_common.startZKConnection(zk_host)
+    retflag, retmsg = pvc_node.ready_node(zk_conn, node)
+    if retflag:
+        retcode = 200
+    else:
+        retcode = 510
+
+    pvc_common.stopZKConnection(zk_conn)
+    output = {
+        'message': retmsg,
+    }
+    return flask.jsonify(output), retcode
+
+
+#@pvcapi.route('/api/v1/node/info', methods=['GET'])
+#@pvcapi.route('/api/v1/node/list', methods=['GET'])
+# VM endpoints
+#@pvcapi.route('/api/v1/vm', methods=['GET'])
+#@pvcapi.route('/api/v1/vm/add', methods=['POST'])
+#@pvcapi.route('/api/v1/vm/define', methods=['POST'])
+#@pvcapi.route('/api/v1/vm/modify', methods=['POST'])
+#@pvcapi.route('/api/v1/vm/undefine', methods=['POST'])
+#@pvcapi.route('/api/v1/vm/dump', methods=['GET'])
+#@pvcapi.route('/api/v1/vm/start', methods=['POST'])
+#@pvcapi.route('/api/v1/vm/restart', methods=['POST'])
+#@pvcapi.route('/api/v1/vm/shutdown', methods=['POST'])
+#@pvcapi.route('/api/v1/vm/stop', methods=['POST'])
+#@pvcapi.route('/api/v1/vm/move', methods=['POST'])
+#@pvcapi.route('/api/v1/vm/migrate', methods=['POST'])
+#@pvcapi.route('/api/v1/vm/unmigrate', methods=['POST'])
+#@pvcapi.route('/api/v1/vm/info', methods=['GET'])
+#@pvcapi.route('/api/v1/vm/list', methods=['GET'])
+# Network endpoints
+#@pvcapi.route('/api/v1/network', methods=['GET'])
+#@pvcapi.route('/api/v1/network/add', methods=['POST'])
+#@pvcapi.route('/api/v1/network/modify', methods=['POST'])
+#@pvcapi.route('/api/v1/network/remove', methods=['POST'])
+#@pvcapi.route('/api/v1/network/info', methods=['GET'])
+#@pvcapi.route('/api/v1/network/list', methods=['GET'])
+#@pvcapi.route('/api/v1/network/dhcp', methods=['GET'])
+#@pvcapi.route('/api/v1/network/dhcp/list', methods=['GET'])
+#@pvcapi.route('/api/v1/network/dhcp/static', methods=['GET'])
+#@pvcapi.route('/api/v1/network/dhcp/static/add', methods=['POST'])
+#@pvcapi.route('/api/v1/network/dhcp/static/remove', methods=['POST'])
+#@pvcapi.route('/api/v1/network/dhcp/static/list', methods=['GET'])
+#@pvcapi.route('/api/v1/network/acl', methods=['GET'])
+#@pvcapi.route('/api/v1/network/acl/add', methods=['POST'])
+#@pvcapi.route('/api/v1/network/acl/remove', methods=['POST'])
+#@pvcapi.route('/api/v1/network/acl/list', methods=['GET'])
+# Ceph endpoints
+#@pvcapi.route('/api/v1/ceph', methods=['GET'])
+#@pvcapi.route('/api/v1/ceph/status', methods=['GET'])
+#@pvcapi.route('/api/v1/ceph/osd', methods=['GET'])
+#@pvcapi.route('/api/v1/ceph/osd/add', methods=['POST'])
+#@pvcapi.route('/api/v1/ceph/osd/remove', methods=['POST'])
+#@pvcapi.route('/api/v1/ceph/osd/in', methods=['POST'])
+#@pvcapi.route('/api/v1/ceph/osd/out', methods=['POST'])
+#@pvcapi.route('/api/v1/ceph/osd/set', methods=['POST'])
+#@pvcapi.route('/api/v1/ceph/osd/unset', methods=['POST'])
+#@pvcapi.route('/api/v1/ceph/osd/list', methods=['GET'])
+#@pvcapi.route('/api/v1/ceph/pool', methods=['GET'])
+#@pvcapi.route('/api/v1/ceph/pool/add', methods=['POST'])
+#@pvcapi.route('/api/v1/ceph/pool/remove', methods=['POST'])
+#@pvcapi.route('/api/v1/ceph/pool/list', methods=['GET'])
+
+pvcapi.run()

client-cli/pvc.py

@@ -30,6 +30,7 @@ import re
 import colorama
 import yaml
 
+import client_lib.ansiprint as ansiprint
 import client_lib.common as pvc_common
 import client_lib.node as pvc_node
 import client_lib.vm as pvc_vm
@@ -162,8 +163,16 @@ def node_info(node, long_output):
     """
 
     zk_conn = pvc_common.startZKConnection(zk_host)
-    retcode, retmsg = pvc_node.get_info(zk_conn, node, long_output)
+    retcode, retdata = pvc_node.get_info(zk_conn, node)
-    cleanup(retcode, retmsg, zk_conn)
+    if retcode:
+        pvc_node.format_info(zk_conn, retdata, long_output)
+        if long_output:
+            click.echo('{}Virtual machines on node:{}'.format(ansiprint.bold(), ansiprint.end()))
+            click.echo('')
+            pvc_vm.get_list(zk_conn, node, None, None, None)
+            click.echo('')
+        retdata = ''
+    cleanup(retcode, retdata, zk_conn)
 
 ###############################################################################
 # pvc node list
@@ -178,8 +187,11 @@ def node_list(limit):
     """
 
     zk_conn = pvc_common.startZKConnection(zk_host)
-    retcode, retmsg = pvc_node.get_list(zk_conn, limit)
+    retcode, retdata = pvc_node.get_list(zk_conn, limit)
-    cleanup(retcode, retmsg, zk_conn)
+    if retcode:
+        pvc_node.format_list(retdata)
+        retdata = ''
+    cleanup(retcode, retdata, zk_conn)
 
 ###############################################################################
 # pvc vm

client-cli/pvc_init.py

@@ -650,6 +650,8 @@ def init_zookeeper(zk_host):
     transaction.create('/ceph', ''.encode('ascii'))
     transaction.create('/ceph/osds', ''.encode('ascii'))
     transaction.create('/ceph/pools', ''.encode('ascii'))
+    transaction.create('/locks', ''.encode('ascii'))
+    transaction.create('/locks/flush_lock', 'False'.encode('ascii'))
     transaction.commit()
 
     # Close the Zookeeper connection

client-common/node.py

@@ -39,7 +39,10 @@ import client_lib.zkhandler as zkhandler
 import client_lib.common as common
 import client_lib.vm as pvc_vm
 
-def getInformationFromNode(zk_conn, node_name, long_output):
+def getInformationFromNode(zk_conn, node_name):
+    """
+    Gather information about a node from the Zookeeper database and return a dict() containing it.
+    """
     node_daemon_state = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(node_name))
     node_coordinator_state = zkhandler.readdata(zk_conn, '/nodes/{}/routerstate'.format(node_name))
     node_domain_state = zkhandler.readdata(zk_conn, '/nodes/{}/domainstate'.format(node_name))
@@ -48,63 +51,36 @@ def getInformationFromNode(zk_conn, node_name, long_output):
     node_kernel = node_static_data[1]
     node_os = node_static_data[2]
     node_arch = node_static_data[3]
+    node_mem_total = int(zkhandler.readdata(zk_conn, '/nodes/{}/memtotal'.format(node_name)))
     node_mem_allocated = int(zkhandler.readdata(zk_conn, '/nodes/{}/memalloc'.format(node_name)))
     node_mem_used = int(zkhandler.readdata(zk_conn, '/nodes/{}/memused'.format(node_name)))
     node_mem_free = int(zkhandler.readdata(zk_conn, '/nodes/{}/memfree'.format(node_name)))
-    node_mem_total = node_mem_used + node_mem_free
     node_load = zkhandler.readdata(zk_conn, '/nodes/{}/cpuload'.format(node_name))
     node_domains_count = zkhandler.readdata(zk_conn, '/nodes/{}/domainscount'.format(node_name))
     node_running_domains = zkhandler.readdata(zk_conn, '/nodes/{}/runningdomains'.format(node_name)).split()
 
-    if node_daemon_state == 'run':
+    # Construct a data structure to represent the data
-        daemon_state_colour = ansiprint.green()
+    node_information = {
-    elif node_daemon_state == 'stop':
+        'name': node_name,
-        daemon_state_colour = ansiprint.red()
+        'daemon_state': node_daemon_state,
-    elif node_daemon_state == 'init':
+        'coordinator_state': node_coordinator_state,
-        daemon_state_colour = ansiprint.yellow()
+        'domain_state': node_domain_state,
-    elif node_daemon_state == 'dead':
+        'cpu_count': node_cpu_count,
-        daemon_state_colour = ansiprint.red() + ansiprint.bold()
+        'kernel': node_kernel,
-    else:
+        'os': node_os,
-        daemon_state_colour = ansiprint.blue()
+        'arch': node_arch,
+        'load': node_load,
+        'domains_count': node_domains_count,
+        'running_domains': node_running_domains,
+        'memory': {
+            'total': node_mem_total,
+            'allocated': node_mem_allocated,
+            'used': node_mem_used,
+            'free': node_mem_free
+        }
+    }
 
-    if node_coordinator_state == 'primary':
+    return node_information
-        coordinator_state_colour = ansiprint.green()
-    elif node_coordinator_state == 'secondary':
-        coordinator_state_colour = ansiprint.blue()
-    else:
-        coordinator_state_colour = ansiprint.purple()
-
-    if node_domain_state == 'ready':
-        domain_state_colour = ansiprint.green()
-    else:
-        domain_state_colour = ansiprint.blue()
-
-    # Format a nice output; do this line-by-line then concat the elements at the end
-    ainformation = []
-    ainformation.append('{}Node information:{}'.format(ansiprint.bold(), ansiprint.end()))
-    ainformation.append('')
-    # Basic information
-    ainformation.append('{}Name:{}                 {}'.format(ansiprint.purple(), ansiprint.end(), node_name))
-    ainformation.append('{}Daemon State:{}         {}{}{}'.format(ansiprint.purple(), ansiprint.end(), daemon_state_colour, node_daemon_state, ansiprint.end()))
-    ainformation.append('{}Coordinator State:{}         {}{}{}'.format(ansiprint.purple(), ansiprint.end(), coordinator_state_colour, node_coordinator_state, ansiprint.end()))
-    ainformation.append('{}Domain State:{}         {}{}{}'.format(ansiprint.purple(), ansiprint.end(), domain_state_colour, node_domain_state, ansiprint.end()))
-    ainformation.append('{}Active VM Count:{}      {}'.format(ansiprint.purple(), ansiprint.end(), node_domains_count))
-    if long_output == True:
-        ainformation.append('')
-        ainformation.append('{}Architecture:{}         {}'.format(ansiprint.purple(), ansiprint.end(), node_arch))
-        ainformation.append('{}Operating System:{}     {}'.format(ansiprint.purple(), ansiprint.end(), node_os))
-        ainformation.append('{}Kernel Version:{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_kernel))
-    ainformation.append('')
-    ainformation.append('{}CPUs:{}                 {}'.format(ansiprint.purple(), ansiprint.end(), node_cpu_count))
-    ainformation.append('{}Load:{}                 {}'.format(ansiprint.purple(), ansiprint.end(), node_load))
-    ainformation.append('{}Total RAM (MiB):{}      {}'.format(ansiprint.purple(), ansiprint.end(), node_mem_total))
-    ainformation.append('{}Used RAM (MiB):{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_mem_used))
-    ainformation.append('{}Free RAM (MiB):{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_mem_free))
-    ainformation.append('{}Allocated RAM (MiB):{}  {}'.format(ansiprint.purple(), ansiprint.end(), node_mem_allocated))
-
-    # Join it all together
-    information = '\n'.join(ainformation)
-    return information
 
 #
 # Direct Functions
@@ -122,14 +98,14 @@ def secondary_node(zk_conn, node):
     # Get current state
     current_state = zkhandler.readdata(zk_conn, '/nodes/{}/routerstate'.format(node))
     if current_state == 'primary':
-        click.echo('Setting node {} in secondary router mode.'.format(node))
+        retmsg = 'Setting node {} in secondary router mode.'.format(node)
         zkhandler.writedata(zk_conn, {
             '/primary_node': 'none'
         })
     else:
-        click.echo('Node {} is already in secondary router mode.'.format(node))
+        return False, 'Node {} is already in secondary router mode.'.format(node)
 
-    return True, ''
+    return True, retmsg
 
 def primary_node(zk_conn, node):
     # Verify node is valid
@@ -144,73 +120,75 @@ def primary_node(zk_conn, node):
     # Get current state
     current_state = zkhandler.readdata(zk_conn, '/nodes/{}/routerstate'.format(node))
     if current_state == 'secondary':
-        click.echo('Setting node {} in primary router mode.'.format(node))
+        retmsg = 'Setting node {} in primary router mode.'.format(node)
         zkhandler.writedata(zk_conn, {
             '/primary_node': node
         })
     else:
-        click.echo('Node {} is already in primary router mode.'.format(node))
+        return False, 'Node {} is already in primary router mode.'.format(node)
 
-    return True, ''
+    return True, retmsg
 
 def flush_node(zk_conn, node, wait):
     # Verify node is valid
     if not common.verifyNode(zk_conn, node):
         return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
 
-    click.echo('Flushing hypervisor {} of running VMs.'.format(node))
+    if zkhandler.readdata(zk_conn, '/locks/flush_lock') == 'True':
+        retmsg = 'Flushing hypervisor {} of running VMs. A flush lock currently exists; flush will continue once the lock is freed.'.format(node)
+        lock_wait = True
+    else:
+        retmsg = 'Flushing hypervisor {} of running VMs.'.format(node)
+        lock_wait = False
+        
+    # Wait cannot be triggered from the API
+    if wait:
+        click.echo(retmsg)
+        retmsg = ""
+        if lock_wait:
+            time.sleep(1)
+            while zkhandler.readdata(zk_conn, '/locks/flush_lock') == 'True':
+                time.sleep(1)
+            click.echo('Previous flush completed. Proceeding with flush.')
 
     # Add the new domain to Zookeeper
     zkhandler.writedata(zk_conn, {
         '/nodes/{}/domainstate'.format(node): 'flush'
     })
 
-    if wait == True:
+    # Wait cannot be triggered from the API
-        while True:
+    if wait:
+        time.sleep(1)
+        while zkhandler.readdata(zk_conn, '/locks/flush_lock') == 'True':
             time.sleep(1)
-            node_state = zkhandler.readdata(zk_conn, '/nodes/{}/domainstate'.format(node))
-            if node_state == "flushed":
-                break
 
-    return True, ''
+    return True, retmsg
 
 def ready_node(zk_conn, node):
     # Verify node is valid
     if not common.verifyNode(zk_conn, node):
         return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
 
-    click.echo('Restoring hypervisor {} to active service.'.format(node))
+    retmsg = 'Restoring hypervisor {} to active service.'.format(node)
 
     # Add the new domain to Zookeeper
     zkhandler.writedata(zk_conn, {
         '/nodes/{}/domainstate'.format(node): 'unflush'
     })
 
-    return True, ''
+    return True, retmsg
 
-def get_info(zk_conn, node, long_output):
+def get_info(zk_conn, node):
     # Verify node is valid
     if not common.verifyNode(zk_conn, node):
         return False, 'ERROR: No node named "{}" is present in the cluster.'.format(node)
 
     # Get information about node in a pretty format
-    information = getInformationFromNode(zk_conn, node, long_output)
+    node_information = getInformationFromNode(zk_conn, node)
-
+    if node_information == None:
-    if information == None:
         return False, 'ERROR: Could not find a node matching that name.'
 
-    click.echo(information)
+    return True, node_information
-
-    if long_output == True:
-        click.echo('')
-        click.echo('{}Virtual machines on node:{}'.format(ansiprint.bold(), ansiprint.end()))
-        click.echo('')
-        # List all VMs on this node
-        pvc_vm.get_list(zk_conn, node, None)
-
-    click.echo('')
-
-    return True, ''
 
 def get_list(zk_conn, limit):
     # Match our limit
@@ -226,41 +204,77 @@ def get_list(zk_conn, limit):
                     limit = limit + '.*'
 
                 if re.match(limit, node) != None:
-                    node_list.append(node)
+                    node_list.append(getInformationFromNode(zk_conn, node))
             except Exception as e:
                 return False, 'Regex Error: {}'.format(e)
         else:
-            node_list.append(node)
+            node_list.append(getInformationFromNode(zk_conn, node))
 
+    return True, node_list
+
+#
+# CLI-specific functions
+#
+def getOutputColours(node_information):
+    if node_information['daemon_state'] == 'run':
+        daemon_state_colour = ansiprint.green()
+    elif node_information['daemon_state'] == 'stop':
+        daemon_state_colour = ansiprint.red()
+    elif node_information['daemon_state'] == 'init':
+        daemon_state_colour = ansiprint.yellow()
+    elif node_information['daemon_state'] == 'dead':
+        daemon_state_colour = ansiprint.red() + ansiprint.bold()
+    else:
+        daemon_state_colour = ansiprint.blue()
+
+    if node_information['coordinator_state'] == 'primary':
+        coordinator_state_colour = ansiprint.green()
+    elif node_information['coordinator_state'] == 'secondary':
+        coordinator_state_colour = ansiprint.blue()
+    else:
+        coordinator_state_colour = ansiprint.purple()
+
+    if node_information['domain_state'] == 'ready':
+        domain_state_colour = ansiprint.green()
+    else:
+        domain_state_colour = ansiprint.blue()
+
+    return daemon_state_colour, coordinator_state_colour, domain_state_colour
+
+def format_info(zk_conn, node_information, long_output):
+    daemon_state_colour, coordinator_state_colour, domain_state_colour = getOutputColours(node_information)
+
+    # Format a nice output; do this line-by-line then concat the elements at the end
+    ainformation = []
+    # Basic information
+    ainformation.append('{}Name:{}                 {}'.format(ansiprint.purple(), ansiprint.end(), node_information['name']))
+    ainformation.append('{}Daemon State:{}         {}{}{}'.format(ansiprint.purple(), ansiprint.end(), daemon_state_colour, node_information['daemon_state'], ansiprint.end()))
+    ainformation.append('{}Coordinator State:{}    {}{}{}'.format(ansiprint.purple(), ansiprint.end(), coordinator_state_colour, node_information['coordinator_state'], ansiprint.end()))
+    ainformation.append('{}Domain State:{}         {}{}{}'.format(ansiprint.purple(), ansiprint.end(), domain_state_colour, node_information['domain_state'], ansiprint.end()))
+    ainformation.append('{}Active VM Count:{}      {}'.format(ansiprint.purple(), ansiprint.end(), node_information['domains_count']))
+    if long_output:
+        ainformation.append('')
+        ainformation.append('{}Architecture:{}         {}'.format(ansiprint.purple(), ansiprint.end(), node_information['arch']))
+        ainformation.append('{}Operating System:{}     {}'.format(ansiprint.purple(), ansiprint.end(), node_information['os']))
+        ainformation.append('{}Kernel Version:{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_information['kernel']))
+    ainformation.append('')
+    ainformation.append('{}CPUs:{}                 {}'.format(ansiprint.purple(), ansiprint.end(), node_information['cpu_count']))
+    ainformation.append('{}Load:{}                 {}'.format(ansiprint.purple(), ansiprint.end(), node_information['load']))
+    ainformation.append('{}Total RAM (MiB):{}      {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['total']))
+    ainformation.append('{}Used RAM (MiB):{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['used']))
+    ainformation.append('{}Free RAM (MiB):{}       {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['free']))
+    ainformation.append('{}Allocated RAM (MiB):{}  {}'.format(ansiprint.purple(), ansiprint.end(), node_information['memory']['allocated']))
+
+    # Join it all together
+    information = '\n'.join(ainformation)
+    click.echo(information)
+
+    click.echo('')
+
+def format_list(node_list):
     node_list_output = []
-    node_daemon_state = {}
-    node_coordinator_state = {}
-    node_domain_state = {}
-    node_cpu_count = {}
-    node_mem_used = {}
-    node_mem_free = {}
-    node_mem_total = {}
-    node_mem_allocated = {}
-    node_domains_count = {}
-    node_running_domains = {}
-    node_load = {}
-
-    # Gather information for printing
-    for node_name in node_list:
-        node_daemon_state[node_name] = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(node_name))
-        node_coordinator_state[node_name] = zkhandler.readdata(zk_conn, '/nodes/{}/routerstate'.format(node_name))
-        node_domain_state[node_name] = zkhandler.readdata(zk_conn, '/nodes/{}/domainstate'.format(node_name))
-        node_cpu_count[node_name] = zkhandler.readdata(zk_conn, '/nodes/{}/staticdata'.format(node_name)).split()[0]
-        node_mem_allocated[node_name] = int(zkhandler.readdata(zk_conn, '/nodes/{}/memalloc'.format(node_name)))
-        node_mem_used[node_name] = int(zkhandler.readdata(zk_conn, '/nodes/{}/memused'.format(node_name)))
-        node_mem_free[node_name] = int(zkhandler.readdata(zk_conn, '/nodes/{}/memfree'.format(node_name)))
-        node_mem_total[node_name] = node_mem_used[node_name] + node_mem_free[node_name]
-        node_load[node_name] = zkhandler.readdata(zk_conn, '/nodes/{}/cpuload'.format(node_name))
-        node_domains_count[node_name] = zkhandler.readdata(zk_conn, '/nodes/{}/domainscount'.format(node_name))
-        node_running_domains[node_name] = zkhandler.readdata(zk_conn, '/nodes/{}/runningdomains'.format(node_name)).split()
 
     # Determine optimal column widths
-    # Dynamic columns: node_name, daemon_state, network_state, domain_state, load
     node_name_length = 5
     daemon_state_length = 7
     coordinator_state_length = 12
@@ -272,49 +286,49 @@ def get_list(zk_conn, limit):
     mem_used_length = 5
     mem_free_length = 5
     mem_alloc_length = 4
-    for node_name in node_list:
+    for node_information in node_list:
         # node_name column
-        _node_name_length = len(node_name) + 1
+        _node_name_length = len(node_information['name']) + 1
         if _node_name_length > node_name_length:
             node_name_length = _node_name_length
         # daemon_state column
-        _daemon_state_length = len(node_daemon_state[node_name]) + 1
+        _daemon_state_length = len(node_information['daemon_state']) + 1
         if _daemon_state_length > daemon_state_length:
             daemon_state_length = _daemon_state_length
         # coordinator_state column
-        _coordinator_state_length = len(node_coordinator_state[node_name]) + 1
+        _coordinator_state_length = len(node_information['coordinator_state']) + 1
         if _coordinator_state_length > coordinator_state_length:
             coordinator_state_length = _coordinator_state_length
         # domain_state column
-        _domain_state_length = len(node_domain_state[node_name]) + 1
+        _domain_state_length = len(node_information['domain_state']) + 1
         if _domain_state_length > domain_state_length:
             domain_state_length = _domain_state_length
         # domains_count column
-        _domains_count_length = len(node_domains_count[node_name]) + 1
+        _domains_count_length = len(node_information['domains_count']) + 1
         if _domains_count_length > domains_count_length:
             domains_count_length = _domains_count_length
         # cpu_count column
-        _cpu_count_length = len(node_cpu_count[node_name]) + 1
+        _cpu_count_length = len(node_information['cpu_count']) + 1
         if _cpu_count_length > cpu_count_length:
             cpu_count_length = _cpu_count_length
         # load column
-        _load_length = len(node_load[node_name]) + 1
+        _load_length = len(node_information['load']) + 1
         if _load_length > load_length:
             load_length = _load_length
         # mem_total column
-        _mem_total_length = len(str(node_mem_total[node_name])) + 1
+        _mem_total_length = len(str(node_information['memory']['total'])) + 1
         if _mem_total_length > mem_total_length:
             mem_total_length = _mem_total_length
         # mem_used column
-        _mem_used_length = len(str(node_mem_used[node_name])) + 1
+        _mem_used_length = len(str(node_information['memory']['used'])) + 1
         if _mem_used_length > mem_used_length:
             mem_used_length = _mem_used_length
         # mem_free column
-        _mem_free_length = len(str(node_mem_free[node_name])) + 1
+        _mem_free_length = len(str(node_information['memory']['free'])) + 1
         if _mem_free_length > mem_free_length:
             mem_free_length = _mem_free_length
         # mem_alloc column
-        _mem_alloc_length = len(str(node_mem_allocated[node_name])) + 1
+        _mem_alloc_length = len(str(node_information['memory']['allocated'])) + 1
         if _mem_alloc_length > mem_alloc_length:
             mem_alloc_length = _mem_alloc_length
 
@@ -356,33 +370,8 @@ Mem (M): {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length
     )
             
     # Format the string (elements)
-    for node_name in node_list:
+    for node_information in node_list:
-        if node_daemon_state[node_name] == 'run':
+        daemon_state_colour, coordinator_state_colour, domain_state_colour = getOutputColours(node_information)
-            daemon_state_colour = ansiprint.green()
-        elif node_daemon_state[node_name] == 'stop':
-            daemon_state_colour = ansiprint.red()
-        elif node_daemon_state[node_name] == 'init':
-            daemon_state_colour = ansiprint.yellow()
-        elif node_daemon_state[node_name] == 'dead':
-            daemon_state_colour = ansiprint.red() + ansiprint.bold()
-        else:
-            daemon_state_colour = ansiprint.blue()
-
-        if node_coordinator_state[node_name] == 'primary':
-            coordinator_state_colour = ansiprint.green()
-        elif node_coordinator_state[node_name] == 'secondary':
-            coordinator_state_colour = ansiprint.blue()
-        else:
-            coordinator_state_colour = ansiprint.purple()
-
-        if node_mem_allocated[node_name] != 0 and node_mem_allocated[node_name] >= node_mem_total[node_name]:
-            node_domain_state[node_name] = 'overprov'
-            domain_state_colour = ansiprint.yellow()
-        elif node_domain_state[node_name] == 'ready':
-            domain_state_colour = ansiprint.green()
-        else:
-            domain_state_colour = ansiprint.blue()
-
         node_list_output.append(
             '{bold}{node_name: <{node_name_length}} \
     {daemon_state_colour}{node_daemon_state: <{daemon_state_length}}{end_colour} {coordinator_state_colour}{node_coordinator_state: <{coordinator_state_length}}{end_colour} {domain_state_colour}{node_domain_state: <{domain_state_length}}{end_colour} \
@@ -405,20 +394,18 @@ Mem (M): {node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length
                 coordinator_state_colour=coordinator_state_colour,
                 domain_state_colour=domain_state_colour,
                 end_colour=ansiprint.end(),
-                node_name=node_name,
+                node_name=node_information['name'],
-                node_daemon_state=node_daemon_state[node_name],
+                node_daemon_state=node_information['daemon_state'],
-                node_coordinator_state=node_coordinator_state[node_name],
+                node_coordinator_state=node_information['coordinator_state'],
-                node_domain_state=node_domain_state[node_name],
+                node_domain_state=node_information['domain_state'],
-                node_domains_count=node_domains_count[node_name],
+                node_domains_count=node_information['domains_count'],
-                node_cpu_count=node_cpu_count[node_name],
+                node_cpu_count=node_information['cpu_count'],
-                node_load=node_load[node_name],
+                node_load=node_information['load'],
-                node_mem_total=node_mem_total[node_name],
+                node_mem_total=node_information['memory']['total'],
-                node_mem_used=node_mem_used[node_name],
+                node_mem_used=node_information['memory']['used'],
-                node_mem_free=node_mem_free[node_name],
+                node_mem_free=node_information['memory']['free'],
-                node_mem_allocated=node_mem_allocated[node_name]
+                node_mem_allocated=node_information['memory']['allocated']
             )
         )
 
     click.echo('\n'.join(sorted(node_list_output)))
-
-    return True, ''

node-daemon/pvcd/Daemon.py

@@ -530,28 +530,29 @@ if zk_conn.exists('/nodes/{}'.format(myhostname)):
 else:
     logger.out("Node is " + logger.fmt_red + "absent" + logger.fmt_end + " in Zookeeper; adding new node", state='i')
     keepalive_time = int(time.time())
-    transaction = zk_conn.transaction()
+    zkhander.writedata(zk_conn, {
-    transaction.create('/nodes/{}'.format(myhostname), config['daemon_mode'].encode('ascii'))
+        '/nodes/{}'.format(myhostname): config['daemon_mode'].encode('ascii'),
     # Basic state information
-    transaction.create('/nodes/{}/daemonmode'.format(myhostname), config['daemon_mode'].encode('ascii'))
+        '/nodes/{}/daemonmode'.format(myhostname): config['daemon_mode'].encode('ascii'),
-    transaction.create('/nodes/{}/daemonstate'.format(myhostname), 'init'.encode('ascii'))
+        '/nodes/{}/daemonstate'.format(myhostname): 'init'.encode('ascii'),
-    transaction.create('/nodes/{}/routerstate'.format(myhostname), 'client'.encode('ascii'))
+        '/nodes/{}/routerstate'.format(myhostname): 'client'.encode('ascii'),
-    transaction.create('/nodes/{}/domainstate'.format(myhostname), 'flushed'.encode('ascii'))
+        '/nodes/{}/domainstate'.format(myhostname): 'flushed'.encode('ascii'),
-    transaction.create('/nodes/{}/staticdata'.format(myhostname), ' '.join(staticdata).encode('ascii'))
+        '/nodes/{}/staticdata'.format(myhostname): ' '.join(staticdata).encode('ascii'),
-    transaction.create('/nodes/{}/memfree'.format(myhostname), '0'.encode('ascii'))
+        '/nodes/{}/memtotal'.format(myhostname): '0'.encode('ascii'),
-    transaction.create('/nodes/{}/memused'.format(myhostname), '0'.encode('ascii'))
+        '/nodes/{}/memfree'.format(myhostname): '0'.encode('ascii'),
-    transaction.create('/nodes/{}/memalloc'.format(myhostname), '0'.encode('ascii'))
+        '/nodes/{}/memused'.format(myhostname): '0'.encode('ascii'),
-    transaction.create('/nodes/{}/vcpualloc'.format(myhostname), '0'.encode('ascii'))
+        '/nodes/{}/memalloc'.format(myhostname): '0'.encode('ascii'),
-    transaction.create('/nodes/{}/cpuload'.format(myhostname), '0.0'.encode('ascii'))
+        '/nodes/{}/vcpualloc'.format(myhostname): '0'.encode('ascii'),
-    transaction.create('/nodes/{}/networkscount'.format(myhostname), '0'.encode('ascii'))
+        '/nodes/{}/cpuload'.format(myhostname): '0.0'.encode('ascii'),
-    transaction.create('/nodes/{}/domainscount'.format(myhostname), '0'.encode('ascii'))
+        '/nodes/{}/networkscount'.format(myhostname): '0'.encode('ascii'),
-    transaction.create('/nodes/{}/runningdomains'.format(myhostname), ''.encode('ascii'))
+        '/nodes/{}/domainscount'.format(myhostname): '0'.encode('ascii'),
+        '/nodes/{}/runningdomains'.format(myhostname): ''.encode('ascii'),
     # Keepalives and fencing information
-    transaction.create('/nodes/{}/keepalive'.format(myhostname), str(keepalive_time).encode('ascii'))
+        '/nodes/{}/keepalive'.format(myhostname): str(keepalive_time).encode('ascii'),
-    transaction.create('/nodes/{}/ipmihostname'.format(myhostname), config['ipmi_hostname'].encode('ascii'))
+        '/nodes/{}/ipmihostname'.format(myhostname): config['ipmi_hostname'].encode('ascii'),
-    transaction.create('/nodes/{}/ipmiusername'.format(myhostname), config['ipmi_username'].encode('ascii'))
+        '/nodes/{}/ipmiusername'.format(myhostname): config['ipmi_username'].encode('ascii'),
-    transaction.create('/nodes/{}/ipmipassword'.format(myhostname), config['ipmi_password'].encode('ascii'))
+        '/nodes/{}/ipmipassword'.format(myhostname): config['ipmi_password'].encode('ascii')
-    transaction.commit()
+    })
 
 # Check that the primary key exists, and create it with us as master if not
 try:
@@ -1032,6 +1033,7 @@ def update_zookeeper():
     if debug:
         print("Set our information in zookeeper")
     #this_node.name = lv_conn.getHostname()
+    this_node.memtotal = int(psutil.virtual_memory().total / 1024 / 1024)
     this_node.memused = int(psutil.virtual_memory().used / 1024 / 1024)
     this_node.memfree = int(psutil.virtual_memory().free / 1024 / 1024)
     this_node.memalloc = memalloc
@@ -1044,6 +1046,7 @@ def update_zookeeper():
     keepalive_time = int(time.time())
     try:
         zkhandler.writedata(zk_conn, {
+            '/nodes/{}/memtotal'.format(this_node.name): str(this_node.memtotal),
             '/nodes/{}/memused'.format(this_node.name): str(this_node.memused),
             '/nodes/{}/memfree'.format(this_node.name): str(this_node.memfree),
             '/nodes/{}/memalloc'.format(this_node.name): str(this_node.memalloc),

node-daemon/pvcd/NodeInstance.py

@@ -316,6 +316,17 @@ class NodeInstance(object):
 
     # Flush all VMs on the host
     def flush(self):
+        # Wait indefinitely for the flush_lock to be freed
+        time.sleep(0.5)
+        while zkhandler.readdata(self.zk_conn, '/locks/flush_lock') == 'True':
+            time.sleep(2)
+
+        # Acquire the flush lock
+        zkhandler.writedata(self.zk_conn, {
+            '/locks/flush_lock'.format(node): 'True'
+        })
+
+        # Begin flush
         self.inflush = True
         self.logger.out('Flushing node "{}" of running VMs'.format(self.name), state='i')
         self.logger.out('Domain list: {}'.format(', '.join(self.domain_list)))
@@ -347,6 +358,11 @@ class NodeInstance(object):
         zkhandler.writedata(self.zk_conn, { '/nodes/{}/domainstate'.format(self.name): 'flushed' })
         self.inflush = False
 
+        # Release the flush lock
+        zkhandler.writedata(self.zk_conn, {
+            '/locks/flush_lock'.format(node): 'False'
+        })
+
     def unflush(self):
         self.inflush = True
         self.logger.out('Restoring node {} to active service.'.format(self.name), state='i')