Move Ceph statistics gathering into thread

node-daemon/pvcnoded/Daemon.py

@@ -44,6 +44,8 @@ import apscheduler.schedulers.background
 
 from distutils.util import strtobool
 
+from queue import Queue
+
 import pvcnoded.log as log
 import pvcnoded.zkhandler as zkhandler
 import pvcnoded.fencing as fencing
@@ -1011,35 +1013,8 @@ if enable_storage:
 # PHASE 9 - Run the daemon
 ###############################################################################
 
-# Keepalive update function
+# Ceph stats update function
-def node_keepalive():
+def collect_ceph_stats(queue):
-    # Set the upstream IP in Zookeeper for clients to read
-    if config['enable_networking']:
-        if this_node.router_state == 'primary':
-            try:
-                if zkhandler.readdata(zk_conn, '/upstream_ip') != config['upstream_floating_ip']:
-                    raise
-            except:
-                zkhandler.writedata(zk_conn, {'/upstream_ip': config['upstream_floating_ip']})
-
-    # Get past state and update if needed
-    if debug:
-        print("Get past state and update if needed")
-    past_state = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(this_node.name))
-    if past_state != 'run':
-        this_node.daemon_state = 'run'
-        zkhandler.writedata(zk_conn, { '/nodes/{}/daemonstate'.format(this_node.name): 'run' })
-    else:
-        this_node.daemon_state = 'run'
-
-    # Ensure the primary key is properly set
-    if debug:
-        print("Ensure the primary key is properly set")
-    if this_node.router_state == 'primary':
-        if zkhandler.readdata(zk_conn, '/primary_node') != this_node.name:
-            zkhandler.writedata(zk_conn, {'/primary_node': this_node.name})
-
-    if enable_storage:
     # Get Ceph cluster health (for local printing)
     if debug:
         print("Get Ceph cluster health (for local printing)")
@@ -1264,6 +1239,44 @@ def node_keepalive():
                     logger.out('Failed to upload OSD stats from dictionary: {}'.format(e), state='w')
                 osds_this_node += 1
 
+    queue.put(ceph_health_colour)
+    queue.put(ceph_health)
+    queue.put(osds_this_node)
+
+# Keepalive update function
+def node_keepalive():
+    # Set the upstream IP in Zookeeper for clients to read
+    if config['enable_networking']:
+        if this_node.router_state == 'primary':
+            try:
+                if zkhandler.readdata(zk_conn, '/upstream_ip') != config['upstream_floating_ip']:
+                    raise
+            except:
+                zkhandler.writedata(zk_conn, {'/upstream_ip': config['upstream_floating_ip']})
+
+    # Get past state and update if needed
+    if debug:
+        print("Get past state and update if needed")
+    past_state = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(this_node.name))
+    if past_state != 'run':
+        this_node.daemon_state = 'run'
+        zkhandler.writedata(zk_conn, { '/nodes/{}/daemonstate'.format(this_node.name): 'run' })
+    else:
+        this_node.daemon_state = 'run'
+
+    # Ensure the primary key is properly set
+    if debug:
+        print("Ensure the primary key is properly set")
+    if this_node.router_state == 'primary':
+        if zkhandler.readdata(zk_conn, '/primary_node') != this_node.name:
+            zkhandler.writedata(zk_conn, {'/primary_node': this_node.name})
+
+    # Run Ceph status collection in separate thread for parallelization
+    if enable_storage:
+        ceph_thread_queue = Queue()
+        ceph_stats_thread = threading.Thread(target=collect_ceph_stats, args=(ceph_thread_queue,), kwargs={})
+        ceph_stats_thread.start()
+    
     # Normalize running VM status
     memalloc = 0
     vcpualloc = 0
@@ -1318,6 +1331,14 @@ def node_keepalive():
     else:
         this_node.domains_count = 0
 
+    # Wait for Ceph thread completion
+    if enable_storage:
+        ceph_stats_thread.join()
+
+        ceph_health_colour = ceph_thread_queue.get()
+        ceph_health = ceph_thread_queue.get()
+        osds_this_node = ceph_thread_queue.get()
+
     # Set our information in zookeeper
     keepalive_time = int(time.time())
     if debug: