parallelvirtualcluster
/
pvc
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 111 Wiki Activity

Move Ceph statistics gathering into thread

This commit is contained in:
Joshua Boniface 2020-06-06 13:23:24 -04:00
parent cebb4bbc1a
commit 2ad6860dfe
1 changed files with 245 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
node-daemon/pvcnoded/Daemon.py
View File

@ -44,6 +44,8 @@ import apscheduler.schedulers.background
from distutils.util import strtobool
from queue import Queue
import pvcnoded.log as log
import pvcnoded.zkhandler as zkhandler
import pvcnoded.fencing as fencing
@ -1011,35 +1013,8 @@ if enable_storage:
# PHASE 9 - Run the daemon
###############################################################################
# Keepalive update function
def node_keepalive():
# Set the upstream IP in Zookeeper for clients to read
if config['enable_networking']:
if this_node.router_state == 'primary':
try:
if zkhandler.readdata(zk_conn, '/upstream_ip') != config['upstream_floating_ip']:
raise
except:
zkhandler.writedata(zk_conn, {'/upstream_ip': config['upstream_floating_ip']})
# Get past state and update if needed
if debug:
print("Get past state and update if needed")
past_state = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(this_node.name))
if past_state != 'run':
this_node.daemon_state = 'run'
zkhandler.writedata(zk_conn, { '/nodes/{}/daemonstate'.format(this_node.name): 'run' })
else:
this_node.daemon_state = 'run'
# Ensure the primary key is properly set
if debug:
print("Ensure the primary key is properly set")
if this_node.router_state == 'primary':
if zkhandler.readdata(zk_conn, '/primary_node') != this_node.name:
zkhandler.writedata(zk_conn, {'/primary_node': this_node.name})
if enable_storage:
# Ceph stats update function
def collect_ceph_stats(queue):
# Get Ceph cluster health (for local printing)
if debug:
print("Get Ceph cluster health (for local printing)")
@ -1264,6 +1239,44 @@ def node_keepalive():
logger.out('Failed to upload OSD stats from dictionary: {}'.format(e), state='w')
osds_this_node += 1
queue.put(ceph_health_colour)
queue.put(ceph_health)
queue.put(osds_this_node)
# Keepalive update function
def node_keepalive():
# Set the upstream IP in Zookeeper for clients to read
if config['enable_networking']:
if this_node.router_state == 'primary':
try:
if zkhandler.readdata(zk_conn, '/upstream_ip') != config['upstream_floating_ip']:
raise
except:
zkhandler.writedata(zk_conn, {'/upstream_ip': config['upstream_floating_ip']})
# Get past state and update if needed
if debug:
print("Get past state and update if needed")
past_state = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(this_node.name))
if past_state != 'run':
this_node.daemon_state = 'run'
zkhandler.writedata(zk_conn, { '/nodes/{}/daemonstate'.format(this_node.name): 'run' })
else:
this_node.daemon_state = 'run'
# Ensure the primary key is properly set
if debug:
print("Ensure the primary key is properly set")
if this_node.router_state == 'primary':
if zkhandler.readdata(zk_conn, '/primary_node') != this_node.name:
zkhandler.writedata(zk_conn, {'/primary_node': this_node.name})
# Run Ceph status collection in separate thread for parallelization
if enable_storage:
ceph_thread_queue = Queue()
ceph_stats_thread = threading.Thread(target=collect_ceph_stats, args=(ceph_thread_queue,), kwargs={})
ceph_stats_thread.start()
# Normalize running VM status
memalloc = 0
vcpualloc = 0
@ -1318,6 +1331,14 @@ def node_keepalive():
else:
this_node.domains_count = 0
# Wait for Ceph thread completion
if enable_storage:
ceph_stats_thread.join()
ceph_health_colour = ceph_thread_queue.get()
ceph_health = ceph_thread_queue.get()
osds_this_node = ceph_thread_queue.get()
# Set our information in zookeeper
keepalive_time = int(time.time())
if debug: