Compare commits
1 Commits
cf59e85eeb
...
a1c5b8aad8
| Author | SHA1 | Date | |
|---|---|---|---|
| a1c5b8aad8 |
@@ -6,7 +6,6 @@
|
||||
* [Node Daemon] Adds network traffic rate calculation subsystem.
|
||||
* [All Daemons] Fixes a printing bug where newlines were not added atomically.
|
||||
* [CLI Client] Fixes a bug listing connections if no default is specified.
|
||||
* [All Daemons] Simplifies debug logging conditionals by moving into the Logger instance itself.
|
||||
|
||||
###### [v0.9.86](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.86)
|
||||
|
||||
|
||||
@@ -115,10 +115,6 @@ class Logger(object):
|
||||
|
||||
# Output function
|
||||
def out(self, message, state=None, prefix=""):
|
||||
# Only handle d-state (debug) messages if we're in debug mode
|
||||
if state in ["d"] and not self.config["debug"]:
|
||||
return
|
||||
|
||||
# Get the date
|
||||
if self.config["log_dates"]:
|
||||
date = "{} ".format(datetime.now().strftime("%Y/%m/%d %H:%M:%S.%f"))
|
||||
|
||||
3
debian/changelog
vendored
3
debian/changelog
vendored
@@ -4,9 +4,8 @@ pvc (0.9.87-0) unstable; urgency=high
|
||||
* [Node Daemon] Adds network traffic rate calculation subsystem.
|
||||
* [All Daemons] Fixes a printing bug where newlines were not added atomically.
|
||||
* [CLI Client] Fixes a bug listing connections if no default is specified.
|
||||
* [All Daemons] Simplifies debug logging conditionals by moving into the Logger instance itself.
|
||||
|
||||
-- Joshua M. Boniface <joshua@boniface.me> Wed, 27 Dec 2023 13:03:14 -0500
|
||||
-- Joshua M. Boniface <joshua@boniface.me> Wed, 27 Dec 2023 12:42:52 -0500
|
||||
|
||||
pvc (0.9.86-0) unstable; urgency=high
|
||||
|
||||
|
||||
@@ -157,6 +157,9 @@ class MonitoringPlugin(object):
|
||||
"w": warning
|
||||
"e": error
|
||||
"""
|
||||
if state == "d" and not self.config["debug"]:
|
||||
return
|
||||
|
||||
self.logger.out(message, state=state, prefix=self.plugin_name)
|
||||
|
||||
#
|
||||
@@ -520,10 +523,11 @@ class MonitoringInstance(object):
|
||||
|
||||
entries = fault_data["entries"]()
|
||||
|
||||
self.logger.out(
|
||||
f"Entries for fault check {fault_type}: {dumps(entries)}",
|
||||
state="d",
|
||||
)
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
f"Entries for fault check {fault_type}: {dumps(entries)}",
|
||||
state="d",
|
||||
)
|
||||
|
||||
for _entry in entries:
|
||||
entry = _entry["entry"]
|
||||
|
||||
@@ -333,11 +333,12 @@ class AXFRDaemonInstance(object):
|
||||
z = dns.zone.from_xfr(axfr)
|
||||
records_raw = [z[n].to_text(n) for n in z.nodes.keys()]
|
||||
except Exception as e:
|
||||
self.logger.out(
|
||||
"{} {} ({})".format(e, dnsmasq_ip, domain),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
"{} {} ({})".format(e, dnsmasq_ip, domain),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
continue
|
||||
|
||||
# Fix the formatting because it's useless
|
||||
@@ -369,11 +370,12 @@ class AXFRDaemonInstance(object):
|
||||
"SELECT * FROM records WHERE domain_id=%s", (domain_id,)
|
||||
)
|
||||
results = list(sql_curs.fetchall())
|
||||
self.logger.out(
|
||||
"SQL query results: {}".format(results),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
"SQL query results: {}".format(results),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.out(
|
||||
"ERROR: Failed to obtain DNS records from database: {}".format(
|
||||
@@ -386,11 +388,12 @@ class AXFRDaemonInstance(object):
|
||||
records_old = list()
|
||||
records_old_ids = list()
|
||||
if not results:
|
||||
self.logger.out(
|
||||
"No results found, skipping.",
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
"No results found, skipping.",
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
continue
|
||||
for record in results:
|
||||
# Skip the non-A
|
||||
@@ -401,21 +404,23 @@ class AXFRDaemonInstance(object):
|
||||
r_data = record[4]
|
||||
# Assemble a list element in the same format as the AXFR data
|
||||
entry = "{} {} IN {} {}".format(r_name, r_ttl, r_type, r_data)
|
||||
self.logger.out(
|
||||
"Found record: {}".format(entry),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
|
||||
# Skip non-A or AAAA records
|
||||
if r_type != "A" and r_type != "AAAA":
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
'Skipping record {}, not A or AAAA: "{}"'.format(
|
||||
entry, r_type
|
||||
),
|
||||
"Found record: {}".format(entry),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
|
||||
# Skip non-A or AAAA records
|
||||
if r_type != "A" and r_type != "AAAA":
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
'Skipping record {}, not A or AAAA: "{}"'.format(
|
||||
entry, r_type
|
||||
),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
continue
|
||||
|
||||
records_old.append(entry)
|
||||
@@ -424,16 +429,17 @@ class AXFRDaemonInstance(object):
|
||||
records_new.sort()
|
||||
records_old.sort()
|
||||
|
||||
self.logger.out(
|
||||
"New: {}".format(records_new),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
self.logger.out(
|
||||
"Old: {}".format(records_old),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
"New: {}".format(records_new),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
self.logger.out(
|
||||
"Old: {}".format(records_old),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
|
||||
# Find the differences between the lists
|
||||
# Basic check one: are they completely equal
|
||||
@@ -444,16 +450,17 @@ class AXFRDaemonInstance(object):
|
||||
in_new_not_in_old = in_new - in_old
|
||||
in_old_not_in_new = in_old - in_new
|
||||
|
||||
self.logger.out(
|
||||
"New but not old: {}".format(in_new_not_in_old),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
self.logger.out(
|
||||
"Old but not new: {}".format(in_old_not_in_new),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
"New but not old: {}".format(in_new_not_in_old),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
self.logger.out(
|
||||
"Old but not new: {}".format(in_old_not_in_new),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
|
||||
# Go through the old list
|
||||
remove_records = list() # list of database IDs
|
||||
@@ -480,11 +487,12 @@ class AXFRDaemonInstance(object):
|
||||
if len(remove_records) > 0:
|
||||
# Remove the invalid old records
|
||||
for record_id in remove_records:
|
||||
self.logger.out(
|
||||
"Removing record: {}".format(record_id),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
"Removing record: {}".format(record_id),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
sql_curs.execute(
|
||||
"DELETE FROM records WHERE id=%s", (record_id,)
|
||||
)
|
||||
@@ -499,11 +507,12 @@ class AXFRDaemonInstance(object):
|
||||
r_ttl = record[1]
|
||||
r_type = record[3]
|
||||
r_data = record[4]
|
||||
self.logger.out(
|
||||
"Add record: {}".format(name),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
"Add record: {}".format(name),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
try:
|
||||
sql_curs.execute(
|
||||
"INSERT INTO records (domain_id, name, ttl, type, prio, content) VALUES (%s, %s, %s, %s, %s, %s)",
|
||||
@@ -511,21 +520,23 @@ class AXFRDaemonInstance(object):
|
||||
)
|
||||
changed = True
|
||||
except psycopg2.IntegrityError as e:
|
||||
self.logger.out(
|
||||
"Failed to add record due to {}: {}".format(
|
||||
e, name
|
||||
),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
"Failed to add record due to {}: {}".format(
|
||||
e, name
|
||||
),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
except psycopg2.errors.InFailedSqlTransaction as e:
|
||||
self.logger.out(
|
||||
"Failed to add record due to {}: {}".format(
|
||||
e, name
|
||||
),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
"Failed to add record due to {}: {}".format(
|
||||
e, name
|
||||
),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
|
||||
if changed:
|
||||
# Increase SOA serial
|
||||
@@ -537,22 +548,24 @@ class AXFRDaemonInstance(object):
|
||||
current_serial = int(soa_record[2])
|
||||
new_serial = current_serial + 1
|
||||
soa_record[2] = str(new_serial)
|
||||
self.logger.out(
|
||||
"Records changed; bumping SOA: {}".format(new_serial),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
"Records changed; bumping SOA: {}".format(new_serial),
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
sql_curs.execute(
|
||||
"UPDATE records SET content=%s WHERE domain_id=%s AND type='SOA'",
|
||||
(" ".join(soa_record), domain_id),
|
||||
)
|
||||
|
||||
# Commit all the previous changes
|
||||
self.logger.out(
|
||||
"Committing database changes and reloading PDNS",
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
if self.config["debug"]:
|
||||
self.logger.out(
|
||||
"Committing database changes and reloading PDNS",
|
||||
state="d",
|
||||
prefix="dns-aggregator",
|
||||
)
|
||||
try:
|
||||
self.sql_conn.commit()
|
||||
except Exception as e:
|
||||
|
||||
@@ -80,7 +80,9 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
pool_list = zkhandler.children("base.pool")
|
||||
osd_list = zkhandler.children("base.osd")
|
||||
|
||||
logger.out("Thread starting", state="d", prefix="ceph-thread")
|
||||
debug = config["debug"]
|
||||
if debug:
|
||||
logger.out("Thread starting", state="d", prefix="ceph-thread")
|
||||
|
||||
# Connect to the Ceph cluster
|
||||
try:
|
||||
@@ -88,7 +90,8 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
conffile=config["ceph_config_file"],
|
||||
conf=dict(keyring=config["ceph_admin_keyring"]),
|
||||
)
|
||||
logger.out("Connecting to cluster", state="d", prefix="ceph-thread")
|
||||
if debug:
|
||||
logger.out("Connecting to cluster", state="d", prefix="ceph-thread")
|
||||
ceph_conn.connect(timeout=1)
|
||||
except Exception as e:
|
||||
logger.out("Failed to open connection to Ceph cluster: {}".format(e), state="e")
|
||||
@@ -97,11 +100,12 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
# Primary-only functions
|
||||
if this_node.coordinator_state == "primary":
|
||||
# Get Ceph status information (pretty)
|
||||
logger.out(
|
||||
"Set Ceph status information in zookeeper (primary only)",
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Set Ceph status information in zookeeper (primary only)",
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
|
||||
command = {"prefix": "status", "format": "pretty"}
|
||||
ceph_status = ceph_conn.mon_command(json.dumps(command), b"", timeout=1)[
|
||||
@@ -113,11 +117,12 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
logger.out("Failed to set Ceph status data: {}".format(e), state="e")
|
||||
|
||||
# Get Ceph health information (JSON)
|
||||
logger.out(
|
||||
"Set Ceph health information in zookeeper (primary only)",
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Set Ceph health information in zookeeper (primary only)",
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
|
||||
command = {"prefix": "health", "format": "json"}
|
||||
ceph_health = ceph_conn.mon_command(json.dumps(command), b"", timeout=1)[
|
||||
@@ -129,11 +134,12 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
logger.out("Failed to set Ceph health data: {}".format(e), state="e")
|
||||
|
||||
# Get Ceph df information (pretty)
|
||||
logger.out(
|
||||
"Set Ceph rados df information in zookeeper (primary only)",
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Set Ceph rados df information in zookeeper (primary only)",
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
|
||||
# Get rados df info
|
||||
command = {"prefix": "df", "format": "pretty"}
|
||||
@@ -145,11 +151,12 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
except Exception as e:
|
||||
logger.out("Failed to set Ceph utilization data: {}".format(e), state="e")
|
||||
|
||||
logger.out(
|
||||
"Set pool information in zookeeper (primary only)",
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Set pool information in zookeeper (primary only)",
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
|
||||
# Get pool info
|
||||
command = {"prefix": "df", "format": "json"}
|
||||
@@ -172,11 +179,12 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
rados_pool_df_raw = []
|
||||
|
||||
pool_count = len(ceph_pool_df_raw)
|
||||
logger.out(
|
||||
"Getting info for {} pools".format(pool_count),
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Getting info for {} pools".format(pool_count),
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
for pool_idx in range(0, pool_count):
|
||||
try:
|
||||
# Combine all the data for this pool
|
||||
@@ -187,18 +195,22 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
|
||||
# Ignore any pools that aren't in our pool list
|
||||
if pool["name"] not in pool_list:
|
||||
logger.out(
|
||||
"Pool {} not in pool list {}".format(pool["name"], pool_list),
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Pool {} not in pool list {}".format(
|
||||
pool["name"], pool_list
|
||||
),
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
continue
|
||||
else:
|
||||
logger.out(
|
||||
"Parsing data for pool {}".format(pool["name"]),
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Parsing data for pool {}".format(pool["name"]),
|
||||
state="d",
|
||||
prefix="ceph-thread",
|
||||
)
|
||||
|
||||
# Assemble a useful data structure
|
||||
pool_df = {
|
||||
@@ -236,7 +248,8 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
osds_this_node = 0
|
||||
if len(osd_list) > 0:
|
||||
# Get data from Ceph OSDs
|
||||
logger.out("Get data from Ceph OSDs", state="d", prefix="ceph-thread")
|
||||
if debug:
|
||||
logger.out("Get data from Ceph OSDs", state="d", prefix="ceph-thread")
|
||||
|
||||
# Parse the dump data
|
||||
osd_dump = dict()
|
||||
@@ -251,7 +264,8 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
logger.out("Failed to obtain OSD data: {}".format(e), state="w")
|
||||
osd_dump_raw = []
|
||||
|
||||
logger.out("Loop through OSD dump", state="d", prefix="ceph-thread")
|
||||
if debug:
|
||||
logger.out("Loop through OSD dump", state="d", prefix="ceph-thread")
|
||||
for osd in osd_dump_raw:
|
||||
osd_dump.update(
|
||||
{
|
||||
@@ -265,7 +279,8 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
)
|
||||
|
||||
# Parse the df data
|
||||
logger.out("Parse the OSD df data", state="d", prefix="ceph-thread")
|
||||
if debug:
|
||||
logger.out("Parse the OSD df data", state="d", prefix="ceph-thread")
|
||||
|
||||
osd_df = dict()
|
||||
|
||||
@@ -278,7 +293,8 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
logger.out("Failed to obtain OSD data: {}".format(e), state="w")
|
||||
osd_df_raw = []
|
||||
|
||||
logger.out("Loop through OSD df", state="d", prefix="ceph-thread")
|
||||
if debug:
|
||||
logger.out("Loop through OSD df", state="d", prefix="ceph-thread")
|
||||
for osd in osd_df_raw:
|
||||
osd_df.update(
|
||||
{
|
||||
@@ -300,7 +316,8 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
)
|
||||
|
||||
# Parse the status data
|
||||
logger.out("Parse the OSD status data", state="d", prefix="ceph-thread")
|
||||
if debug:
|
||||
logger.out("Parse the OSD status data", state="d", prefix="ceph-thread")
|
||||
|
||||
osd_status = dict()
|
||||
|
||||
@@ -313,7 +330,8 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
logger.out("Failed to obtain OSD status data: {}".format(e), state="w")
|
||||
osd_status_raw = []
|
||||
|
||||
logger.out("Loop through OSD status data", state="d", prefix="ceph-thread")
|
||||
if debug:
|
||||
logger.out("Loop through OSD status data", state="d", prefix="ceph-thread")
|
||||
|
||||
for line in osd_status_raw.split("\n"):
|
||||
# Strip off colour
|
||||
@@ -382,7 +400,8 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
)
|
||||
|
||||
# Merge them together into a single meaningful dict
|
||||
logger.out("Merge OSD data together", state="d", prefix="ceph-thread")
|
||||
if debug:
|
||||
logger.out("Merge OSD data together", state="d", prefix="ceph-thread")
|
||||
|
||||
osd_stats = dict()
|
||||
|
||||
@@ -402,7 +421,10 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
|
||||
# Upload OSD data for the cluster (primary-only)
|
||||
if this_node.coordinator_state == "primary":
|
||||
logger.out("Trigger updates for each OSD", state="d", prefix="ceph-thread")
|
||||
if debug:
|
||||
logger.out(
|
||||
"Trigger updates for each OSD", state="d", prefix="ceph-thread"
|
||||
)
|
||||
|
||||
for osd in osd_list:
|
||||
try:
|
||||
@@ -419,16 +441,20 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
|
||||
|
||||
queue.put(osds_this_node)
|
||||
|
||||
logger.out("Thread finished", state="d", prefix="ceph-thread")
|
||||
if debug:
|
||||
logger.out("Thread finished", state="d", prefix="ceph-thread")
|
||||
|
||||
|
||||
# VM stats update function
|
||||
def collect_vm_stats(logger, config, zkhandler, this_node, queue):
|
||||
logger.out("Thread starting", state="d", prefix="vm-thread")
|
||||
debug = config["debug"]
|
||||
if debug:
|
||||
logger.out("Thread starting", state="d", prefix="vm-thread")
|
||||
|
||||
# Connect to libvirt
|
||||
libvirt_name = "qemu:///system"
|
||||
logger.out("Connecting to libvirt", state="d", prefix="vm-thread")
|
||||
if debug:
|
||||
logger.out("Connecting to libvirt", state="d", prefix="vm-thread")
|
||||
try:
|
||||
lv_conn = libvirt.open(libvirt_name)
|
||||
if lv_conn is None:
|
||||
@@ -441,11 +467,12 @@ def collect_vm_stats(logger, config, zkhandler, this_node, queue):
|
||||
memprov = 0
|
||||
vcpualloc = 0
|
||||
# Toggle state management of dead VMs to restart them
|
||||
logger.out(
|
||||
"Toggle state management of dead VMs to restart them",
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Toggle state management of dead VMs to restart them",
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
# Make a copy of the d_domain; if not, and it changes in flight, this can fail
|
||||
fixed_d_domain = this_node.d_domain.copy()
|
||||
for domain, instance in fixed_d_domain.items():
|
||||
@@ -491,11 +518,12 @@ def collect_vm_stats(logger, config, zkhandler, this_node, queue):
|
||||
domain_name = domain.name()
|
||||
|
||||
# Get all the raw information about the VM
|
||||
logger.out(
|
||||
"Getting general statistics for VM {}".format(domain_name),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Getting general statistics for VM {}".format(domain_name),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
(
|
||||
domain_state,
|
||||
domain_maxmem,
|
||||
@@ -509,25 +537,29 @@ def collect_vm_stats(logger, config, zkhandler, this_node, queue):
|
||||
domain_memory_stats = domain.memoryStats()
|
||||
domain_cpu_stats = domain.getCPUStats(True)[0]
|
||||
except Exception as e:
|
||||
try:
|
||||
logger.out(
|
||||
"Failed getting VM information for {}: {}".format(domain.name(), e),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
if debug:
|
||||
try:
|
||||
logger.out(
|
||||
"Failed getting VM information for {}: {}".format(
|
||||
domain.name(), e
|
||||
),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
continue
|
||||
|
||||
# Ensure VM is present in the domain_list
|
||||
if domain_uuid not in this_node.domain_list:
|
||||
this_node.domain_list.append(domain_uuid)
|
||||
|
||||
logger.out(
|
||||
"Getting disk statistics for VM {}".format(domain_name),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Getting disk statistics for VM {}".format(domain_name),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
domain_disk_stats = []
|
||||
try:
|
||||
for disk in tree.findall("devices/disk"):
|
||||
@@ -546,21 +578,23 @@ def collect_vm_stats(logger, config, zkhandler, this_node, queue):
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
try:
|
||||
logger.out(
|
||||
"Failed getting disk stats for {}: {}".format(domain.name(), e),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
if debug:
|
||||
try:
|
||||
logger.out(
|
||||
"Failed getting disk stats for {}: {}".format(domain.name(), e),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
continue
|
||||
|
||||
logger.out(
|
||||
"Getting network statistics for VM {}".format(domain_name),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Getting network statistics for VM {}".format(domain_name),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
domain_network_stats = []
|
||||
try:
|
||||
for interface in tree.findall("devices/interface"):
|
||||
@@ -585,14 +619,17 @@ def collect_vm_stats(logger, config, zkhandler, this_node, queue):
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
try:
|
||||
logger.out(
|
||||
"Failed getting network stats for {}: {}".format(domain.name(), e),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
if debug:
|
||||
try:
|
||||
logger.out(
|
||||
"Failed getting network stats for {}: {}".format(
|
||||
domain.name(), e
|
||||
),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
continue
|
||||
|
||||
# Create the final dictionary
|
||||
@@ -608,42 +645,48 @@ def collect_vm_stats(logger, config, zkhandler, this_node, queue):
|
||||
"net_stats": domain_network_stats,
|
||||
}
|
||||
|
||||
logger.out(
|
||||
"Writing statistics for VM {} to Zookeeper".format(domain_name),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Writing statistics for VM {} to Zookeeper".format(domain_name),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
|
||||
try:
|
||||
zkhandler.write(
|
||||
[(("domain.stats", domain_uuid), str(json.dumps(domain_stats)))]
|
||||
)
|
||||
except Exception as e:
|
||||
logger.out(
|
||||
"Failed to write domain statistics: {}".format(e),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Failed to write domain statistics: {}".format(e),
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
|
||||
# Close the Libvirt connection
|
||||
lv_conn.close()
|
||||
|
||||
logger.out(
|
||||
f"VM stats: doms: {len(running_domains)}; memalloc: {memalloc}; memprov: {memprov}; vcpualloc: {vcpualloc}",
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
f"VM stats: doms: {len(running_domains)}; memalloc: {memalloc}; memprov: {memprov}; vcpualloc: {vcpualloc}",
|
||||
state="d",
|
||||
prefix="vm-thread",
|
||||
)
|
||||
|
||||
queue.put(len(running_domains))
|
||||
queue.put(memalloc)
|
||||
queue.put(memprov)
|
||||
queue.put(vcpualloc)
|
||||
|
||||
logger.out("Thread finished", state="d", prefix="vm-thread")
|
||||
if debug:
|
||||
logger.out("Thread finished", state="d", prefix="vm-thread")
|
||||
|
||||
|
||||
# Keepalive update function
|
||||
def node_keepalive(logger, config, zkhandler, this_node, netstats):
|
||||
debug = config["debug"]
|
||||
|
||||
# Display node information to the terminal
|
||||
if config["log_keepalives"]:
|
||||
if this_node.coordinator_state == "primary":
|
||||
@@ -703,7 +746,10 @@ def node_keepalive(logger, config, zkhandler, this_node, netstats):
|
||||
)
|
||||
|
||||
# Get past state and update if needed
|
||||
logger.out("Get past state and update if needed", state="d", prefix="main-thread")
|
||||
if debug:
|
||||
logger.out(
|
||||
"Get past state and update if needed", state="d", prefix="main-thread"
|
||||
)
|
||||
|
||||
past_state = zkhandler.read(("node.state.daemon", this_node.name))
|
||||
if past_state != "run" and past_state != "shutdown":
|
||||
@@ -713,9 +759,10 @@ def node_keepalive(logger, config, zkhandler, this_node, netstats):
|
||||
this_node.daemon_state = "run"
|
||||
|
||||
# Ensure the primary key is properly set
|
||||
logger.out(
|
||||
"Ensure the primary key is properly set", state="d", prefix="main-thread"
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Ensure the primary key is properly set", state="d", prefix="main-thread"
|
||||
)
|
||||
if this_node.coordinator_state == "primary":
|
||||
if zkhandler.read("base.config.primary_node") != this_node.name:
|
||||
zkhandler.write([("base.config.primary_node", this_node.name)])
|
||||
@@ -796,7 +843,8 @@ def node_keepalive(logger, config, zkhandler, this_node, netstats):
|
||||
|
||||
# Set our information in zookeeper
|
||||
keepalive_time = int(time.time())
|
||||
logger.out("Set our information in zookeeper", state="d", prefix="main-thread")
|
||||
if debug:
|
||||
logger.out("Set our information in zookeeper", state="d", prefix="main-thread")
|
||||
try:
|
||||
zkhandler.write(
|
||||
[
|
||||
@@ -884,9 +932,10 @@ def node_keepalive(logger, config, zkhandler, this_node, netstats):
|
||||
|
||||
# Look for dead nodes and fence them
|
||||
if not this_node.maintenance:
|
||||
logger.out(
|
||||
"Look for dead nodes and fence them", state="d", prefix="main-thread"
|
||||
)
|
||||
if debug:
|
||||
logger.out(
|
||||
"Look for dead nodes and fence them", state="d", prefix="main-thread"
|
||||
)
|
||||
if config["daemon_mode"] == "coordinator":
|
||||
for node_name in zkhandler.children("base.node"):
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user