From 0e5c681adac84d39195fb9cf4616bcb12c2ae01c Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Tue, 11 Aug 2020 11:46:41 -0400 Subject: [PATCH] Clean up imports Make several imports more specific to reduce redundant code imports and improve memory utilization. --- node-daemon/pvcnoded/CephInstance.py | 1 - node-daemon/pvcnoded/DNSAggregatorInstance.py | 9 +++---- node-daemon/pvcnoded/Daemon.py | 25 ++++++++----------- node-daemon/pvcnoded/MetadataAPIInstance.py | 6 ++--- node-daemon/pvcnoded/NodeInstance.py | 16 +++++------- .../pvcnoded/VMConsoleWatcherInstance.py | 10 +++----- node-daemon/pvcnoded/VMInstance.py | 9 +++---- node-daemon/pvcnoded/VXNetworkInstance.py | 2 +- node-daemon/pvcnoded/common.py | 12 ++++----- node-daemon/pvcnoded/fencing.py | 1 - node-daemon/pvcnoded/zkhandler.py | 1 - 11 files changed, 37 insertions(+), 55 deletions(-) diff --git a/node-daemon/pvcnoded/CephInstance.py b/node-daemon/pvcnoded/CephInstance.py index 8cdf7e8c..5580c013 100644 --- a/node-daemon/pvcnoded/CephInstance.py +++ b/node-daemon/pvcnoded/CephInstance.py @@ -21,7 +21,6 @@ ############################################################################### import time -import ast import json import psutil diff --git a/node-daemon/pvcnoded/DNSAggregatorInstance.py b/node-daemon/pvcnoded/DNSAggregatorInstance.py index a3b4bfab..94a78710 100644 --- a/node-daemon/pvcnoded/DNSAggregatorInstance.py +++ b/node-daemon/pvcnoded/DNSAggregatorInstance.py @@ -20,14 +20,13 @@ # ############################################################################### -import os -import sys import time -import threading import dns.zone import dns.query import psycopg2 +from threading import Thread, Event + import pvcnoded.log as log import pvcnoded.zkhandler as zkhandler import pvcnoded.common as common @@ -292,7 +291,7 @@ class AXFRDaemonInstance(object): self.config = self.aggregator.config self.logger = self.aggregator.logger self.dns_networks = self.aggregator.dns_networks - self.thread_stopper = threading.Event() + self.thread_stopper = Event() self.thread = None self.sql_conn = None @@ -302,7 +301,7 @@ class AXFRDaemonInstance(object): def start(self): # Create the thread self.thread_stopper.clear() - self.thread = threading.Thread(target=self.run, args=(), kwargs={}) + self.thread = Thread(target=self.run, args=(), kwargs={}) # Start a local instance of the SQL connection # Trying to use the instance from the main DNS Aggregator can result in connection failures diff --git a/node-daemon/pvcnoded/Daemon.py b/node-daemon/pvcnoded/Daemon.py index 2c10be31..d4e63eeb 100644 --- a/node-daemon/pvcnoded/Daemon.py +++ b/node-daemon/pvcnoded/Daemon.py @@ -28,20 +28,17 @@ import libvirt import sys import os import signal -import atexit -import socket import psutil import subprocess -import uuid import time import re -import configparser -import threading import yaml import json -import ipaddress -import apscheduler.schedulers.background +from socket import gethostname +from threading import Thread +from ipaddress import ip_address, ip_network +from apscheduler.schedulers.background import BackgroundScheduler from distutils.util import strtobool from queue import Queue from xml.etree import ElementTree @@ -80,7 +77,7 @@ import pvcnoded.MetadataAPIInstance as MetadataAPIInstance # Create timer to update this node in Zookeeper def startKeepaliveTimer(): # Create our timer object - update_timer = apscheduler.schedulers.background.BackgroundScheduler() + update_timer = BackgroundScheduler() interval = int(config['keepalive_interval']) logger.out('Starting keepalive timer ({} second interval)'.format(interval), state='s') update_timer.add_job(node_keepalive, 'interval', seconds=interval) @@ -108,7 +105,7 @@ except: exit(1) # Set local hostname and domain variables -myfqdn = socket.gethostname() +myfqdn = gethostname() #myfqdn = 'pvc-hv1.domain.net' myhostname = myfqdn.split('.', 1)[0] mydomainname = ''.join(myfqdn.split('.', 1)[1:]) @@ -234,7 +231,7 @@ def readConfig(pvcnoded_config_file, myhostname): # Verify the network provided is valid try: - network = ipaddress.ip_network(config[network_key]) + network = ip_network(config[network_key]) except Exception as e: print('ERROR: Network address {} for {} is not valid!'.format(config[network_key], network_key)) exit(1) @@ -251,7 +248,7 @@ def readConfig(pvcnoded_config_file, myhostname): try: # Set the ipaddr - floating_addr = ipaddress.ip_address(config[floating_key].split('/')[0]) + floating_addr = ip_address(config[floating_key].split('/')[0]) # Verify we're in the network if not floating_addr in list(network.hosts()): raise @@ -1446,13 +1443,13 @@ def node_keepalive(): # Run VM statistics collection in separate thread for parallelization if enable_hypervisor: vm_thread_queue = Queue() - vm_stats_thread = threading.Thread(target=collect_vm_stats, args=(vm_thread_queue,), kwargs={}) + vm_stats_thread = Thread(target=collect_vm_stats, args=(vm_thread_queue,), kwargs={}) vm_stats_thread.start() # Run Ceph status collection in separate thread for parallelization if enable_storage: ceph_thread_queue = Queue() - ceph_stats_thread = threading.Thread(target=collect_ceph_stats, args=(ceph_thread_queue,), kwargs={}) + ceph_stats_thread = Thread(target=collect_ceph_stats, args=(ceph_thread_queue,), kwargs={}) ceph_stats_thread.start() # Get node performance statistics @@ -1597,7 +1594,7 @@ def node_keepalive(): # Ensures that, if we lost the lock race and come out of waiting, # we won't try to trigger our own fence thread. if zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(node_name)) != 'dead': - fence_thread = threading.Thread(target=fencing.fenceNode, args=(node_name, zk_conn, config, logger), kwargs={}) + fence_thread = Thread(target=fencing.fenceNode, args=(node_name, zk_conn, config, logger), kwargs={}) fence_thread.start() # Write the updated data after we start the fence thread zkhandler.writedata(zk_conn, { '/nodes/{}/daemonstate'.format(node_name): 'dead' }) diff --git a/node-daemon/pvcnoded/MetadataAPIInstance.py b/node-daemon/pvcnoded/MetadataAPIInstance.py index c51c0be8..83d20e48 100644 --- a/node-daemon/pvcnoded/MetadataAPIInstance.py +++ b/node-daemon/pvcnoded/MetadataAPIInstance.py @@ -22,13 +22,13 @@ import gevent.pywsgi import flask -import threading import sys import time import psycopg2 + +from threading import Thread from psycopg2.extras import RealDictCursor -# The metadata server requires client libraries import daemon_lib.vm as pvc_vm import daemon_lib.network as pvc_network @@ -105,7 +105,7 @@ class MetadataAPIInstance(object): def start(self): # Launch Metadata API self.logger.out('Starting Metadata API at 169.254.169.254:80', state='i') - self.thread = threading.Thread(target=self.launch_wsgi) + self.thread = Thread(target=self.launch_wsgi) self.thread.start() self.logger.out('Successfully started Metadata API thread', state='o') diff --git a/node-daemon/pvcnoded/NodeInstance.py b/node-daemon/pvcnoded/NodeInstance.py index 046a89ef..e5efc1db 100644 --- a/node-daemon/pvcnoded/NodeInstance.py +++ b/node-daemon/pvcnoded/NodeInstance.py @@ -20,13 +20,9 @@ # ############################################################################### -import os -import sys -import psutil -import socket import time -import libvirt -import threading + +from threading import Thread import pvcnoded.log as log import pvcnoded.zkhandler as zkhandler @@ -119,13 +115,13 @@ class NodeInstance(object): if self.config['enable_networking']: if self.router_state == 'takeover': self.logger.out('Setting node {} to primary state'.format(self.name), state='i') - transition_thread = threading.Thread(target=self.become_primary, args=(), kwargs={}) + transition_thread = Thread(target=self.become_primary, args=(), kwargs={}) transition_thread.start() if self.router_state == 'relinquish': # Skip becoming secondary unless already running if self.daemon_state == 'run' or self.daemon_state == 'shutdown': self.logger.out('Setting node {} to secondary state'.format(self.name), state='i') - transition_thread = threading.Thread(target=self.become_secondary, args=(), kwargs={}) + transition_thread = Thread(target=self.become_secondary, args=(), kwargs={}) transition_thread.start() else: # We did nothing, so just become secondary state @@ -157,11 +153,11 @@ class NodeInstance(object): # Do flushing in a thread so it doesn't block the migrates out if self.domain_state == 'flush': - self.flush_thread = threading.Thread(target=self.flush, args=(), kwargs={}) + self.flush_thread = Thread(target=self.flush, args=(), kwargs={}) self.flush_thread.start() # Do unflushing in a thread so it doesn't block the migrates in if self.domain_state == 'unflush': - self.flush_thread = threading.Thread(target=self.unflush, args=(), kwargs={}) + self.flush_thread = Thread(target=self.unflush, args=(), kwargs={}) self.flush_thread.start() @self.zk_conn.DataWatch('/nodes/{}/memfree'.format(self.name)) diff --git a/node-daemon/pvcnoded/VMConsoleWatcherInstance.py b/node-daemon/pvcnoded/VMConsoleWatcherInstance.py index 9d1150b2..8105957d 100644 --- a/node-daemon/pvcnoded/VMConsoleWatcherInstance.py +++ b/node-daemon/pvcnoded/VMConsoleWatcherInstance.py @@ -21,17 +21,13 @@ ############################################################################### import os -import sys import uuid import time -import threading import libvirt +from threading import Thread, Event from collections import deque -import fcntl -import signal - import pvcnoded.log as log import pvcnoded.zkhandler as zkhandler @@ -62,12 +58,12 @@ class VMConsoleWatcherInstance(object): # Thread options self.thread = None - self.thread_stopper = threading.Event() + self.thread_stopper = Event() # Start execution thread def start(self): self.thread_stopper.clear() - self.thread = threading.Thread(target=self.run, args=(), kwargs={}) + self.thread = Thread(target=self.run, args=(), kwargs={}) self.logger.out('Starting VM log parser', state='i', prefix='Domain {}:'.format(self.domuuid)) self.thread.start() diff --git a/node-daemon/pvcnoded/VMInstance.py b/node-daemon/pvcnoded/VMInstance.py index 8a900101..81d4584b 100644 --- a/node-daemon/pvcnoded/VMInstance.py +++ b/node-daemon/pvcnoded/VMInstance.py @@ -20,16 +20,13 @@ # ############################################################################### -import os -import sys import uuid -import socket import time -import threading import libvirt -import kazoo.client import json +from threading import Thread + import pvcnoded.log as log import pvcnoded.zkhandler as zkhandler import pvcnoded.common as common @@ -138,7 +135,7 @@ class VMInstance(object): # Perform a management command self.logger.out('Updating state of VM {}'.format(self.domuuid), state='i') - state_thread = threading.Thread(target=self.manage_vm_state, args=(), kwargs={}) + state_thread = Thread(target=self.manage_vm_state, args=(), kwargs={}) state_thread.start() # Get data functions diff --git a/node-daemon/pvcnoded/VXNetworkInstance.py b/node-daemon/pvcnoded/VXNetworkInstance.py index 9553340a..9ebbde3a 100644 --- a/node-daemon/pvcnoded/VXNetworkInstance.py +++ b/node-daemon/pvcnoded/VXNetworkInstance.py @@ -21,8 +21,8 @@ ############################################################################### import os -import sys import time + from textwrap import dedent import pvcnoded.log as log diff --git a/node-daemon/pvcnoded/common.py b/node-daemon/pvcnoded/common.py index 8fccf951..0047d287 100644 --- a/node-daemon/pvcnoded/common.py +++ b/node-daemon/pvcnoded/common.py @@ -21,18 +21,18 @@ ############################################################################### import subprocess -import threading import signal -import os import time -import shlex + +from threading import Thread +from shlex import split as shlex_split import pvcnoded.log as log import pvcnoded.zkhandler as zkhandler class OSDaemon(object): def __init__(self, command_string, environment, logfile): - command = shlex.split(command_string) + command = shlex_split(command_string) # Set stdout to be a logfile if set if logfile: stdout = open(logfile, 'a') @@ -63,7 +63,7 @@ def run_os_daemon(command_string, environment=None, logfile=None): # Run a oneshot command, optionally without blocking def run_os_command(command_string, background=False, environment=None, timeout=None): - command = shlex.split(command_string) + command = shlex_split(command_string) if background: def runcmd(): try: @@ -76,7 +76,7 @@ def run_os_command(command_string, background=False, environment=None, timeout=N ) except subprocess.TimeoutExpired: pass - thread = threading.Thread(target=runcmd, args=()) + thread = Thread(target=runcmd, args=()) thread.start() return 0, None, None else: diff --git a/node-daemon/pvcnoded/fencing.py b/node-daemon/pvcnoded/fencing.py index 0a26b613..c333895e 100644 --- a/node-daemon/pvcnoded/fencing.py +++ b/node-daemon/pvcnoded/fencing.py @@ -21,7 +21,6 @@ ############################################################################### import time -import threading import pvcnoded.zkhandler as zkhandler import pvcnoded.common as common diff --git a/node-daemon/pvcnoded/zkhandler.py b/node-daemon/pvcnoded/zkhandler.py index 138c9e15..199ff5eb 100644 --- a/node-daemon/pvcnoded/zkhandler.py +++ b/node-daemon/pvcnoded/zkhandler.py @@ -20,7 +20,6 @@ # ############################################################################### -import kazoo.client import uuid # Child list function