Clean up imports

Make several imports more specific to reduce redundant code imports and
improve memory utilization.

node-daemon/pvcnoded/CephInstance.py

@@ -21,7 +21,6 @@
 ###############################################################################
 
 import time
-import ast
 import json
 import psutil
 

node-daemon/pvcnoded/DNSAggregatorInstance.py

@@ -20,14 +20,13 @@
 #
 ###############################################################################
 
-import os
-import sys
 import time
-import threading
 import dns.zone
 import dns.query
 import psycopg2
 
+from threading import Thread, Event
+
 import pvcnoded.log as log
 import pvcnoded.zkhandler as zkhandler
 import pvcnoded.common as common
@@ -292,7 +291,7 @@ class AXFRDaemonInstance(object):
         self.config = self.aggregator.config
         self.logger = self.aggregator.logger
         self.dns_networks = self.aggregator.dns_networks
-        self.thread_stopper = threading.Event()
+        self.thread_stopper = Event()
         self.thread = None
         self.sql_conn = None
 
@@ -302,7 +301,7 @@ class AXFRDaemonInstance(object):
     def start(self):
         # Create the thread
         self.thread_stopper.clear()
-        self.thread = threading.Thread(target=self.run, args=(), kwargs={})
+        self.thread = Thread(target=self.run, args=(), kwargs={})
 
         # Start a local instance of the SQL connection
         # Trying to use the instance from the main DNS Aggregator can result in connection failures

node-daemon/pvcnoded/Daemon.py

@@ -28,20 +28,17 @@ import libvirt
 import sys
 import os
 import signal
-import atexit
-import socket
 import psutil
 import subprocess
-import uuid
 import time
 import re
-import configparser
-import threading
 import yaml
 import json
-import ipaddress
-import apscheduler.schedulers.background
 
+from socket import gethostname
+from threading import Thread
+from ipaddress import ip_address, ip_network
+from apscheduler.schedulers.background import BackgroundScheduler
 from distutils.util import strtobool
 from queue import Queue
 from xml.etree import ElementTree
@@ -80,7 +77,7 @@ import pvcnoded.MetadataAPIInstance as MetadataAPIInstance
 # Create timer to update this node in Zookeeper
 def startKeepaliveTimer():
     # Create our timer object
-    update_timer = apscheduler.schedulers.background.BackgroundScheduler()
+    update_timer = BackgroundScheduler()
     interval = int(config['keepalive_interval'])
     logger.out('Starting keepalive timer ({} second interval)'.format(interval), state='s')
     update_timer.add_job(node_keepalive, 'interval', seconds=interval)
@@ -108,7 +105,7 @@ except:
     exit(1)
 
 # Set local hostname and domain variables
-myfqdn = socket.gethostname()
+myfqdn = gethostname()
 #myfqdn = 'pvc-hv1.domain.net'
 myhostname = myfqdn.split('.', 1)[0]
 mydomainname = ''.join(myfqdn.split('.', 1)[1:])
@@ -234,7 +231,7 @@ def readConfig(pvcnoded_config_file, myhostname):
 
             # Verify the network provided is valid
             try:
-                network = ipaddress.ip_network(config[network_key])
+                network = ip_network(config[network_key])
             except Exception as e:
                 print('ERROR: Network address {} for {} is not valid!'.format(config[network_key], network_key))
                 exit(1)
@@ -251,7 +248,7 @@ def readConfig(pvcnoded_config_file, myhostname):
 
             try:
                 # Set the ipaddr
-                floating_addr = ipaddress.ip_address(config[floating_key].split('/')[0])
+                floating_addr = ip_address(config[floating_key].split('/')[0])
                 # Verify we're in the network
                 if not floating_addr in list(network.hosts()):
                     raise
@@ -1446,13 +1443,13 @@ def node_keepalive():
     # Run VM statistics collection in separate thread for parallelization
     if enable_hypervisor:
         vm_thread_queue = Queue()
-        vm_stats_thread = threading.Thread(target=collect_vm_stats, args=(vm_thread_queue,), kwargs={})
+        vm_stats_thread = Thread(target=collect_vm_stats, args=(vm_thread_queue,), kwargs={})
         vm_stats_thread.start()
     
     # Run Ceph status collection in separate thread for parallelization
     if enable_storage:
         ceph_thread_queue = Queue()
-        ceph_stats_thread = threading.Thread(target=collect_ceph_stats, args=(ceph_thread_queue,), kwargs={})
+        ceph_stats_thread = Thread(target=collect_ceph_stats, args=(ceph_thread_queue,), kwargs={})
         ceph_stats_thread.start()
     
     # Get node performance statistics
@@ -1597,7 +1594,7 @@ def node_keepalive():
                         # Ensures that, if we lost the lock race and come out of waiting,
                         # we won't try to trigger our own fence thread.
                         if zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(node_name)) != 'dead':
-                            fence_thread = threading.Thread(target=fencing.fenceNode, args=(node_name, zk_conn, config, logger), kwargs={})
+                            fence_thread = Thread(target=fencing.fenceNode, args=(node_name, zk_conn, config, logger), kwargs={})
                             fence_thread.start()
                             # Write the updated data after we start the fence thread
                             zkhandler.writedata(zk_conn, { '/nodes/{}/daemonstate'.format(node_name): 'dead' })

node-daemon/pvcnoded/MetadataAPIInstance.py

@@ -22,13 +22,13 @@
 
 import gevent.pywsgi
 import flask
-import threading
 import sys
 import time
 import psycopg2
+
+from threading import Thread
 from psycopg2.extras import RealDictCursor
 
-# The metadata server requires client libraries
 import daemon_lib.vm as pvc_vm
 import daemon_lib.network as pvc_network
 
@@ -105,7 +105,7 @@ class MetadataAPIInstance(object):
     def start(self):
         # Launch Metadata API
         self.logger.out('Starting Metadata API at 169.254.169.254:80', state='i')
-        self.thread = threading.Thread(target=self.launch_wsgi)
+        self.thread = Thread(target=self.launch_wsgi)
         self.thread.start()
         self.logger.out('Successfully started Metadata API thread', state='o')
 

node-daemon/pvcnoded/NodeInstance.py

@@ -20,13 +20,9 @@
 #
 ###############################################################################
 
-import os
-import sys
-import psutil
-import socket
 import time
-import libvirt
-import threading
+
+from threading import Thread
 
 import pvcnoded.log as log
 import pvcnoded.zkhandler as zkhandler
@@ -119,13 +115,13 @@ class NodeInstance(object):
                     if self.config['enable_networking']:
                         if self.router_state == 'takeover':
                             self.logger.out('Setting node {} to primary state'.format(self.name), state='i')
-                            transition_thread = threading.Thread(target=self.become_primary, args=(), kwargs={})
+                            transition_thread = Thread(target=self.become_primary, args=(), kwargs={})
                             transition_thread.start()
                         if self.router_state == 'relinquish':
                             # Skip becoming secondary unless already running
                             if self.daemon_state == 'run' or self.daemon_state == 'shutdown':
                                 self.logger.out('Setting node {} to secondary state'.format(self.name), state='i')
-                                transition_thread = threading.Thread(target=self.become_secondary, args=(), kwargs={})
+                                transition_thread = Thread(target=self.become_secondary, args=(), kwargs={})
                                 transition_thread.start()
                             else:
                                 # We did nothing, so just become secondary state
@@ -157,11 +153,11 @@ class NodeInstance(object):
 
                     # Do flushing in a thread so it doesn't block the migrates out
                     if self.domain_state == 'flush':
-                        self.flush_thread = threading.Thread(target=self.flush, args=(), kwargs={})
+                        self.flush_thread = Thread(target=self.flush, args=(), kwargs={})
                         self.flush_thread.start()
                     # Do unflushing in a thread so it doesn't block the migrates in
                     if self.domain_state == 'unflush':
-                        self.flush_thread = threading.Thread(target=self.unflush, args=(), kwargs={})
+                        self.flush_thread = Thread(target=self.unflush, args=(), kwargs={})
                         self.flush_thread.start()
 
         @self.zk_conn.DataWatch('/nodes/{}/memfree'.format(self.name))

node-daemon/pvcnoded/VMConsoleWatcherInstance.py

@@ -21,17 +21,13 @@
 ###############################################################################
 
 import os
-import sys
 import uuid
 import time
-import threading
 import libvirt
 
+from threading import Thread, Event
 from collections import deque
 
-import fcntl
-import signal
-
 import pvcnoded.log as log
 import pvcnoded.zkhandler as zkhandler
 
@@ -62,12 +58,12 @@ class VMConsoleWatcherInstance(object):
 
         # Thread options
         self.thread = None
-        self.thread_stopper = threading.Event()
+        self.thread_stopper = Event()
 
     # Start execution thread
     def start(self):
         self.thread_stopper.clear()
-        self.thread = threading.Thread(target=self.run, args=(), kwargs={})
+        self.thread = Thread(target=self.run, args=(), kwargs={})
         self.logger.out('Starting VM log parser', state='i', prefix='Domain {}:'.format(self.domuuid))
         self.thread.start()
 

node-daemon/pvcnoded/VMInstance.py

@@ -20,16 +20,13 @@
 #
 ###############################################################################
 
-import os
-import sys
 import uuid
-import socket
 import time
-import threading
 import libvirt
-import kazoo.client
 import json
 
+from threading import Thread
+
 import pvcnoded.log as log
 import pvcnoded.zkhandler as zkhandler
 import pvcnoded.common as common
@@ -138,7 +135,7 @@ class VMInstance(object):
 
             # Perform a management command
             self.logger.out('Updating state of VM {}'.format(self.domuuid), state='i')
-            state_thread = threading.Thread(target=self.manage_vm_state, args=(), kwargs={})
+            state_thread = Thread(target=self.manage_vm_state, args=(), kwargs={})
             state_thread.start()
 
     # Get data functions

node-daemon/pvcnoded/VXNetworkInstance.py

@@ -21,8 +21,8 @@
 ###############################################################################
 
 import os
-import sys
 import time
+
 from textwrap import dedent
 
 import pvcnoded.log as log

node-daemon/pvcnoded/common.py

@@ -21,18 +21,18 @@
 ###############################################################################
 
 import subprocess
-import threading
 import signal
-import os
 import time
-import shlex
+
+from threading import Thread
+from shlex import split as shlex_split
 
 import pvcnoded.log as log
 import pvcnoded.zkhandler as zkhandler
 
 class OSDaemon(object):
     def __init__(self, command_string, environment, logfile):
-        command = shlex.split(command_string)
+        command = shlex_split(command_string)
         # Set stdout to be a logfile if set
         if logfile:
             stdout = open(logfile, 'a')
@@ -63,7 +63,7 @@ def run_os_daemon(command_string, environment=None, logfile=None):
 
 # Run a oneshot command, optionally without blocking
 def run_os_command(command_string, background=False, environment=None, timeout=None):
-    command = shlex.split(command_string)
+    command = shlex_split(command_string)
     if background:
         def runcmd():
             try:
@@ -76,7 +76,7 @@ def run_os_command(command_string, background=False, environment=None, timeout=N
                 )
             except subprocess.TimeoutExpired:
                 pass
-        thread = threading.Thread(target=runcmd, args=())
+        thread = Thread(target=runcmd, args=())
         thread.start()
         return 0, None, None
     else:

node-daemon/pvcnoded/fencing.py

@@ -21,7 +21,6 @@
 ###############################################################################
 
 import time
-import threading
 
 import pvcnoded.zkhandler as zkhandler
 import pvcnoded.common as common

node-daemon/pvcnoded/zkhandler.py

@@ -20,7 +20,6 @@
 #
 ###############################################################################
 
-import kazoo.client
 import uuid
 
 # Child list function