Make the IP failover work including threaded background os commands

daemon-common/daemon_lib/common.py

@@ -21,12 +21,24 @@
 ###############################################################################
 
 import subprocess
+import threading
 
-def run_os_command(command_string):
-    command = command_string.split() 
-    command_output = subprocess.run(
-        command,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE
-    )
-    return command_output.returncode
+def run_os_command(command_string, background=False):
+    command = command_string.split()
+    if background:
+        def runcmd():
+            subprocess.run(
+                command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE
+            )
+        thread = threading.Thread(target=runcmd, args=())
+        thread.start()
+        return 0
+    else:
+        command_output = subprocess.run(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE
+        )
+        return command_output.returncode

debian/control

@@ -35,7 +35,7 @@ Description: Parallel Virtual Cluster network daemon (Python 3)
 
 Package: pvc-router-daemon
 Architecture: all
-Depends: pvc-daemon-common
+Depends: pvc-daemon-common, arping
 Description: Parallel Virtual Cluster router daemon (Python 3)
  The Parallel Virtual Cluster provides a management solution for QEMU/KVM virtual clusters.
  .

router-daemon/pvcrd/Daemon.py

@@ -33,6 +33,7 @@ import apscheduler.schedulers.background
 
 import daemon_lib.ansiiprint as ansiiprint
 import daemon_lib.zkhandler as zkhandler
+import daemon_lib.common as common
 
 import pvcrd.RouterInstance as RouterInstance
 import pvcrd.VXNetworkInstance as VXNetworkInstance
@@ -97,6 +98,13 @@ def readConfig(pvcrd_config_file, myhostname):
 # Get config
 config = readConfig(pvcrd_config_file, myhostname)
 
+# Set up our VNI interface
+vni_dev = config['vni_dev']
+vni_dev_ip = config['vni_dev_ip']
+print('Setting up VNI interface {} with IP {}'.format(vni_dev, vni_dev_ip)
+common.run_os_command('ip link set {} up'.format(vni_dev))
+common.run_os_command('ip address add {} dev {}'.format(vni_dev_ip, vni_dev))
+
 # Connect to local zookeeper
 zk_conn = kazoo.client.KazooClient(hosts=config['zookeeper'])
 try:

router-daemon/pvcrd/RouterInstance.py

@@ -98,11 +98,13 @@ class RouterInstance():
     def become_secondary(self):
         ansiiprint.echo('Setting router {} to secondary state'.format(self.name), '', 'i')
         ansiiprint.echo('Network list: {}'.format(', '.join(self.network_list)), '', 'c')
-        for network in self.s_network:
-            self.s_network[network].removeAddress()
         for router in self.t_router:
             if self.t_router[router].getname() != self.this_router:
-                zkhandler.writedata(self.zk_conn, { '/routers/{}/networkstate'.format(self.t_router[router].getname()): 'primary' })
+                if self.t_router[router].getnetworkstate() != 'primary':
+                    zkhandler.writedata(self.zk_conn, { '/routers/{}/networkstate'.format(self.t_router[router].getname()): 'primary' })
+        time.sleep(1)
+        for network in self.s_network:
+            self.s_network[network].removeAddress()
 
     def set_secondary(self):
         zkhandler.writedata(self.zk_conn, { '/routers/{}/networkstate'.format(self.name): 'secondary' })
@@ -114,7 +116,8 @@ class RouterInstance():
             self.s_network[network].createAddress()
         for router in self.t_router:
             if self.t_router[router].getname() != self.this_router:
-                zkhandler.writedata(self.zk_conn, { '/routers/{}/networkstate'.format(self.t_router[router].getname()): 'secondary' })
+                if self.t_router[router].getnetworkstate() != 'secondary':
+                    zkhandler.writedata(self.zk_conn, { '/routers/{}/networkstate'.format(self.t_router[router].getname()): 'secondary' })
 
     def set_primary(self):
         zkhandler.writedata(self.zk_conn, { '/routers/{}/networkstate'.format(self.name): 'primary' })

router-daemon/pvcrd/VXNetworkInstance.py

@@ -89,6 +89,7 @@ class VXNetworkInstance():
         if self.this_router.getnetworkstate() == 'primary':
             ansiiprint.echo('Creating gateway {} on interface {}'.format(self.ip_gateway, self.vni_dev), '', 'o')
             common.run_os_command('ip address add {}/{} dev {}'.format(self.ip_gateway, self.ip_cidrnetmask, self.bridge_nic))
+            common.run_os_command('arping -A -c1 -I {} {}'.format(self.bridge_nic, self.ip_gateway), background=True)
 
     def removeNetwork(self):
         ansiiprint.echo('Removing VNI {} device on interface {}'.format(self.vni, self.vni_dev), '', 'o')