Correct multiple issues with DHCP and add firewall control logic

daemon-common/daemon_lib/common.py

@@ -23,6 +23,8 @@
 import subprocess
 import threading
 import signal
+import os
+import time
 
 import daemon_lib.ansiiprint as ansiiprint
 
@@ -38,7 +40,8 @@ class OSDaemon(object):
     def signal(self, sent_signal):
         signal_map = {
             'hup': signal.SIGHUP,
-            'int': signal.SIGINT
+            'int': signal.SIGINT,
+            'term': signal.SIGTERM
         }
         self.proc.send_signal(signal_map[sent_signal])
 
@@ -52,7 +55,7 @@ def run_os_command(command_string, background=False, environment=None):
     command = command_string.split()
     if background:
         def runcmd():
-            subprocess.Popen(
+            subprocess.run(
                 command,
                 env=environment,
                 stdout=subprocess.PIPE,
@@ -60,12 +63,20 @@ def run_os_command(command_string, background=False, environment=None):
             )
         thread = threading.Thread(target=runcmd, args=())
         thread.start()
-        return 0
+        return 0, None, None
     else:
-        command_output = subprocess.Popen(
+        command_output = subprocess.run(
             command,
             env=environment,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
-        return command_output.returncode
+        return command_output.returncode, command_output.stdout.decode('ascii'), command_output.stderr.decode('ascii')
+
+# Reload the firewall rules of the system
+def reload_firewall_rules(rules_dir):
+    ansiiprint.echo('Updating firewall rules', '', 'o')
+    rules_file = '{}/base.nft'.format(rules_dir)
+    retcode, stdout, stderr = run_os_command('/usr/sbin/nft -f {}'.format(rules_file))
+    if retcode != 0:
+        ansiiprint.echo('Failed to reload rules: {}'.format(stderr), '', 'e')

router-daemon/pvcrd/Daemon.py

@@ -44,8 +44,13 @@ print(ansiiprint.bold() + "pvcrd - Parallel Virtual Cluster router daemon" + ans
 # Set sysctl to enable routing before we do anything else
 common.run_os_command('sysctl net.ipv4.ip_forward=1')
 common.run_os_command('sysctl net.ipv4.conf.all.send_redirects=1')
+common.run_os_command('sysctl net.ipv4.conf.all.rp_filter=0')
+common.run_os_command('sysctl net.ipv4.conf.default.rp_filter=0')
+common.run_os_command('sysctl net.ipv4.conf.all.accept_source_route=1')
 common.run_os_command('sysctl net.ipv4.conf.all.accept_source_route=1')
 common.run_os_command('sysctl net.ipv6.ip_forward=1')
+common.run_os_command('sysctl net.ipv6.conf.all.rp_filter=0')
+common.run_os_command('sysctl net.ipv6.conf.default.rp_filter=0')
 common.run_os_command('sysctl net.ipv6.conf.all.send_redirects=1')
 common.run_os_command('sysctl net.ipv6.conf.all.accept_source_route=1')
 
@@ -108,6 +113,10 @@ def readConfig(pvcrd_config_file, myhostname):
 # Get config
 config = readConfig(pvcrd_config_file, myhostname)
 
+# Add some static config elements
+config['nftables_rules_dir'] = '/var/lib/pvc/nftables'
+config['dnsmasq_hosts_dir'] = '/var/lib/pvc/dnsmasq'
+
 # Set up our VNI interface
 vni_dev = config['vni_dev']
 vni_dev_ip = config['vni_dev_ip']
@@ -231,6 +240,45 @@ s_network = dict()
 router_list = []
 network_list = []
 
+# Create our config dirs
+common.run_os_command(
+    '/bin/mkdir --parents {}/networks'.format(
+        config['nftables_rules_dir']
+     )
+)
+common.run_os_command(
+    '/bin/mkdir --parents {}/static'.format(
+        config['nftables_rules_dir']
+     )
+)
+common.run_os_command(
+    '/bin/mkdir --parents {}'.format(
+        config['dnsmasq_hosts_dir']
+     )
+)
+
+# Set up the basic features of the nftables firewall
+nftables_base_rules = """# Base rules
+flush ruleset
+add table inet filter
+add chain inet filter forward {{ type filter hook forward priority 0; }}
+include "{rulesdir}/static/*"
+include "{rulesdir}/networks/*"
+""".format(
+    rulesdir=config['nftables_rules_dir']
+)
+
+# Write the basic firewall config
+print(nftables_base_rules)
+nftables_base_filename = '{}/base.nft'.format(config['nftables_rules_dir'])
+nftables_update_filename = '{}/update'.format(config['nftables_rules_dir'])
+with open(nftables_base_filename, 'w') as nfbasefile:
+    nfbasefile.write(nftables_base_rules)
+    open(nftables_update_filename, 'a').close()
+
+#
+# Router instances
+#
 @zk_conn.ChildrenWatch('/routers')
 def updaterouters(new_router_list):
     global router_list
@@ -246,6 +294,9 @@ def updaterouters(new_router_list):
 this_router = t_router[myhostname]
 update_zookeeper = this_router.update_zookeeper
 
+#
+# Network instances
+#
 @zk_conn.ChildrenWatch('/networks')
 def updatenetworks(new_network_list):
     global network_list
@@ -260,6 +311,7 @@ def updatenetworks(new_network_list):
             if this_router.network_state == 'primary':
                 s_network[network].stopDHCPServer()
                 s_network[network].removeGatewayAddress()
+            s_network[network].removeFirewall()
             s_network[network].removeNetwork()
             del(s_network[network])
     network_list = new_network_list

router-daemon/pvcrd/RouterInstance.py

@@ -236,6 +236,11 @@ class RouterInstance():
         ansiiprint.echo('{}Secondary router:{} {}'.format(ansiiprint.bold(), ansiiprint.end(), ' '.join(self.secondary_router_list)), '', 'c')
         ansiiprint.echo('{}Inactive routers:{} {}'.format(ansiiprint.bold(), ansiiprint.end(), ' '.join(self.inactive_router_list)), '', 'c')
 
+        # Reload firewall rules if needed
+        if os.path.isfile('{}/update'.format(self.config['nftables_rules_dir'])):
+            common.reload_firewall_rules(self.config['nftables_rules_dir'])
+            os.remove('{}/update'.format(self.config['nftables_rules_dir']))
+
 #
 # Fence thread entry function
 #

router-daemon/pvcrd/VXNetworkInstance.py

@@ -22,6 +22,7 @@
 
 import os
 import sys
+from textwrap import dedent
 
 import daemon_lib.ansiiprint as ansiiprint
 import daemon_lib.zkhandler as zkhandler
@@ -48,14 +49,13 @@ class VXNetworkInstance():
         self.vxlan_nic = 'vxlan{}'.format(self.vni)
         self.bridge_nic = 'br{}'.format(self.vni)
 
-        self.firewall_rules = {}
+        self.nftables_update_filename = '{}/update'.format(config['nftables_rules_dir'])
+        self.nftables_netconf_filename = '{}/networks/{}.nft'.format(config['nftables_rules_dir'], self.vni)
+        self.firewall_rules = []
 
         self.dhcp_server_daemon = None
-
+        self.dnsmasq_hostsdir = '{}/{}'.format(config['dnsmasq_hosts_dir'], self.vni)
-        self.dnsmasq_hostsdir = '/var/lib/dnsmasq/{}'.format(self.vni)
+        self.dhcp_reservations = None
-        self.dhcp_reservations = zkhandler.listchildren(self.zk_conn, '/networks/{}/dhcp_reservations'.format(self.vni))
-
-        self.createNetwork()
 
         # Zookeper handlers for changed states
         @self.zk_conn.DataWatch('/networks/{}'.format(self.vni))
@@ -142,15 +142,38 @@ class VXNetworkInstance():
                 self.dhcp_end = data.decode('ascii')
 
         @self.zk_conn.ChildrenWatch('/networks/{}/dhcp_reservations'.format(self.vni))
-        def watch_network_dhcp_reservations(reservations, event=''):
+        def watch_network_dhcp_reservations(new_reservations, event=''):
             if event and event.type == 'DELETED':
                 # The key has been deleted after existing before; terminate this watcher
                 # because this class instance is about to be reaped in Daemon.py
                 return False
 
-            if self.dhcp_reservations != reservations:
+            if self.dhcp_reservations != new_reservations:
-                for reservation in reservations:
+                old_reservations = self.dhcp_reservations
-                    if reservation not in self.dhcp_reservations:
+                self.dhcp_reservations = new_reservations
+                self.updateDHCPReservations(old_reservations, new_reservations)
+
+        @self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules'.format(self.vni))
+        def watch_network_firewall_rules(new_rules, event=''):
+            if event and event.type == 'DELETED':
+                # The key has been deleted after existing before; terminate this watcher
+                # because this class instance is about to be reaped in Daemon.py
+                return False
+
+            if self.firewall_rules != new_rules:
+                old_rules = self.firewall_rules
+                self.firewall_rules = new_rules
+                self.updateFirewallRules(old_rules, new_rules)
+
+        self.createNetwork()
+        self.createFirewall()
+
+    def getvni(self):
+        return self.vni
+
+    def updateDHCPReservations(self, old_reservations_list, new_reservations_list):
+        for reservation in new_reservations_list:
+            if reservation not in old_reservations_list:
                 # Add new reservation file
                 filename = '{}/{}'.format(self.dnsmasq_hostsdir, reservation)
                 ipaddr = zkhandler.readdata(
@@ -165,20 +188,25 @@ class VXNetworkInstance():
                 outfile.write(entry)
                 outfile.close()
 
-                for reservation in self.dhcp_reservations:
+        for reservation in old_reservations_list:
-                    if reservation not in reservations:
+            if reservation not in new_reservations_list:
-                        filename = '{}/{}'.format(self.dnsmasq_hostsdir, reservation)
                 # Remove old reservation file
+                filename = '{}/{}'.format(self.dnsmasq_hostsdir, reservation)
                 try:
                     os.remove(filename)
                     self.dhcp_server_daemon.signal('hup')
                 except:
                     pass
 
-                self.dhcp_reservations = reservations
+    def updateFirewallRules(self, old_rules_list, new_rules_list):
+        for rule in new_rules_list:
+            if rule not in old_rules_list:
+                # Add new rule entry
+                pass
 
-    def getvni(self):
+        for rule in old_rules_list:
-        return self.vni
+            if rule not in new_rules_list:
+                pass
 
     def createNetwork(self):
         ansiiprint.echo(
@@ -218,6 +246,23 @@ class VXNetworkInstance():
             )
         )
 
+    def createFirewall(self):
+        nftables_network_rules = """# Rules for network {chainname}
+add chain inet filter {chainname}
+add rule inet filter {chainname} counter
+# Jump from forward chain to this chain when matching netaddr
+add rule inet filter forward ip saddr {netaddr} counter jump {chainname}
+add rule inet filter forward ip daddr {netaddr} counter jump {chainname}
+""".format(
+            netaddr=self.ip_network,
+            chainname=self.vxlan_nic
+        )
+        print(nftables_network_rules)
+        with open(self.nftables_netconf_filename, 'w') as nfbasefile:
+            nfbasefile.write(dedent(nftables_network_rules))
+            open(self.nftables_update_filename, 'a').close()
+        pass
+
     def createGatewayAddress(self):
         if self.this_router.getnetworkstate() == 'primary':
             ansiiprint.echo(
@@ -229,6 +274,12 @@ class VXNetworkInstance():
                 '',
                 'o'
             )
+            print('ip address add {}/{} dev {}'.format(
+                                self.ip_gateway,
+                                self.ip_cidrnetmask,
+                                self.bridge_nic
+                            ))
+
             common.run_os_command(
                 'ip address add {}/{} dev {}'.format(
                     self.ip_gateway,
@@ -330,6 +381,11 @@ class VXNetworkInstance():
             )
         )
 
+    def removeFirewall(self):
+        os.remove(self.nftables_netconf_filename)
+        open(self.nftables_update_filename, 'a').close()
+        pass
+
     def removeGatewayAddress(self):
         ansiiprint.echo(
             'Removing gateway {} from interface {} (VNI {})'.format(
@@ -358,4 +414,4 @@ class VXNetworkInstance():
                 '',
                 'o'
             )
-            self.dhcp_server_daemon.signal('int')
+            self.dhcp_server_daemon.signal('term')