Make a whole bunch of things work

client-cli/pvc.py

@@ -883,9 +883,9 @@ def net_acl_remove(net, rule, direction):
 @click.option(
     '--in/--out', 'direction',
     is_flag=True,
-    required=True,
+    required=False,
     default=None,
-    help='Inbound or outbound rule set.'
+    help='Inbound or outbound rule set only.'
 )
 @click.argument(
     'net'

client-common/network.py

@@ -110,17 +110,23 @@ def getNetworkDHCPReservations(zk_conn, vni):
     dhcp_reservations = zkhandler.listchildren(zk_conn, '/networks/{}/dhcp_reservations'.format(vni))
     return sorted(dhcp_reservations)
 
-def getNetworkACLs(zk_conn, vni, direction):
+def getNetworkACLs(zk_conn, vni, _direction):
     # Get the (sorted) list of active ACLs
+    if _direction == 'both':
+        directions = ['in', 'out']
+    else:
+        directions = [_direction]
+
+    full_acl_list = []
+    for direction in directions:
         unordered_acl_list = zkhandler.listchildren(zk_conn, '/networks/{}/firewall_rules/{}'.format(vni, direction))
         ordered_acls = {}
-    full_acl_list = []
         for acl in unordered_acl_list:
             order = zkhandler.readdata(zk_conn, '/networks/{}/firewall_rules/{}/{}/order'.format(vni, direction, acl))
             ordered_acls[order] = acl
 
         for order in sorted(ordered_acls.keys()):
-        full_acl_list.append(ordered_acls[order])
+            full_acl_list.append({'direction': direction, 'description': ordered_acls[order]})
 
     return full_acl_list
 
@@ -154,7 +160,6 @@ def getACLInformation(zk_conn, vni, direction, description):
     rule = zkhandler.readdata(zk_conn, '/networks/{}/firewall_rules/{}/{}/rule'.format(vni, direction, description))
     return order, description, rule
 
-
 def formatNetworkInformation(zk_conn, vni, long_output):
     description, domain, ip_network, ip_gateway, dhcp_flag, dhcp_start, dhcp_end = getNetworkInformation(zk_conn, vni)
 
@@ -394,36 +399,46 @@ def formatDHCPLeaseList(zk_conn, vni, dhcp_leases_list, reservations=False):
     output_string = dhcp_lease_list_output_header + '\n' + '\n'.join(sorted(dhcp_lease_list_output))
     return output_string
 
-def formatACLList(zk_conn, vni, direction, acl_list):
+def formatACLList(zk_conn, vni, _direction, acl_list):
     acl_list_output = []
+    direction = {}
     order = {}
     description = {}
     rule = {}
 
+    if _direction is None:
+        directions = ['in', 'out']
+    else:
+        directions = [_direction]
+
     # Gather information for printing
     for acl in acl_list:
-            order[acl], description[acl], rule[acl] = getACLInformation(zk_conn, vni, direction, acl)
+        acld = acl['description']
+        order[acld], description[acld], rule[acld] = getACLInformation(zk_conn, vni, acl['direction'], acl['description'])
+        direction[acld] = acl['direction']
 
     # Determine optimal column widths
     acl_order_length = 6
     acl_description_length = 12
     acl_rule_length = 5
     for acl in acl_list:
+        acld = acl['description']
         # order column
-        _acl_order_length = len(order[acl]) + 1
+        _acl_order_length = len(order[acld]) + 1
         if _acl_order_length > acl_order_length:
             acl_order_length = _acl_order_length
         # description column
-        _acl_description_length = len(description[acl]) + 1
+        _acl_description_length = len(description[acld]) + 1
         if _acl_description_length > acl_description_length:
             acl_description_length = _acl_description_length
         # rule column
-        _acl_rule_length = len(rule[acl]) + 1
+        _acl_rule_length = len(rule[acld]) + 1
         if _acl_rule_length > acl_rule_length:
             acl_rule_length = _acl_rule_length
 
     # Format the string (header)
     acl_list_output_header = '{bold}\
+{acl_direction: <10} \
 {acl_order: <{acl_order_length}} \
 {acl_description: <{acl_description_length}} \
 {acl_rule: <{acl_rule_length}} \
@@ -433,13 +448,16 @@ def formatACLList(zk_conn, vni, direction, acl_list):
         acl_order_length=acl_order_length,
         acl_description_length=acl_description_length,
         acl_rule_length=acl_rule_length,
+        acl_direction='Direction',
         acl_order='Order',
         acl_description='Description',
         acl_rule='Rule',
     )
 
     for acl in acl_list:
+        acld = acl['description']
         acl_list_output.append('{bold}\
+{acl_direction: <10} \
 {acl_order: <{acl_order_length}} \
 {acl_description: <{acl_description_length}} \
 {acl_rule: <{acl_rule_length}} \
@@ -449,9 +467,10 @@ def formatACLList(zk_conn, vni, direction, acl_list):
                 acl_order_length=acl_order_length,
                 acl_description_length=acl_description_length,
                 acl_rule_length=acl_rule_length,
-                acl_order=order[acl],
+                acl_direction=direction[acld],
-                acl_description=description[acl],
+                acl_order=order[acld],
-                acl_rule=rule[acl],
+                acl_description=description[acld],
+                acl_rule=rule[acld],
             )
         )
 
@@ -639,21 +658,23 @@ def add_acl(zk_conn, network, direction, description, rule, order):
         order = int(order)
       
     # Insert into the array at order-1
-    full_acl_list.insert(order, description)
+    full_acl_list.insert(order, {'direction': direction, 'description': description})
 
     # Update the existing ordering
     updated_orders = {}
     for idx, acl in enumerate(full_acl_list):
-        # We haven't added ourselves yet
+        if acl['description'] == description:
-        if acl == description:
             continue
 
         updated_orders[
-            '/networks/{}/firewall_rules/{}/{}/order'.format(net_vni, direction, acl)
+            '/networks/{}/firewall_rules/{}/{}/order'.format(net_vni, direction, acl['description'])
         ] = idx
 
     if updated_orders:
+        try:
             zkhandler.writedata(zk_conn, updated_orders)
+        except Exception as e:
+            return False, 'ERROR: Failed to write to Zookeeper! Exception: "{}".'.format(e)
 
     # Add the new rule
     try:
@@ -684,8 +705,8 @@ def remove_acl(zk_conn, network, rule, direction):
     # Check if the ACL matches a description currently in the database
     acl_list = getNetworkACLs(zk_conn, net_vni, direction)
     for acl in acl_list:
-        if acl == rule:
+        if acl['description'] == rule:
-            match_description = acl
+            match_description = acl['description']
     
     if not match_description:
         return False, 'ERROR: No firewall rule exists matching description "{}"!'.format(rule)
@@ -697,11 +718,11 @@ def remove_acl(zk_conn, network, rule, direction):
         return False, 'ERROR: Failed to write to Zookeeper! Exception: "{}".'.format(e)
 
     # Update the existing ordering
-    full_acl_list = getNetworkACLs(zk_conn, net_vni, direction)
+    updated_acl_list = getNetworkACLs(zk_conn, net_vni, direction)
     updated_orders = {}
-    for idx, acl in enumerate(full_acl_list):
+    for idx, acl in enumerate(updated_acl_list):
         updated_orders[
-            '/networks/{}/firewall_rules/{}/{}/order'.format(net_vni, direction, acl)
+            '/networks/{}/firewall_rules/{}/{}/order'.format(net_vni, direction, acl['description'])
         ] = idx
 
     if updated_orders:
@@ -803,9 +824,11 @@ def get_list_acl(zk_conn, network, limit, direction):
         return False, 'ERROR: Could not find network "{}" in the cluster!'.format(network)
 
     # Change direction to something more usable
-    if direction:
+    if direction is None:
+        direction = "both"
+    elif direction is True:
         direction = "in"
-    else:
+    elif direction is False:
         direction = "out"
 
     acl_list = []
@@ -821,12 +844,10 @@ def get_list_acl(zk_conn, network, limit, direction):
         except Exception as e:
             return False, 'Regex Error: {}'.format(e)
 
-    for idx, acl in enumerate(full_acl_list):
+    for acl in full_acl_list:
         valid_acl = False
         if limit:
-            if re.match(limit, acl) != None:
+            if re.match(limit, acl['description']) != None:
-                valid_acl = True
-            if re.match(limit, str(idx)) != None:
                 valid_acl = True
         else:
             valid_acl = True

node-daemon/pvcd.service

@@ -10,7 +10,7 @@ Environment = PYTHONUNBUFFERED=true
 Environment = PVCD_CONFIG_FILE=/etc/pvc/pvcd.conf
 ExecStart = /usr/share/pvc/pvcd.py
 KillSignal = SIGINT
-Restart = on-failure
+Restart = never
 
 [Install]
 WantedBy = multi-user.target

node-daemon/pvcd/DNSAggregatorInstance.py

@@ -124,7 +124,6 @@ class DNSAggregatorInstance(object):
         # Connect to the database
         sql_conn = sqlite3.connect(self.database_file)
         sql_curs = sql_conn.cursor()
-        print(network_domain)
         sql_curs.execute(
             'delete from domains where name=?',
             (network_domain,)
@@ -141,7 +140,13 @@ class DNSAggregatorInstance(object):
             prefix='DNS aggregator',
             state='o'
         )
-        common.run_os_command('/usr/bin/pdns_control --socket-dir={} retrieve {}'.format(self.config['pdns_dynamic_directory'], self.d_network[network].domain))
+        common.run_os_command(
+            '/usr/bin/pdns_control --socket-dir={} retrieve {}'.format(
+                self.config['pdns_dynamic_directory'],
+                self.d_network[network].domain
+            ),
+            background=True
+        )
 
     # Start up the PowerDNS instance
     def start_aggregator(self):
@@ -192,6 +197,7 @@ class DNSAggregatorInstance(object):
                 'Stopping PowerDNS zone aggregator',
                 state='o'
             )
-            self.dns_server_daemon.signal('int')
+            # Terminate, then kill
-            time.sleep(0.2)
             self.dns_server_daemon.signal('term')
+            time.sleep(0.2)
+            self.dns_server_daemon.signal('kill')

node-daemon/pvcd/Daemon.py

@@ -484,11 +484,9 @@ include "{rulesdir}/networks/*"
 
 # Write the basic firewall config
 nftables_base_filename = '{}/base.nft'.format(config['nft_dynamic_directory'])
-nftables_update_filename = '{}/update'.format(config['nft_dynamic_directory'])
 with open(nftables_base_filename, 'w') as nfbasefile:
     nfbasefile.write(nftables_base_rules)
-    # Notify a reload of the firewall rules on next keepalive update
+common.reload_firewall_rules(logger, nftables_base_filename)
-    open(nftables_update_filename, 'a').close()
 
 ###############################################################################
 # PHASE 7d - Ensure DNSMASQ is not running

node-daemon/pvcd/NodeInstance.py

@@ -529,26 +529,6 @@ class NodeInstance(object):
         # Close the Libvirt connection
         lv_conn.close()
 
-        # Display node information to the terminal
-        self.logger.out('{}{} keepalive{}'.format(self.logger.fmt_purple, self.name, self.logger.fmt_end), state='t')
-        self.logger.out(
-            '{bold}Domains:{nobold} {domcount}  '
-            '{bold}Networks:{nobold} {netcount}  '
-            '{bold}VM memory [MiB]:{nobold} {allocmem}  '
-            '{bold}Free memory [MiB]:{nobold} {freemem}  '
-            '{bold}Used memory [MiB]:{nobold} {usedmem}  '
-            '{bold}Load:{nobold} {load}'.format(
-                bold=self.logger.fmt_bold,
-                nobold=self.logger.fmt_end,
-                domcount=self.domains_count,
-                freemem=self.memfree,
-                usedmem=self.memused,
-                load=self.cpuload,
-                allocmem=self.memalloc,
-                netcount=self.networks_count
-            ),
-        )
-
         # Update our local node lists
         for node_name in self.d_node:
             try:
@@ -610,6 +590,26 @@ class NodeInstance(object):
                 if node in self.inactive_node_list:
                     secondary_node_list.remove(node)
 
+        # Display node information to the terminal
+        self.logger.out('{}{} keepalive{}'.format(self.logger.fmt_purple, self.name, self.logger.fmt_end), state='t')
+        self.logger.out(
+            '{bold}Domains:{nobold} {domcount}  '
+            '{bold}Networks:{nobold} {netcount}  '
+            '{bold}VM memory [MiB]:{nobold} {allocmem}  '
+            '{bold}Free memory [MiB]:{nobold} {freemem}  '
+            '{bold}Used memory [MiB]:{nobold} {usedmem}  '
+            '{bold}Load:{nobold} {load}'.format(
+                bold=self.logger.fmt_bold,
+                nobold=self.logger.fmt_end,
+                domcount=self.domains_count,
+                freemem=self.memfree,
+                usedmem=self.memused,
+                load=self.cpuload,
+                allocmem=self.memalloc,
+                netcount=self.networks_count
+            ),
+        )
+
         # Display cluster information to the terminal
         self.logger.out('{}Cluster status{}'.format(self.logger.fmt_purple, self.logger.fmt_end), state='t')
         self.logger.out('{}Primary coordinator:{} {}'.format(self.logger.fmt_bold, self.logger.fmt_end, self.primary_node))

node-daemon/pvcd/VXNetworkInstance.py

@@ -43,8 +43,8 @@ class VXNetworkInstance(object):
         self.description = None
         self.domain = None
         self.ip_gateway = zkhandler.readdata(self.zk_conn, '/networks/{}/ip_gateway'.format(self.vni))
-        self.ip_network = None
+        self.ip_network = zkhandler.readdata(self.zk_conn, '/networks/{}/ip_network'.format(self.vni))
-        self.ip_cidrnetmask = None
+        self.ip_cidrnetmask = zkhandler.readdata(self.zk_conn, '/networks/{}/ip_network'.format(self.vni)).split('/')[-1]
         self.dhcp_flag = zkhandler.readdata(self.zk_conn, '/networks/{}/dhcp_flag'.format(self.vni))
         self.dhcp_start = None
         self.dhcp_end = None
@@ -52,7 +52,6 @@ class VXNetworkInstance(object):
         self.vxlan_nic = 'vxlan{}'.format(self.vni)
         self.bridge_nic = 'br{}'.format(self.vni)
 
-        self.nftables_update_filename = '{}/update'.format(config['nft_dynamic_directory'])
         self.nftables_netconf_filename = '{}/networks/{}.nft'.format(config['nft_dynamic_directory'], self.vni)
         self.firewall_rules = []
 
@@ -60,6 +59,30 @@ class VXNetworkInstance(object):
         self.dnsmasq_hostsdir = '{}/{}'.format(config['dnsmasq_dynamic_directory'], self.vni)
         self.dhcp_reservations = []
 
+        self.firewall_rules_base = """# Rules for network {vxlannic}
+add chain inet filter {vxlannic}-in
+add chain inet filter {vxlannic}-out
+add rule inet filter {vxlannic}-in counter
+add rule inet filter {vxlannic}-out counter
+# Jump from forward chain to this chain when matching net
+add rule inet filter forward ip daddr {netaddr} counter jump {vxlannic}-in
+add rule inet filter forward ip saddr {netaddr} counter jump {vxlannic}-out
+# Allow ICMP traffic into the router from network
+add rule inet filter input ip protocol icmp meta iifname {bridgenic} counter accept
+# Allow DNS and DHCP traffic into the router from network
+add rule inet filter input tcp dport 53 meta iifname {bridgenic} counter accept
+add rule inet filter input udp dport 53 meta iifname {bridgenic} counter accept
+add rule inet filter input udp dport 67 meta iifname {bridgenic} counter accept
+# Block traffic into the router from network
+add rule inet filter input meta iifname {bridgenic} counter drop
+""".format(
+            netaddr=self.ip_network,
+            vxlannic=self.vxlan_nic,
+            bridgenic=self.bridge_nic
+        )
+        self.firewall_rules_in = zkhandler.listchildren(self.zk_conn, '/networks/{}/firewall_rules/in'.format(self.vni))
+        self.firewall_rules_out = zkhandler.listchildren(self.zk_conn, '/networks/{}/firewall_rules/out'.format(self.vni))
+
         # Zookeper handlers for changed states
         @self.zk_conn.DataWatch('/networks/{}'.format(self.vni))
         def watch_network_description(data, stat, event=''):
@@ -157,18 +180,29 @@ class VXNetworkInstance(object):
                 if self.this_node.router_state == 'primary':
                     self.updateDHCPReservations(old_reservations, new_reservations)
 
-        @self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules'.format(self.vni))
+        @self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules/in'.format(self.vni))
         def watch_network_firewall_rules(new_rules, event=''):
             if event and event.type == 'DELETED':
                 # The key has been deleted after existing before; terminate this watcher
                 # because this class instance is about to be reaped in Daemon.py
                 return False
 
-            if self.firewall_rules != new_rules:
+            # Don't run on the first pass
-                old_rules = self.firewall_rules
+            if self.firewall_rules_in != new_rules:
-                self.firewall_rules = new_rules
+                self.firewall_rules_in = new_rules
-                if self.this_node.router_state == 'primary':
+                self.updateFirewallRules()
-                    self.updateFirewallRules(old_rules, new_rules)
+
+        @self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules/out'.format(self.vni))
+        def watch_network_firewall_rules(new_rules, event=''):
+            if event and event.type == 'DELETED':
+                # The key has been deleted after existing before; terminate this watcher
+                # because this class instance is about to be reaped in Daemon.py
+                return False
+
+            # Don't run on the first pass
+            if self.firewall_rules_out != new_rules:
+                self.firewall_rules_out = new_rules
+                self.updateFirewallRules()
 
         self.createNetwork()
         self.createFirewall()
@@ -203,17 +237,47 @@ class VXNetworkInstance(object):
                 except:
                     pass
 
-    def updateFirewallRules(self, old_rules_list, new_rules_list):
+    def updateFirewallRules(self):
-        for rule in new_rules_list:
+        self.logger.out(
-            if rule not in old_rules_list:
+            'Updating firewall rules',
-                # Add new rule entry
+            prefix='VNI {}'.format(self.vni),
-                print(rule)
+            state='o'
-                pass
+        )
+        ordered_acls_in = {}
+        ordered_acls_out = {}
+        sorted_acl_list = {'in': [], 'out': []}
+        full_ordered_rules = []
 
-        for rule in old_rules_list:
+        for acl in self.firewall_rules_in:
-            if rule not in new_rules_list:
+            order = zkhandler.readdata(self.zk_conn, '/networks/{}/firewall_rules/in/{}/order'.format(self.vni, acl))
-                print(rule)
+            ordered_acls_in[order] = acl
-                pass
+        for acl in self.firewall_rules_out:
+            order = zkhandler.readdata(self.zk_conn, '/networks/{}/firewall_rules/out/{}/order'.format(self.vni, acl))
+            ordered_acls_out[order] = acl
+    
+        for order in sorted(ordered_acls_in.keys()):
+            sorted_acl_list['in'].append(ordered_acls_in[order])
+        for order in sorted(ordered_acls_out.keys()):
+            sorted_acl_list['out'].append(ordered_acls_out[order])
+   
+        for direction in 'in', 'out': 
+            for acl in sorted_acl_list[direction]:
+                rule_prefix = "add rule inet filter vxlan{}-{} counter".format(self.vni, direction)
+                rule_data = zkhandler.readdata(self.zk_conn, '/networks/{}/firewall_rules/{}/{}/rule'.format(self.vni, direction, acl))
+                rule = '{} {}'.format(rule_prefix, rule_data)
+                full_ordered_rules.append(rule)
+
+        output = "{}\n# User rules\n{}\n".format(
+                     self.firewall_rules_base,
+                     '\n'.join(full_ordered_rules)
+                 )
+
+        with open(self.nftables_netconf_filename, 'w') as nfnetfile:
+            nfnetfile.write(dedent(output))
+
+        # Reload firewall rules
+        nftables_base_filename = '{}/base.nft'.format(self.config['nft_dynamic_directory'])
+        common.reload_firewall_rules(self.logger, nftables_base_filename)
 
     def createNetwork(self):
         self.logger.out(
@@ -253,31 +317,8 @@ class VXNetworkInstance(object):
         )
 
     def createFirewall(self):
-        nftables_network_rules = """# Rules for network {vxlannic}
+        # For future use
-add chain inet filter {vxlannic}-in
+        self.updateFirewallRules()
-add chain inet filter {vxlannic}-out
-add rule inet filter {vxlannic}-in counter
-add rule inet filter {vxlannic}-out counter
-# Jump from forward chain to this chain when matching net
-add rule inet filter forward ip daddr {netaddr} counter jump {vxlannic}-in
-add rule inet filter forward ip saddr {netaddr} counter jump {vxlannic}-out
-# Allow ICMP traffic into the router from network
-add rule inet filter input ip protocol icmp meta iifname {bridgenic} counter accept
-# Allow DNS and DHCP traffic into the router from network
-add rule inet filter input tcp dport 53 meta iifname {bridgenic} counter accept
-add rule inet filter input udp dport 53 meta iifname {bridgenic} counter accept
-add rule inet filter input udp dport 67 meta iifname {bridgenic} counter accept
-# Block traffic into the router from network
-add rule inet filter input meta iifname {bridgenic} counter drop
-""".format(
-            netaddr=self.ip_network,
-            vxlannic=self.vxlan_nic,
-            bridgenic=self.bridge_nic
-        )
-        with open(self.nftables_netconf_filename, 'w') as nfbasefile:
-            nfbasefile.write(dedent(nftables_network_rules))
-            open(self.nftables_update_filename, 'a').close()
-        pass
 
     def createGatewayAddress(self):
         if self.this_node.router_state == 'primary':
@@ -342,11 +383,10 @@ add rule inet filter input meta iifname {bridgenic} counter drop
                 '--listen-address={}'.format(self.ip_gateway),
                 '--bind-interfaces',
                 '--leasefile-ro',
-                '--dhcp-script=./pvcd/dnsmasq-zookeeper-leases.py',
+                '--dhcp-script={}/pvcd/dnsmasq-zookeeper-leases.py'.format(os.getcwd()),
                 '--dhcp-range={},{},48h'.format(self.dhcp_start, self.dhcp_end),
                 '--dhcp-hostsdir={}'.format(self.dnsmasq_hostsdir),
                 '--log-facility=-',
-                '--log-queries=extra',
                 '--keep-in-foreground'
             ]
             # Start the dnsmasq process in a thread
@@ -394,10 +434,21 @@ add rule inet filter input meta iifname {bridgenic} counter drop
         )
 
     def removeFirewall(self):
+        self.logger.out(
+            'Removing firewall rules',
+            prefix='VNI {}'.format(self.vni),
+            state='o'
+        )
+
+        try:
             os.remove(self.nftables_netconf_filename)
-        open(self.nftables_update_filename, 'a').close()
+        except:
             pass
 
+        # Reload firewall rules
+        nftables_base_filename = '{}/base.nft'.format(self.config['nft_dynamic_directory'])
+        common.reload_firewall_rules(self.logger, nftables_base_filename)
+
     def removeGatewayAddress(self):
         self.logger.out(
             'Removing gateway {} from interface {}'.format(

node-daemon/pvcd/common.py

@@ -83,9 +83,8 @@ def run_os_command(command_string, background=False, environment=None):
         return command_output.returncode, command_output.stdout.decode('ascii'), command_output.stderr.decode('ascii')
 
 # Reload the firewall rules of the system
-def reload_firewall_rules(rules_dir):
+def reload_firewall_rules(logger, rules_file):
-    log.echo('Updating firewall rules', '', 'o')
+    logger.out('Reloading firewall configuration', state='o')
-    rules_file = '{}/base.nft'.format(rules_dir)
     retcode, stdout, stderr = run_os_command('/usr/sbin/nft -f {}'.format(rules_file))
     if retcode != 0:
-        log.echo('Failed to reload rules: {}'.format(stderr), '', 'e')
+        logger.out('Failed to reload configuration: {}'.format(stderr), state='e')