From 187a572c13055d65204ed7e4a7a382320542e2c9 Mon Sep 17 00:00:00 2001 From: Joshua Boniface Date: Wed, 17 Oct 2018 20:05:22 -0400 Subject: [PATCH] Make a whole bunch of things work --- client-cli/pvc.py | 4 +- client-common/network.py | 89 ++++++++----- node-daemon/pvcd.service | 2 +- node-daemon/pvcd/DNSAggregatorInstance.py | 14 +- node-daemon/pvcd/Daemon.py | 4 +- node-daemon/pvcd/NodeInstance.py | 40 +++--- node-daemon/pvcd/VXNetworkInstance.py | 149 +++++++++++++++------- node-daemon/pvcd/common.py | 7 +- 8 files changed, 192 insertions(+), 117 deletions(-) diff --git a/client-cli/pvc.py b/client-cli/pvc.py index 87d9570f..a60c9939 100755 --- a/client-cli/pvc.py +++ b/client-cli/pvc.py @@ -883,9 +883,9 @@ def net_acl_remove(net, rule, direction): @click.option( '--in/--out', 'direction', is_flag=True, - required=True, + required=False, default=None, - help='Inbound or outbound rule set.' + help='Inbound or outbound rule set only.' ) @click.argument( 'net' diff --git a/client-common/network.py b/client-common/network.py index f86ec019..70faa2d6 100644 --- a/client-common/network.py +++ b/client-common/network.py @@ -110,17 +110,23 @@ def getNetworkDHCPReservations(zk_conn, vni): dhcp_reservations = zkhandler.listchildren(zk_conn, '/networks/{}/dhcp_reservations'.format(vni)) return sorted(dhcp_reservations) -def getNetworkACLs(zk_conn, vni, direction): +def getNetworkACLs(zk_conn, vni, _direction): # Get the (sorted) list of active ACLs - unordered_acl_list = zkhandler.listchildren(zk_conn, '/networks/{}/firewall_rules/{}'.format(vni, direction)) - ordered_acls = {} - full_acl_list = [] - for acl in unordered_acl_list: - order = zkhandler.readdata(zk_conn, '/networks/{}/firewall_rules/{}/{}/order'.format(vni, direction, acl)) - ordered_acls[order] = acl + if _direction == 'both': + directions = ['in', 'out'] + else: + directions = [_direction] - for order in sorted(ordered_acls.keys()): - full_acl_list.append(ordered_acls[order]) + full_acl_list = [] + for direction in directions: + unordered_acl_list = zkhandler.listchildren(zk_conn, '/networks/{}/firewall_rules/{}'.format(vni, direction)) + ordered_acls = {} + for acl in unordered_acl_list: + order = zkhandler.readdata(zk_conn, '/networks/{}/firewall_rules/{}/{}/order'.format(vni, direction, acl)) + ordered_acls[order] = acl + + for order in sorted(ordered_acls.keys()): + full_acl_list.append({'direction': direction, 'description': ordered_acls[order]}) return full_acl_list @@ -154,7 +160,6 @@ def getACLInformation(zk_conn, vni, direction, description): rule = zkhandler.readdata(zk_conn, '/networks/{}/firewall_rules/{}/{}/rule'.format(vni, direction, description)) return order, description, rule - def formatNetworkInformation(zk_conn, vni, long_output): description, domain, ip_network, ip_gateway, dhcp_flag, dhcp_start, dhcp_end = getNetworkInformation(zk_conn, vni) @@ -394,36 +399,46 @@ def formatDHCPLeaseList(zk_conn, vni, dhcp_leases_list, reservations=False): output_string = dhcp_lease_list_output_header + '\n' + '\n'.join(sorted(dhcp_lease_list_output)) return output_string -def formatACLList(zk_conn, vni, direction, acl_list): +def formatACLList(zk_conn, vni, _direction, acl_list): acl_list_output = [] + direction = {} order = {} description = {} rule = {} + if _direction is None: + directions = ['in', 'out'] + else: + directions = [_direction] + # Gather information for printing for acl in acl_list: - order[acl], description[acl], rule[acl] = getACLInformation(zk_conn, vni, direction, acl) + acld = acl['description'] + order[acld], description[acld], rule[acld] = getACLInformation(zk_conn, vni, acl['direction'], acl['description']) + direction[acld] = acl['direction'] # Determine optimal column widths acl_order_length = 6 acl_description_length = 12 acl_rule_length = 5 for acl in acl_list: + acld = acl['description'] # order column - _acl_order_length = len(order[acl]) + 1 + _acl_order_length = len(order[acld]) + 1 if _acl_order_length > acl_order_length: acl_order_length = _acl_order_length # description column - _acl_description_length = len(description[acl]) + 1 + _acl_description_length = len(description[acld]) + 1 if _acl_description_length > acl_description_length: acl_description_length = _acl_description_length # rule column - _acl_rule_length = len(rule[acl]) + 1 + _acl_rule_length = len(rule[acld]) + 1 if _acl_rule_length > acl_rule_length: acl_rule_length = _acl_rule_length # Format the string (header) acl_list_output_header = '{bold}\ +{acl_direction: <10} \ {acl_order: <{acl_order_length}} \ {acl_description: <{acl_description_length}} \ {acl_rule: <{acl_rule_length}} \ @@ -433,13 +448,16 @@ def formatACLList(zk_conn, vni, direction, acl_list): acl_order_length=acl_order_length, acl_description_length=acl_description_length, acl_rule_length=acl_rule_length, + acl_direction='Direction', acl_order='Order', acl_description='Description', acl_rule='Rule', ) for acl in acl_list: + acld = acl['description'] acl_list_output.append('{bold}\ +{acl_direction: <10} \ {acl_order: <{acl_order_length}} \ {acl_description: <{acl_description_length}} \ {acl_rule: <{acl_rule_length}} \ @@ -449,9 +467,10 @@ def formatACLList(zk_conn, vni, direction, acl_list): acl_order_length=acl_order_length, acl_description_length=acl_description_length, acl_rule_length=acl_rule_length, - acl_order=order[acl], - acl_description=description[acl], - acl_rule=rule[acl], + acl_direction=direction[acld], + acl_order=order[acld], + acl_description=description[acld], + acl_rule=rule[acld], ) ) @@ -639,21 +658,23 @@ def add_acl(zk_conn, network, direction, description, rule, order): order = int(order) # Insert into the array at order-1 - full_acl_list.insert(order, description) + full_acl_list.insert(order, {'direction': direction, 'description': description}) # Update the existing ordering updated_orders = {} for idx, acl in enumerate(full_acl_list): - # We haven't added ourselves yet - if acl == description: + if acl['description'] == description: continue updated_orders[ - '/networks/{}/firewall_rules/{}/{}/order'.format(net_vni, direction, acl) + '/networks/{}/firewall_rules/{}/{}/order'.format(net_vni, direction, acl['description']) ] = idx if updated_orders: - zkhandler.writedata(zk_conn, updated_orders) + try: + zkhandler.writedata(zk_conn, updated_orders) + except Exception as e: + return False, 'ERROR: Failed to write to Zookeeper! Exception: "{}".'.format(e) # Add the new rule try: @@ -684,8 +705,8 @@ def remove_acl(zk_conn, network, rule, direction): # Check if the ACL matches a description currently in the database acl_list = getNetworkACLs(zk_conn, net_vni, direction) for acl in acl_list: - if acl == rule: - match_description = acl + if acl['description'] == rule: + match_description = acl['description'] if not match_description: return False, 'ERROR: No firewall rule exists matching description "{}"!'.format(rule) @@ -697,11 +718,11 @@ def remove_acl(zk_conn, network, rule, direction): return False, 'ERROR: Failed to write to Zookeeper! Exception: "{}".'.format(e) # Update the existing ordering - full_acl_list = getNetworkACLs(zk_conn, net_vni, direction) + updated_acl_list = getNetworkACLs(zk_conn, net_vni, direction) updated_orders = {} - for idx, acl in enumerate(full_acl_list): + for idx, acl in enumerate(updated_acl_list): updated_orders[ - '/networks/{}/firewall_rules/{}/{}/order'.format(net_vni, direction, acl) + '/networks/{}/firewall_rules/{}/{}/order'.format(net_vni, direction, acl['description']) ] = idx if updated_orders: @@ -803,9 +824,11 @@ def get_list_acl(zk_conn, network, limit, direction): return False, 'ERROR: Could not find network "{}" in the cluster!'.format(network) # Change direction to something more usable - if direction: + if direction is None: + direction = "both" + elif direction is True: direction = "in" - else: + elif direction is False: direction = "out" acl_list = [] @@ -821,12 +844,10 @@ def get_list_acl(zk_conn, network, limit, direction): except Exception as e: return False, 'Regex Error: {}'.format(e) - for idx, acl in enumerate(full_acl_list): + for acl in full_acl_list: valid_acl = False if limit: - if re.match(limit, acl) != None: - valid_acl = True - if re.match(limit, str(idx)) != None: + if re.match(limit, acl['description']) != None: valid_acl = True else: valid_acl = True diff --git a/node-daemon/pvcd.service b/node-daemon/pvcd.service index d8aa89f4..6adff3a9 100644 --- a/node-daemon/pvcd.service +++ b/node-daemon/pvcd.service @@ -10,7 +10,7 @@ Environment = PYTHONUNBUFFERED=true Environment = PVCD_CONFIG_FILE=/etc/pvc/pvcd.conf ExecStart = /usr/share/pvc/pvcd.py KillSignal = SIGINT -Restart = on-failure +Restart = never [Install] WantedBy = multi-user.target diff --git a/node-daemon/pvcd/DNSAggregatorInstance.py b/node-daemon/pvcd/DNSAggregatorInstance.py index 45b2a809..ffad35f0 100644 --- a/node-daemon/pvcd/DNSAggregatorInstance.py +++ b/node-daemon/pvcd/DNSAggregatorInstance.py @@ -124,7 +124,6 @@ class DNSAggregatorInstance(object): # Connect to the database sql_conn = sqlite3.connect(self.database_file) sql_curs = sql_conn.cursor() - print(network_domain) sql_curs.execute( 'delete from domains where name=?', (network_domain,) @@ -141,7 +140,13 @@ class DNSAggregatorInstance(object): prefix='DNS aggregator', state='o' ) - common.run_os_command('/usr/bin/pdns_control --socket-dir={} retrieve {}'.format(self.config['pdns_dynamic_directory'], self.d_network[network].domain)) + common.run_os_command( + '/usr/bin/pdns_control --socket-dir={} retrieve {}'.format( + self.config['pdns_dynamic_directory'], + self.d_network[network].domain + ), + background=True + ) # Start up the PowerDNS instance def start_aggregator(self): @@ -192,6 +197,7 @@ class DNSAggregatorInstance(object): 'Stopping PowerDNS zone aggregator', state='o' ) - self.dns_server_daemon.signal('int') - time.sleep(0.2) + # Terminate, then kill self.dns_server_daemon.signal('term') + time.sleep(0.2) + self.dns_server_daemon.signal('kill') diff --git a/node-daemon/pvcd/Daemon.py b/node-daemon/pvcd/Daemon.py index 741ba26f..db7bb294 100644 --- a/node-daemon/pvcd/Daemon.py +++ b/node-daemon/pvcd/Daemon.py @@ -484,11 +484,9 @@ include "{rulesdir}/networks/*" # Write the basic firewall config nftables_base_filename = '{}/base.nft'.format(config['nft_dynamic_directory']) -nftables_update_filename = '{}/update'.format(config['nft_dynamic_directory']) with open(nftables_base_filename, 'w') as nfbasefile: nfbasefile.write(nftables_base_rules) - # Notify a reload of the firewall rules on next keepalive update - open(nftables_update_filename, 'a').close() +common.reload_firewall_rules(logger, nftables_base_filename) ############################################################################### # PHASE 7d - Ensure DNSMASQ is not running diff --git a/node-daemon/pvcd/NodeInstance.py b/node-daemon/pvcd/NodeInstance.py index 074cfa7f..00e4860f 100644 --- a/node-daemon/pvcd/NodeInstance.py +++ b/node-daemon/pvcd/NodeInstance.py @@ -529,26 +529,6 @@ class NodeInstance(object): # Close the Libvirt connection lv_conn.close() - # Display node information to the terminal - self.logger.out('{}{} keepalive{}'.format(self.logger.fmt_purple, self.name, self.logger.fmt_end), state='t') - self.logger.out( - '{bold}Domains:{nobold} {domcount} ' - '{bold}Networks:{nobold} {netcount} ' - '{bold}VM memory [MiB]:{nobold} {allocmem} ' - '{bold}Free memory [MiB]:{nobold} {freemem} ' - '{bold}Used memory [MiB]:{nobold} {usedmem} ' - '{bold}Load:{nobold} {load}'.format( - bold=self.logger.fmt_bold, - nobold=self.logger.fmt_end, - domcount=self.domains_count, - freemem=self.memfree, - usedmem=self.memused, - load=self.cpuload, - allocmem=self.memalloc, - netcount=self.networks_count - ), - ) - # Update our local node lists for node_name in self.d_node: try: @@ -610,6 +590,26 @@ class NodeInstance(object): if node in self.inactive_node_list: secondary_node_list.remove(node) + # Display node information to the terminal + self.logger.out('{}{} keepalive{}'.format(self.logger.fmt_purple, self.name, self.logger.fmt_end), state='t') + self.logger.out( + '{bold}Domains:{nobold} {domcount} ' + '{bold}Networks:{nobold} {netcount} ' + '{bold}VM memory [MiB]:{nobold} {allocmem} ' + '{bold}Free memory [MiB]:{nobold} {freemem} ' + '{bold}Used memory [MiB]:{nobold} {usedmem} ' + '{bold}Load:{nobold} {load}'.format( + bold=self.logger.fmt_bold, + nobold=self.logger.fmt_end, + domcount=self.domains_count, + freemem=self.memfree, + usedmem=self.memused, + load=self.cpuload, + allocmem=self.memalloc, + netcount=self.networks_count + ), + ) + # Display cluster information to the terminal self.logger.out('{}Cluster status{}'.format(self.logger.fmt_purple, self.logger.fmt_end), state='t') self.logger.out('{}Primary coordinator:{} {}'.format(self.logger.fmt_bold, self.logger.fmt_end, self.primary_node)) diff --git a/node-daemon/pvcd/VXNetworkInstance.py b/node-daemon/pvcd/VXNetworkInstance.py index 85b7afbc..c18b2c15 100644 --- a/node-daemon/pvcd/VXNetworkInstance.py +++ b/node-daemon/pvcd/VXNetworkInstance.py @@ -43,8 +43,8 @@ class VXNetworkInstance(object): self.description = None self.domain = None self.ip_gateway = zkhandler.readdata(self.zk_conn, '/networks/{}/ip_gateway'.format(self.vni)) - self.ip_network = None - self.ip_cidrnetmask = None + self.ip_network = zkhandler.readdata(self.zk_conn, '/networks/{}/ip_network'.format(self.vni)) + self.ip_cidrnetmask = zkhandler.readdata(self.zk_conn, '/networks/{}/ip_network'.format(self.vni)).split('/')[-1] self.dhcp_flag = zkhandler.readdata(self.zk_conn, '/networks/{}/dhcp_flag'.format(self.vni)) self.dhcp_start = None self.dhcp_end = None @@ -52,7 +52,6 @@ class VXNetworkInstance(object): self.vxlan_nic = 'vxlan{}'.format(self.vni) self.bridge_nic = 'br{}'.format(self.vni) - self.nftables_update_filename = '{}/update'.format(config['nft_dynamic_directory']) self.nftables_netconf_filename = '{}/networks/{}.nft'.format(config['nft_dynamic_directory'], self.vni) self.firewall_rules = [] @@ -60,6 +59,30 @@ class VXNetworkInstance(object): self.dnsmasq_hostsdir = '{}/{}'.format(config['dnsmasq_dynamic_directory'], self.vni) self.dhcp_reservations = [] + self.firewall_rules_base = """# Rules for network {vxlannic} +add chain inet filter {vxlannic}-in +add chain inet filter {vxlannic}-out +add rule inet filter {vxlannic}-in counter +add rule inet filter {vxlannic}-out counter +# Jump from forward chain to this chain when matching net +add rule inet filter forward ip daddr {netaddr} counter jump {vxlannic}-in +add rule inet filter forward ip saddr {netaddr} counter jump {vxlannic}-out +# Allow ICMP traffic into the router from network +add rule inet filter input ip protocol icmp meta iifname {bridgenic} counter accept +# Allow DNS and DHCP traffic into the router from network +add rule inet filter input tcp dport 53 meta iifname {bridgenic} counter accept +add rule inet filter input udp dport 53 meta iifname {bridgenic} counter accept +add rule inet filter input udp dport 67 meta iifname {bridgenic} counter accept +# Block traffic into the router from network +add rule inet filter input meta iifname {bridgenic} counter drop +""".format( + netaddr=self.ip_network, + vxlannic=self.vxlan_nic, + bridgenic=self.bridge_nic + ) + self.firewall_rules_in = zkhandler.listchildren(self.zk_conn, '/networks/{}/firewall_rules/in'.format(self.vni)) + self.firewall_rules_out = zkhandler.listchildren(self.zk_conn, '/networks/{}/firewall_rules/out'.format(self.vni)) + # Zookeper handlers for changed states @self.zk_conn.DataWatch('/networks/{}'.format(self.vni)) def watch_network_description(data, stat, event=''): @@ -157,18 +180,29 @@ class VXNetworkInstance(object): if self.this_node.router_state == 'primary': self.updateDHCPReservations(old_reservations, new_reservations) - @self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules'.format(self.vni)) + @self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules/in'.format(self.vni)) def watch_network_firewall_rules(new_rules, event=''): if event and event.type == 'DELETED': # The key has been deleted after existing before; terminate this watcher # because this class instance is about to be reaped in Daemon.py return False - if self.firewall_rules != new_rules: - old_rules = self.firewall_rules - self.firewall_rules = new_rules - if self.this_node.router_state == 'primary': - self.updateFirewallRules(old_rules, new_rules) + # Don't run on the first pass + if self.firewall_rules_in != new_rules: + self.firewall_rules_in = new_rules + self.updateFirewallRules() + + @self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules/out'.format(self.vni)) + def watch_network_firewall_rules(new_rules, event=''): + if event and event.type == 'DELETED': + # The key has been deleted after existing before; terminate this watcher + # because this class instance is about to be reaped in Daemon.py + return False + + # Don't run on the first pass + if self.firewall_rules_out != new_rules: + self.firewall_rules_out = new_rules + self.updateFirewallRules() self.createNetwork() self.createFirewall() @@ -203,17 +237,47 @@ class VXNetworkInstance(object): except: pass - def updateFirewallRules(self, old_rules_list, new_rules_list): - for rule in new_rules_list: - if rule not in old_rules_list: - # Add new rule entry - print(rule) - pass + def updateFirewallRules(self): + self.logger.out( + 'Updating firewall rules', + prefix='VNI {}'.format(self.vni), + state='o' + ) + ordered_acls_in = {} + ordered_acls_out = {} + sorted_acl_list = {'in': [], 'out': []} + full_ordered_rules = [] - for rule in old_rules_list: - if rule not in new_rules_list: - print(rule) - pass + for acl in self.firewall_rules_in: + order = zkhandler.readdata(self.zk_conn, '/networks/{}/firewall_rules/in/{}/order'.format(self.vni, acl)) + ordered_acls_in[order] = acl + for acl in self.firewall_rules_out: + order = zkhandler.readdata(self.zk_conn, '/networks/{}/firewall_rules/out/{}/order'.format(self.vni, acl)) + ordered_acls_out[order] = acl + + for order in sorted(ordered_acls_in.keys()): + sorted_acl_list['in'].append(ordered_acls_in[order]) + for order in sorted(ordered_acls_out.keys()): + sorted_acl_list['out'].append(ordered_acls_out[order]) + + for direction in 'in', 'out': + for acl in sorted_acl_list[direction]: + rule_prefix = "add rule inet filter vxlan{}-{} counter".format(self.vni, direction) + rule_data = zkhandler.readdata(self.zk_conn, '/networks/{}/firewall_rules/{}/{}/rule'.format(self.vni, direction, acl)) + rule = '{} {}'.format(rule_prefix, rule_data) + full_ordered_rules.append(rule) + + output = "{}\n# User rules\n{}\n".format( + self.firewall_rules_base, + '\n'.join(full_ordered_rules) + ) + + with open(self.nftables_netconf_filename, 'w') as nfnetfile: + nfnetfile.write(dedent(output)) + + # Reload firewall rules + nftables_base_filename = '{}/base.nft'.format(self.config['nft_dynamic_directory']) + common.reload_firewall_rules(self.logger, nftables_base_filename) def createNetwork(self): self.logger.out( @@ -253,31 +317,8 @@ class VXNetworkInstance(object): ) def createFirewall(self): - nftables_network_rules = """# Rules for network {vxlannic} -add chain inet filter {vxlannic}-in -add chain inet filter {vxlannic}-out -add rule inet filter {vxlannic}-in counter -add rule inet filter {vxlannic}-out counter -# Jump from forward chain to this chain when matching net -add rule inet filter forward ip daddr {netaddr} counter jump {vxlannic}-in -add rule inet filter forward ip saddr {netaddr} counter jump {vxlannic}-out -# Allow ICMP traffic into the router from network -add rule inet filter input ip protocol icmp meta iifname {bridgenic} counter accept -# Allow DNS and DHCP traffic into the router from network -add rule inet filter input tcp dport 53 meta iifname {bridgenic} counter accept -add rule inet filter input udp dport 53 meta iifname {bridgenic} counter accept -add rule inet filter input udp dport 67 meta iifname {bridgenic} counter accept -# Block traffic into the router from network -add rule inet filter input meta iifname {bridgenic} counter drop -""".format( - netaddr=self.ip_network, - vxlannic=self.vxlan_nic, - bridgenic=self.bridge_nic - ) - with open(self.nftables_netconf_filename, 'w') as nfbasefile: - nfbasefile.write(dedent(nftables_network_rules)) - open(self.nftables_update_filename, 'a').close() - pass + # For future use + self.updateFirewallRules() def createGatewayAddress(self): if self.this_node.router_state == 'primary': @@ -342,11 +383,10 @@ add rule inet filter input meta iifname {bridgenic} counter drop '--listen-address={}'.format(self.ip_gateway), '--bind-interfaces', '--leasefile-ro', - '--dhcp-script=./pvcd/dnsmasq-zookeeper-leases.py', + '--dhcp-script={}/pvcd/dnsmasq-zookeeper-leases.py'.format(os.getcwd()), '--dhcp-range={},{},48h'.format(self.dhcp_start, self.dhcp_end), '--dhcp-hostsdir={}'.format(self.dnsmasq_hostsdir), '--log-facility=-', - '--log-queries=extra', '--keep-in-foreground' ] # Start the dnsmasq process in a thread @@ -394,9 +434,20 @@ add rule inet filter input meta iifname {bridgenic} counter drop ) def removeFirewall(self): - os.remove(self.nftables_netconf_filename) - open(self.nftables_update_filename, 'a').close() - pass + self.logger.out( + 'Removing firewall rules', + prefix='VNI {}'.format(self.vni), + state='o' + ) + + try: + os.remove(self.nftables_netconf_filename) + except: + pass + + # Reload firewall rules + nftables_base_filename = '{}/base.nft'.format(self.config['nft_dynamic_directory']) + common.reload_firewall_rules(self.logger, nftables_base_filename) def removeGatewayAddress(self): self.logger.out( diff --git a/node-daemon/pvcd/common.py b/node-daemon/pvcd/common.py index 7b217d5d..ebf63ce4 100644 --- a/node-daemon/pvcd/common.py +++ b/node-daemon/pvcd/common.py @@ -83,9 +83,8 @@ def run_os_command(command_string, background=False, environment=None): return command_output.returncode, command_output.stdout.decode('ascii'), command_output.stderr.decode('ascii') # Reload the firewall rules of the system -def reload_firewall_rules(rules_dir): - log.echo('Updating firewall rules', '', 'o') - rules_file = '{}/base.nft'.format(rules_dir) +def reload_firewall_rules(logger, rules_file): + logger.out('Reloading firewall configuration', state='o') retcode, stdout, stderr = run_os_command('/usr/sbin/nft -f {}'.format(rules_file)) if retcode != 0: - log.echo('Failed to reload rules: {}'.format(stderr), '', 'e') + logger.out('Failed to reload configuration: {}'.format(stderr), state='e')