diff --git a/client-common/network.py b/client-common/network.py index d2761daf..543e947e 100644 --- a/client-common/network.py +++ b/client-common/network.py @@ -101,14 +101,14 @@ def getNetworkDescription(zk_conn, network): return net_description def getNetworkDHCPLeases(zk_conn, vni): - # Get a list of DHCP leases by listing the children of /networks//dhcp_leases - dhcp_leases = zkhandler.listchildren(zk_conn, '/networks/{}/dhcp_leases'.format(vni)) - return sorted(dhcp_leases) + # Get a list of DHCP leases by listing the children of /networks//dhcp4_leases + dhcp4_leases = zkhandler.listchildren(zk_conn, '/networks/{}/dhcp4_leases'.format(vni)) + return sorted(dhcp4_leases) def getNetworkDHCPReservations(zk_conn, vni): - # Get a list of DHCP reservations by listing the children of /networks//dhcp_reservations - dhcp_reservations = zkhandler.listchildren(zk_conn, '/networks/{}/dhcp_reservations'.format(vni)) - return sorted(dhcp_reservations) + # Get a list of DHCP reservations by listing the children of /networks//dhcp4_reservations + dhcp4_reservations = zkhandler.listchildren(zk_conn, '/networks/{}/dhcp4_reservations'.format(vni)) + return sorted(dhcp4_reservations) def getNetworkACLs(zk_conn, vni, _direction): # Get the (sorted) list of active ACLs @@ -144,17 +144,17 @@ def getNetworkInformation(zk_conn, vni): return description, domain, ip6_network, ip6_gateway, dhcp6_flag, ip4_network, ip4_gateway, dhcp4_flag, dhcp4_start, dhcp4_end def getDHCPLeaseInformation(zk_conn, vni, mac_address): - hostname = zkhandler.readdata(zk_conn, '/networks/{}/dhcp_leases/{}/hostname'.format(vni, mac_address)) - ip4_address = zkhandler.readdata(zk_conn, '/networks/{}/dhcp_leases/{}/ipaddr'.format(vni, mac_address)) + hostname = zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_leases/{}/hostname'.format(vni, mac_address)) + ip4_address = zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_leases/{}/ipaddr'.format(vni, mac_address)) try: - timestamp = zkhandler.readdata(zk_conn, '/networks/{}/dhcp_leases/{}/expiry'.format(vni, mac_address)) + timestamp = zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_leases/{}/expiry'.format(vni, mac_address)) except: timestamp = 'static' return hostname, ip4_address, mac_address, timestamp def getDHCPReservationInformation(zk_conn, vni, mac_address): - hostname = zkhandler.readdata(zk_conn, '/networks/{}/dhcp_reservations/{}/hostname'.format(vni, mac_address)) - ip4_address = zkhandler.readdata(zk_conn, '/networks/{}/dhcp_reservations/{}/ipaddr'.format(vni, mac_address)) + hostname = zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_reservations/{}/hostname'.format(vni, mac_address)) + ip4_address = zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_reservations/{}/ipaddr'.format(vni, mac_address)) timestamp = 'static' return hostname, ip4_address, mac_address, timestamp @@ -360,7 +360,7 @@ def formatNetworkList(zk_conn, net_list): output_string = net_list_output_header + '\n' + '\n'.join(sorted(net_list_output)) return output_string -def formatDHCPLeaseList(zk_conn, vni, dhcp_leases_list, reservations=False): +def formatDHCPLeaseList(zk_conn, vni, dhcp4_leases_list, reservations=False): dhcp_lease_list_output = [] hostname = dict() ip4_address = dict() @@ -368,7 +368,7 @@ def formatDHCPLeaseList(zk_conn, vni, dhcp_leases_list, reservations=False): timestamp = dict() # Gather information for printing - for dhcp_lease in dhcp_leases_list: + for dhcp_lease in dhcp4_leases_list: if reservations: hostname[dhcp_lease], ip4_address[dhcp_lease], mac_address[dhcp_lease], timestamp[dhcp_lease] = getDHCPReservationInformation(zk_conn, vni, dhcp_lease) else: @@ -379,7 +379,7 @@ def formatDHCPLeaseList(zk_conn, vni, dhcp_leases_list, reservations=False): lease_ip4_address_length = 11 lease_mac_address_length = 13 lease_timestamp_length = 13 - for dhcp_lease in dhcp_leases_list: + for dhcp_lease in dhcp4_leases_list: # hostname column _lease_hostname_length = len(hostname[dhcp_lease]) + 1 if _lease_hostname_length > lease_hostname_length: @@ -412,7 +412,7 @@ def formatDHCPLeaseList(zk_conn, vni, dhcp_leases_list, reservations=False): lease_timestamp='Timestamp' ) - for dhcp_lease in dhcp_leases_list: + for dhcp_lease in dhcp4_leases_list: dhcp_lease_list_output.append('{bold}\ {lease_hostname: <{lease_hostname_length}} \ {lease_ip4_address: <{lease_ip4_address_length}} \ @@ -655,15 +655,15 @@ def add_dhcp_reservation(zk_conn, network, ipaddress, macaddress, hostname): if not isValidIP(ipaddress): return False, 'ERROR: IP address "{}" is not valid!'.format(macaddress) - if zkhandler.exists(zk_conn, '/networks/{}/dhcp_reservations/{}'.format(net_vni, macaddress)): + if zkhandler.exists(zk_conn, '/networks/{}/dhcp4_reservations/{}'.format(net_vni, macaddress)): return False, 'ERROR: A reservation with MAC "{}" already exists!'.format(macaddress) # Add the new static lease to ZK try: zkhandler.writedata(zk_conn, { - '/networks/{}/dhcp_reservations/{}'.format(net_vni, macaddress): 'static', - '/networks/{}/dhcp_reservations/{}/hostname'.format(net_vni, macaddress): hostname, - '/networks/{}/dhcp_reservations/{}/ipaddr'.format(net_vni, macaddress): ipaddress + '/networks/{}/dhcp4_reservations/{}'.format(net_vni, macaddress): 'static', + '/networks/{}/dhcp4_reservations/{}/hostname'.format(net_vni, macaddress): hostname, + '/networks/{}/dhcp4_reservations/{}/ipaddr'.format(net_vni, macaddress): ipaddress }) except Exception as e: return False, 'ERROR: Failed to write to Zookeeper! Exception: "{}".'.format(e) @@ -679,10 +679,10 @@ def remove_dhcp_reservation(zk_conn, network, reservation): match_description = '' # Check if the reservation matches a description, a mac, or an IP address currently in the database - dhcp_reservations_list = getNetworkDHCPReservations(zk_conn, net_vni) - for macaddr in dhcp_reservations_list: - hostname = zkhandler.readdata(zk_conn, '/networks/{}/dhcp_reservations/{}/hostname'.format(net_vni, macaddr)) - ipaddress = zkhandler.readdata(zk_conn, '/networks/{}/dhcp_reservations/{}/ipaddr'.format(net_vni, macaddr)) + dhcp4_reservations_list = getNetworkDHCPReservations(zk_conn, net_vni) + for macaddr in dhcp4_reservations_list: + hostname = zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_reservations/{}/hostname'.format(net_vni, macaddr)) + ipaddress = zkhandler.readdata(zk_conn, '/networks/{}/dhcp4_reservations/{}/ipaddr'.format(net_vni, macaddr)) if reservation == macaddr or reservation == hostname or reservation == ipaddress: match_description = macaddr @@ -691,7 +691,7 @@ def remove_dhcp_reservation(zk_conn, network, reservation): # Remove the entry from zookeeper try: - zkhandler.deletekey(zk_conn, '/networks/{}/dhcp_reservations/{}'.format(net_vni, match_description)) + zkhandler.deletekey(zk_conn, '/networks/{}/dhcp4_reservations/{}'.format(net_vni, match_description)) except: return False, 'ERROR: Failed to write to Zookeeper!' diff --git a/node-daemon/pvcd/Daemon.py b/node-daemon/pvcd/Daemon.py index 23025948..0aebbbdc 100644 --- a/node-daemon/pvcd/Daemon.py +++ b/node-daemon/pvcd/Daemon.py @@ -71,11 +71,13 @@ import pvcd.CephInstance as CephInstance # Create timer to update this node in Zookeeper def startKeepaliveTimer(): - global update_timer + # Create our timer object + update_timer = apscheduler.schedulers.background.BackgroundScheduler() interval = int(config['keepalive_interval']) logger.out('Starting keepalive timer ({} second interval)'.format(interval), state='s') update_timer.add_job(update_zookeeper, 'interval', seconds=interval) update_timer.start() + return update_timer def stopKeepaliveTimer(): global update_timer @@ -111,9 +113,6 @@ staticdata.append(subprocess.run(['uname', '-r'], stdout=subprocess.PIPE).stdout staticdata.append(subprocess.run(['uname', '-o'], stdout=subprocess.PIPE).stdout.decode('ascii').strip()) staticdata.append(subprocess.run(['uname', '-m'], stdout=subprocess.PIPE).stdout.decode('ascii').strip()) -# Create our timer object -update_timer = apscheduler.schedulers.background.BackgroundScheduler() - # Config values dictionary config_values = [ 'coordinators', @@ -288,13 +287,19 @@ except Exception as e: # Handle zookeeper failures def zk_listener(state): global zk_conn, update_timer - if state == kazoo.client.KazooState.SUSPENDED: - logger.out('Connection to Zookeeper lost; retrying', state='w') + if state == kazoo.client.KazooState.CONNECTED: + logger.out('Connection to Zookeeper restarted', state='o') + # Start keepalive thread + if update_timer: + update_timer = startKeepaliveTimer() + else: # Stop keepalive thread if update_timer: stopKeepaliveTimer() + logger.out('Connection to Zookeeper lost; retrying', state='w') + while True: _zk_conn = kazoo.client.KazooClient(hosts=config['coordinators']) try: @@ -303,14 +308,6 @@ def zk_listener(state): break except: time.sleep(1) - elif state == kazoo.client.KazooState.CONNECTED: - logger.out('Connection to Zookeeper restarted', state='o') - - # Start keepalive thread - if update_timer: - update_timer = startKeepaliveTimer() - else: - pass zk_conn.add_listener(zk_listener) ############################################################################### @@ -422,14 +419,14 @@ else: vni_dev = config['vni_dev'] vni_dev_ip = config['vni_dev_ip'] logger.out('Setting up VNI network on interface {} with IP {}'.format(vni_dev, vni_dev_ip), state='i') -common.run_os_command('ip link set {} up'.format(vni_dev)) +common.run_os_command('ip link set {} mtu 9000 up'.format(vni_dev)) common.run_os_command('ip address add {} dev {}'.format(vni_dev_ip, vni_dev)) # Storage configuration storage_dev = config['storage_dev'] storage_dev_ip = config['storage_dev_ip'] logger.out('Setting up Storage network on interface {} with IP {}'.format(storage_dev, storage_dev_ip), state='i') -common.run_os_command('ip link set {} up'.format(storage_dev)) +common.run_os_command('ip link set {} mtu 9000 up'.format(storage_dev)) common.run_os_command('ip address add {} dev {}'.format(storage_dev_ip, storage_dev)) # Upstream configuration @@ -899,6 +896,11 @@ def update_zookeeper(): else: ceph_health_colour = logger.fmt_red + # DNS aggregator retransfer + if this_node.router_state == 'primary': + for network in d_network: + dns_aggregator.get_axfr(network) + # Set ceph health information in zookeeper (primary only) if this_node.router_state == 'primary': # Get status info @@ -1145,7 +1147,7 @@ def update_zookeeper(): # Start keepalive thread and immediately update Zookeeper -startKeepaliveTimer() +update_timer = startKeepaliveTimer() update_zookeeper() # Tick loop; does nothing since everything else is async diff --git a/node-daemon/pvcd/VXNetworkInstance.py b/node-daemon/pvcd/VXNetworkInstance.py index 33ebb047..355942cc 100644 --- a/node-daemon/pvcd/VXNetworkInstance.py +++ b/node-daemon/pvcd/VXNetworkInstance.py @@ -77,11 +77,13 @@ add rule inet filter {vxlannic}-in counter add rule inet filter {vxlannic}-out counter # Allow ICMP traffic into the router from network add rule inet filter input ip protocol icmp meta iifname {bridgenic} counter accept +add rule inet filter input ip6 nexthdr icmpv6 meta iifname {bridgenic} counter accept # Allow DNS, DHCP, and NTP traffic into the router from network add rule inet filter input tcp dport 53 meta iifname {bridgenic} counter accept add rule inet filter input udp dport 53 meta iifname {bridgenic} counter accept add rule inet filter input udp dport 67 meta iifname {bridgenic} counter accept add rule inet filter input udp dport 123 meta iifname {bridgenic} counter accept +add rule inet filter input ip6 nexthdr udp udp dport 547 meta iifname {bridgenic} counter accept # Block traffic into the router from network add rule inet filter input meta iifname {bridgenic} counter drop """.format( @@ -151,11 +153,15 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out if data and self.ip6_gateway != data.decode('ascii'): orig_gateway = self.ip6_gateway - self.ip6_gateway = data.decode('ascii') if self.this_node.router_state == 'primary': if orig_gateway: self.removeGateway6Address() + self.ip6_gateway = data.decode('ascii') + if self.this_node.router_state == 'primary': self.createGateway6Address() + if self.dhcp_server_daemon: + self.stopDHCPServer() + self.startDHCPServer() @self.zk_conn.DataWatch('/networks/{}/dhcp6_flag'.format(self.vni)) def watch_network_dhcp_status(data, stat, event=''): @@ -192,11 +198,15 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out if data and self.ip4_gateway != data.decode('ascii'): orig_gateway = self.ip4_gateway - self.ip4_gateway = data.decode('ascii') if self.this_node.router_state == 'primary': if orig_gateway: self.removeGateway4Address() + self.ip4_gateway = data.decode('ascii') + if self.this_node.router_state == 'primary': self.createGateway4Address() + if self.dhcp_server_daemon: + self.stopDHCPServer() + self.startDHCPServer() @self.zk_conn.DataWatch('/networks/{}/dhcp4_flag'.format(self.vni)) def watch_network_dhcp_status(data, stat, event=''): @@ -380,12 +390,12 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out ) ) common.run_os_command( - 'ip link set {} up'.format( + 'ip link set {} mtu 8800 up'.format( self.vxlan_nic ) ) common.run_os_command( - 'ip link set {} up'.format( + 'ip link set {} mtu 8800 up'.format( self.bridge_nic ) ) @@ -435,13 +445,15 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out prefix='VNI {}'.format(self.vni), state='o' ) + # Recreate the environment we need for dnsmasq pvcd_config_file = os.environ['PVCD_CONFIG_FILE'] dhcp_environment = { 'DNSMASQ_BRIDGE_INTERFACE': self.bridge_nic, 'PVCD_CONFIG_FILE': pvcd_config_file } - # Define the dnsmasq config + + # Define the dnsmasq config fragments dhcp_configuration_base = [ '--domain-needed', '--bogus-priv', @@ -454,9 +466,10 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out '--local=/{}/'.format(self.domain), '--auth-zone={}'.format(self.domain), '--log-facility=-', + '--log-dhcp', '--keep-in-foreground', '--leasefile-ro', - '--auth-soa=1,pvc@localhost,10,10', +# '--auth-soa=1,pvc@localhost,10,10', '--dhcp-script={}/pvcd/dnsmasq-zookeeper-leases.py'.format(os.getcwd()), '--dhcp-hostsdir={}'.format(self.dnsmasq_hostsdir), '--bind-interfaces', @@ -474,15 +487,28 @@ add rule inet filter forward ip6 saddr {netaddr6} counter jump {vxlannic}-out '--listen-address={}'.format(self.ip6_gateway), '--auth-peer={}'.format(self.ip6_gateway), '--auth-sec-servers={}'.format(self.ip6_gateway), - '--dhcp-option=option6:ntp-server,{}'.format(self.ip6_gateway), + '--dhcp-option=option6:dns-server,{}'.format(self.ip6_gateway), + '--dhcp-option=option6:sntp-server,{}'.format(self.ip6_gateway), + ] + dhcp_configuration_v6_dualstack = [ '--dhcp-range=net:{nic},::,constructor:{nic},ra-stateless,ra-names'.format(nic=self.bridge_nic), '--enable-ra', ] + dhcp_configuration_v6_only = [ + '--dhcp-range=net:{nic},::2,::ffff:ffff:ffff:ffff,constructor:{nic},64,24h'.format(nic=self.bridge_nic), + ] + + # Assemble the DHCP configuration dhcp_configuration = dhcp_configuration_base if self.dhcp4_flag: dhcp_configuration += dhcp_configuration_v4 if self.dhcp6_flag: dhcp_configuration += dhcp_configuration_v6 + if self.dhcp4_flag: + dhcp_configuration += dhcp_configuration_v6_dualstack + else: + dhcp_configuration += dhcp_configuration_v6_only + # Start the dnsmasq process in a thread print('/usr/sbin/dnsmasq {}'.format(' '.join(dhcp_configuration))) self.dhcp_server_daemon = common.run_os_daemon(