diff --git a/node-daemon/pvcd.conf.sample b/node-daemon/pvcd.conf.sample index 70b950e7..d428db17 100644 --- a/node-daemon/pvcd.conf.sample +++ b/node-daemon/pvcd.conf.sample @@ -31,6 +31,13 @@ # flush action; can be "mem", "load", "vcpus", or "vms" (defaults # to "mem"); the best choice based on this field is selected for # each VM to be migrated +# vni_floating_ip: the IP address (in CIDR format) for the floating IP on the VNI network, +# used to provide a consistent view of the dynamic primary node to other +# machines in the VNI network, e.g. for slaving DNS or sending in routes. +# upstream_floating_ip: the IP address (in CIDR format) for the floating IP on the upstream +# network, used to provide a consistent view of the dynamic primary +# node to machines in the upstream network, e.g. for slaving DNS or +# sending in routes. # The following values are required for each node specifically (usually node-unique): # vni_dev: the lower-level network device to bind VNI traffic to # vni_dev_ip: the IP address (in CIDR format) of the lower-level network device, used by frr @@ -58,6 +65,8 @@ suicide_intervals = 0 successful_fence = migrate failed_fence = none migration_target_selector = mem +vni_floating_ip = 10.255.0.254/24 +upstream_floating_ip = 10.101.0.30/24 [pvc-hv1] vni_dev = ens4 diff --git a/node-daemon/pvcd/DNSAggregatorInstance.py b/node-daemon/pvcd/DNSAggregatorInstance.py index 827c9bc1..45b2a809 100644 --- a/node-daemon/pvcd/DNSAggregatorInstance.py +++ b/node-daemon/pvcd/DNSAggregatorInstance.py @@ -40,7 +40,12 @@ class DNSAggregatorInstance(object): self.logger = logger self.d_network = d_network - self.active = False + # Floating upstreams + self.vni_dev = self.config['vni_dev'] + self.vni_ipaddr, self.vni_cidrnetmask = self.config['vni_floating_ip'].split('/') + self.upstream_dev = self.config['upstream_dev'] + self.upstream_ipaddr, self.upstream_cidrnetmask = self.config['upstream_floating_ip'].split('/') + self.database_file = self.config['pdns_dynamic_directory'] + '/pdns-aggregator.sqlite3' self.dns_server_daemon = None @@ -98,7 +103,7 @@ class DNSAggregatorInstance(object): if write_domain: sql_curs.execute( - 'insert into domains (name, master, type, account) values (?, ?, "SLAVE", "internal")', + 'insert into domains (name, master, type, account) values (?, ?, "MASTER", "internal")', (network_domain, network_gateway) ) sql_conn.commit() @@ -146,22 +151,30 @@ class DNSAggregatorInstance(object): ) # Define the PowerDNS config dns_configuration = [ + # Option # Explanation '--no-config', - '--daemon=no', - '--disable-syslog=yes', - '--disable-axfr=no', - '--guardian=yes', - '--local-address=0.0.0.0', - '--local-port=10053', - '--log-dns-details=on', - '--loglevel=3', - '--master=no', - '--slave=yes', - '--version-string=powerdns', + '--daemon=no', # Start directly + '--guardian=yes', # Use a guardian + '--disable-syslog=yes', # Log only to stdout (which is then captured) + '--disable-axfr=no', # Allow AXFRs + '--allow-axfr-ips=0.0.0.0/0', # Allow AXFRs to anywhere + '--also-notify=10.101.0.60', # Notify upstreams + '--local-address={},{}'.format(self.vni_ipaddr, self.upstream_ipaddr), + # Listen on floating IPs + '--local-port=10053', # On port 10053 + '--log-dns-details=on', # Log details + '--loglevel=3', # Log info + '--master=yes', # Enable master mode + '--slave=yes', # Enable slave mode + '--slave-renotify=yes', # Renotify out for our slaved zones + '--version-string=powerdns', # Set the version string + '--default-soa-name=dns.pvc.local', # Override dnsmasq's invalid name '--socket-dir={}'.format(self.config['pdns_dynamic_directory']), - '--launch=gsqlite3', + # Standard socket directory + '--launch=gsqlite3', # Use the sqlite3 backend '--gsqlite3-database={}'.format(self.database_file), - '--gsqlite3-dnssec=no' + # Database file + '--gsqlite3-dnssec=no' # Don't do DNSSEC here ] # Start the pdns process in a thread self.dns_server_daemon = common.run_os_daemon( diff --git a/node-daemon/pvcd/Daemon.py b/node-daemon/pvcd/Daemon.py index b3806ad0..741ba26f 100644 --- a/node-daemon/pvcd/Daemon.py +++ b/node-daemon/pvcd/Daemon.py @@ -123,10 +123,12 @@ config_values = [ 'migration_target_selector', 'vni_dev', 'vni_dev_ip', + 'vni_floating_ip', 'storage_dev', 'storage_dev_ip', 'upstream_dev', 'upstream_dev_ip', + 'upstream_floating_ip', 'ipmi_hostname', 'ipmi_username', 'ipmi_password' @@ -549,9 +551,9 @@ def update_networks(new_network_list): for network in new_network_list: if not network in network_list: d_network[network] = VXNetworkInstance.VXNetworkInstance(network, zk_conn, config, logger, this_node) - dns_aggregator.add_client_network(network) # Start primary functionality if this_node.router_state == 'primary': + dns_aggregator.add_client_network(network) d_network[network].createGatewayAddress() d_network[network].startDHCPServer() @@ -562,10 +564,10 @@ def update_networks(new_network_list): if this_node.router_state == 'primary': d_network[network].stopDHCPServer() d_network[network].removeGatewayAddress() + dns_aggregator.remove_client_network(network) # Stop general functionality d_network[network].removeFirewall() d_network[network].removeNetwork() - dns_aggregator.remove_client_network(network) # Delete the object del(d_network[network]) diff --git a/node-daemon/pvcd/NodeInstance.py b/node-daemon/pvcd/NodeInstance.py index ec5deecc..074cfa7f 100644 --- a/node-daemon/pvcd/NodeInstance.py +++ b/node-daemon/pvcd/NodeInstance.py @@ -69,6 +69,11 @@ class NodeInstance(object): self.memfree = 0 self.memalloc = 0 self.vcpualloc = 0 + # Floating upstreams + self.vni_dev = self.config['vni_dev'] + self.vni_ipaddr, self.vni_cidrnetmask = self.config['vni_floating_ip'].split('/') + self.upstream_dev = self.config['upstream_dev'] + self.upstream_ipaddr, self.upstream_cidrnetmask = self.config['upstream_floating_ip'].split('/') # Flags self.inflush = False @@ -295,10 +300,12 @@ class NodeInstance(object): self.d_network[network].stopDHCPServer() self.d_network[network].removeGatewayAddress() self.dns_aggregator.stop_aggregator() + self.removeFloatingAddresses() def become_primary(self): self.logger.out('Setting router {} to primary state.'.format(self.name), state='i') self.logger.out('Network list: {}'.format(', '.join(self.network_list))) + self.createFloatingAddresses() self.dns_aggregator.start_aggregator() time.sleep(0.5) # Start up the gateways and DHCP servers @@ -310,6 +317,88 @@ class NodeInstance(object): for network in self.d_network: self.dns_aggregator.get_axfr(network) + def createFloatingAddresses(self): + # VNI floating IP + self.logger.out( + 'Creating floating management IP {}/{} on interface {}'.format( + self.vni_ipaddr, + self.vni_cidrnetmask, + self.vni_dev + ), + state='o' + ) + common.run_os_command( + 'ip address add {}/{} dev {}'.format( + self.vni_ipaddr, + self.vni_cidrnetmask, + self.vni_dev + ) + ) + common.run_os_command( + 'arping -A -c2 -I {} {}'.format( + self.vni_dev, + self.vni_ipaddr + ), + background=True + ) + # Upstream floating IP + self.logger.out( + 'Creating floating upstream IP {}/{} on interface {}'.format( + self.upstream_ipaddr, + self.upstream_cidrnetmask, + self.upstream_dev + ), + state='o' + ) + common.run_os_command( + 'ip address add {}/{} dev {}'.format( + self.upstream_ipaddr, + self.upstream_cidrnetmask, + self.upstream_dev + ) + ) + common.run_os_command( + 'arping -A -c2 -I {} {}'.format( + self.upstream_dev, + self.upstream_ipaddr + ), + background=True + ) + + def removeFloatingAddresses(self): + # VNI floating IP + self.logger.out( + 'Removing floating management IP {}/{} from interface {}'.format( + self.vni_ipaddr, + self.vni_cidrnetmask, + self.vni_dev + ), + state='o' + ) + common.run_os_command( + 'ip address delete {}/{} dev {}'.format( + self.vni_ipaddr, + self.vni_cidrnetmask, + self.vni_dev + ) + ) + # Upstream floating IP + self.logger.out( + 'Removing floating upstream IP {}/{} from interface {}'.format( + self.upstream_ipaddr, + self.upstream_cidrnetmask, + self.upstream_dev + ), + state='o' + ) + common.run_os_command( + 'ip address delete {}/{} dev {}'.format( + self.upstream_ipaddr, + self.upstream_cidrnetmask, + self.upstream_dev + ) + ) + # Flush all VMs on the host def flush(self): self.inflush = True diff --git a/node-daemon/pvcd/VXNetworkInstance.py b/node-daemon/pvcd/VXNetworkInstance.py index 35663e75..85b7afbc 100644 --- a/node-daemon/pvcd/VXNetworkInstance.py +++ b/node-daemon/pvcd/VXNetworkInstance.py @@ -22,6 +22,7 @@ import os import sys +import time from textwrap import dedent import pvcd.log as log @@ -153,7 +154,8 @@ class VXNetworkInstance(object): if self.dhcp_reservations != new_reservations: old_reservations = self.dhcp_reservations self.dhcp_reservations = new_reservations - self.updateDHCPReservations(old_reservations, new_reservations) + if self.this_node.router_state == 'primary': + self.updateDHCPReservations(old_reservations, new_reservations) @self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules'.format(self.vni)) def watch_network_firewall_rules(new_rules, event=''): @@ -165,7 +167,8 @@ class VXNetworkInstance(object): if self.firewall_rules != new_rules: old_rules = self.firewall_rules self.firewall_rules = new_rules - self.updateFirewallRules(old_rules, new_rules) + if self.this_node.router_state == 'primary': + self.updateFirewallRules(old_rules, new_rules) self.createNetwork() self.createFirewall() @@ -204,10 +207,12 @@ class VXNetworkInstance(object): for rule in new_rules_list: if rule not in old_rules_list: # Add new rule entry + print(rule) pass for rule in old_rules_list: if rule not in new_rules_list: + print(rule) pass def createNetwork(self): @@ -337,7 +342,7 @@ add rule inet filter input meta iifname {bridgenic} counter drop '--listen-address={}'.format(self.ip_gateway), '--bind-interfaces', '--leasefile-ro', - '--dhcp-script=/usr/share/pvc/pvcd/dnsmasq-zookeeper-leases.py', + '--dhcp-script=./pvcd/dnsmasq-zookeeper-leases.py', '--dhcp-range={},{},48h'.format(self.dhcp_start, self.dhcp_end), '--dhcp-hostsdir={}'.format(self.dnsmasq_hostsdir), '--log-facility=-', @@ -419,6 +424,7 @@ add rule inet filter input meta iifname {bridgenic} counter drop prefix='VNI {}'.format(self.vni), state='o' ) - self.dhcp_server_daemon.signal('int') - time.sleep(0.2) + # Terminate, then kill self.dhcp_server_daemon.signal('term') + time.sleep(0.2) + self.dhcp_server_daemon.signal('kill') diff --git a/node-daemon/pvcd/common.py b/node-daemon/pvcd/common.py index 05478fc4..7b217d5d 100644 --- a/node-daemon/pvcd/common.py +++ b/node-daemon/pvcd/common.py @@ -50,7 +50,8 @@ class OSDaemon(object): signal_map = { 'hup': signal.SIGHUP, 'int': signal.SIGINT, - 'term': signal.SIGTERM + 'term': signal.SIGTERM, + 'kill': signal.SIGKILL } self.proc.send_signal(signal_map[sent_signal])