Add floating IPs and better termination of daemons

node-daemon/pvcd.conf.sample

@@ -31,6 +31,13 @@
 #                               flush action; can be "mem", "load", "vcpus", or "vms" (defaults
 #                               to "mem"); the best choice based on this field is selected for
 #                               each VM to be migrated
+#    vni_floating_ip: the IP address (in CIDR format) for the floating IP on the VNI network,
+#                     used to provide a consistent view of the dynamic primary node to other
+#                     machines in the VNI network, e.g. for slaving DNS or sending in routes.
+#    upstream_floating_ip: the IP address (in CIDR format) for the floating IP on the upstream
+#                          network, used to provide a consistent view of the dynamic primary
+#                          node to machines in the upstream network, e.g. for slaving DNS or
+#                          sending in routes.
 # The following values are required for each node specifically (usually node-unique):
 #    vni_dev: the lower-level network device to bind VNI traffic to
 #    vni_dev_ip: the IP address (in CIDR format) of the lower-level network device, used by frr
@@ -58,6 +65,8 @@ suicide_intervals = 0
 successful_fence = migrate
 failed_fence = none
 migration_target_selector = mem
+vni_floating_ip = 10.255.0.254/24
+upstream_floating_ip = 10.101.0.30/24
 
 [pvc-hv1]
 vni_dev = ens4

node-daemon/pvcd/DNSAggregatorInstance.py

@@ -40,7 +40,12 @@ class DNSAggregatorInstance(object):
         self.logger = logger
         self.d_network = d_network
 
-        self.active = False
+        # Floating upstreams
+        self.vni_dev = self.config['vni_dev']
+        self.vni_ipaddr, self.vni_cidrnetmask = self.config['vni_floating_ip'].split('/')
+        self.upstream_dev = self.config['upstream_dev']
+        self.upstream_ipaddr, self.upstream_cidrnetmask = self.config['upstream_floating_ip'].split('/')
+
         self.database_file = self.config['pdns_dynamic_directory'] + '/pdns-aggregator.sqlite3'
 
         self.dns_server_daemon = None
@@ -98,7 +103,7 @@ class DNSAggregatorInstance(object):
 
         if write_domain:
             sql_curs.execute(
-                'insert into domains (name, master, type, account) values (?, ?, "SLAVE", "internal")',
+                'insert into domains (name, master, type, account) values (?, ?, "MASTER", "internal")',
                 (network_domain, network_gateway)
             )
             sql_conn.commit()
@@ -146,22 +151,30 @@ class DNSAggregatorInstance(object):
         )
         # Define the PowerDNS config
         dns_configuration = [
+            # Option                            # Explanation
             '--no-config',
-            '--daemon=no',
+            '--daemon=no',                      # Start directly
-            '--disable-syslog=yes',
+            '--guardian=yes',                   # Use a guardian
-            '--disable-axfr=no',
+            '--disable-syslog=yes',             # Log only to stdout (which is then captured)
-            '--guardian=yes',
+            '--disable-axfr=no',                # Allow AXFRs
-            '--local-address=0.0.0.0',
+            '--allow-axfr-ips=0.0.0.0/0',       # Allow AXFRs to anywhere
-            '--local-port=10053',
+            '--also-notify=10.101.0.60',        # Notify upstreams
-            '--log-dns-details=on',
+            '--local-address={},{}'.format(self.vni_ipaddr, self.upstream_ipaddr),
-            '--loglevel=3',
+                                                # Listen on floating IPs
-            '--master=no',
+            '--local-port=10053',               # On port 10053
-            '--slave=yes',
+            '--log-dns-details=on',             # Log details
-            '--version-string=powerdns',
+            '--loglevel=3',                     # Log info
+            '--master=yes',                     # Enable master mode
+            '--slave=yes',                      # Enable slave mode
+            '--slave-renotify=yes',             # Renotify out for our slaved zones
+            '--version-string=powerdns',        # Set the version string
+            '--default-soa-name=dns.pvc.local', # Override dnsmasq's invalid name
             '--socket-dir={}'.format(self.config['pdns_dynamic_directory']),
-            '--launch=gsqlite3',
+                                                # Standard socket directory
+            '--launch=gsqlite3',                # Use the sqlite3 backend
             '--gsqlite3-database={}'.format(self.database_file),
-            '--gsqlite3-dnssec=no'
+                                                # Database file
+            '--gsqlite3-dnssec=no'              # Don't do DNSSEC here
         ]
         # Start the pdns process in a thread
         self.dns_server_daemon = common.run_os_daemon(

node-daemon/pvcd/Daemon.py

@@ -123,10 +123,12 @@ config_values = [
     'migration_target_selector',
     'vni_dev',
     'vni_dev_ip',
+    'vni_floating_ip',
     'storage_dev',
     'storage_dev_ip',
     'upstream_dev',
     'upstream_dev_ip',
+    'upstream_floating_ip',
     'ipmi_hostname',
     'ipmi_username',
     'ipmi_password'
@@ -549,9 +551,9 @@ def update_networks(new_network_list):
     for network in new_network_list:
         if not network in network_list:
             d_network[network] = VXNetworkInstance.VXNetworkInstance(network, zk_conn, config, logger, this_node)
-            dns_aggregator.add_client_network(network)
             # Start primary functionality
             if this_node.router_state == 'primary':
+                dns_aggregator.add_client_network(network)
                 d_network[network].createGatewayAddress()
                 d_network[network].startDHCPServer()
 
@@ -562,10 +564,10 @@ def update_networks(new_network_list):
             if this_node.router_state == 'primary':
                 d_network[network].stopDHCPServer()
                 d_network[network].removeGatewayAddress()
+                dns_aggregator.remove_client_network(network)
             # Stop general functionality
             d_network[network].removeFirewall()
             d_network[network].removeNetwork()
-            dns_aggregator.remove_client_network(network)
             # Delete the object
             del(d_network[network])
 

node-daemon/pvcd/NodeInstance.py

@@ -69,6 +69,11 @@ class NodeInstance(object):
         self.memfree = 0
         self.memalloc = 0
         self.vcpualloc = 0
+        # Floating upstreams
+        self.vni_dev = self.config['vni_dev']
+        self.vni_ipaddr, self.vni_cidrnetmask = self.config['vni_floating_ip'].split('/')
+        self.upstream_dev = self.config['upstream_dev']
+        self.upstream_ipaddr, self.upstream_cidrnetmask = self.config['upstream_floating_ip'].split('/')
         # Flags
         self.inflush = False
 
@@ -295,10 +300,12 @@ class NodeInstance(object):
             self.d_network[network].stopDHCPServer()
             self.d_network[network].removeGatewayAddress()
         self.dns_aggregator.stop_aggregator()
+        self.removeFloatingAddresses()
 
     def become_primary(self):
         self.logger.out('Setting router {} to primary state.'.format(self.name), state='i')
         self.logger.out('Network list: {}'.format(', '.join(self.network_list)))
+        self.createFloatingAddresses()
         self.dns_aggregator.start_aggregator()
         time.sleep(0.5)
         # Start up the gateways and DHCP servers
@@ -310,6 +317,88 @@ class NodeInstance(object):
         for network in self.d_network: 
             self.dns_aggregator.get_axfr(network)
 
+    def createFloatingAddresses(self):
+        # VNI floating IP
+        self.logger.out(
+            'Creating floating management IP {}/{} on interface {}'.format(
+                self.vni_ipaddr,
+                self.vni_cidrnetmask,
+                self.vni_dev
+            ),
+            state='o'
+        )
+        common.run_os_command(
+            'ip address add {}/{} dev {}'.format(
+                self.vni_ipaddr,
+                self.vni_cidrnetmask,
+                self.vni_dev
+            )
+        )
+        common.run_os_command(
+            'arping -A -c2 -I {} {}'.format(
+                self.vni_dev,
+                self.vni_ipaddr
+            ),
+            background=True
+        )
+        # Upstream floating IP
+        self.logger.out(
+            'Creating floating upstream IP {}/{} on interface {}'.format(
+                self.upstream_ipaddr,
+                self.upstream_cidrnetmask,
+                self.upstream_dev
+            ),
+            state='o'
+        )
+        common.run_os_command(
+            'ip address add {}/{} dev {}'.format(
+                self.upstream_ipaddr,
+                self.upstream_cidrnetmask,
+                self.upstream_dev
+            )
+        )
+        common.run_os_command(
+            'arping -A -c2 -I {} {}'.format(
+                self.upstream_dev,
+                self.upstream_ipaddr
+            ),
+            background=True
+        )
+
+    def removeFloatingAddresses(self):
+        # VNI floating IP
+        self.logger.out(
+            'Removing floating management IP {}/{} from interface {}'.format(
+                self.vni_ipaddr,
+                self.vni_cidrnetmask,
+                self.vni_dev
+            ),
+            state='o'
+        )
+        common.run_os_command(
+            'ip address delete {}/{} dev {}'.format(
+                self.vni_ipaddr,
+                self.vni_cidrnetmask,
+                self.vni_dev
+            )
+        )
+        # Upstream floating IP
+        self.logger.out(
+            'Removing floating upstream IP {}/{} from interface {}'.format(
+                self.upstream_ipaddr,
+                self.upstream_cidrnetmask,
+                self.upstream_dev
+            ),
+            state='o'
+        )
+        common.run_os_command(
+            'ip address delete {}/{} dev {}'.format(
+                self.upstream_ipaddr,
+                self.upstream_cidrnetmask,
+                self.upstream_dev
+            )
+        )
+
     # Flush all VMs on the host
     def flush(self):
         self.inflush = True

node-daemon/pvcd/VXNetworkInstance.py

@@ -22,6 +22,7 @@
 
 import os
 import sys
+import time
 from textwrap import dedent
 
 import pvcd.log as log
@@ -153,6 +154,7 @@ class VXNetworkInstance(object):
             if self.dhcp_reservations != new_reservations:
                 old_reservations = self.dhcp_reservations
                 self.dhcp_reservations = new_reservations
+                if self.this_node.router_state == 'primary':
                     self.updateDHCPReservations(old_reservations, new_reservations)
 
         @self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules'.format(self.vni))
@@ -165,6 +167,7 @@ class VXNetworkInstance(object):
             if self.firewall_rules != new_rules:
                 old_rules = self.firewall_rules
                 self.firewall_rules = new_rules
+                if self.this_node.router_state == 'primary':
                     self.updateFirewallRules(old_rules, new_rules)
 
         self.createNetwork()
@@ -204,10 +207,12 @@ class VXNetworkInstance(object):
         for rule in new_rules_list:
             if rule not in old_rules_list:
                 # Add new rule entry
+                print(rule)
                 pass
 
         for rule in old_rules_list:
             if rule not in new_rules_list:
+                print(rule)
                 pass
 
     def createNetwork(self):
@@ -337,7 +342,7 @@ add rule inet filter input meta iifname {bridgenic} counter drop
                 '--listen-address={}'.format(self.ip_gateway),
                 '--bind-interfaces',
                 '--leasefile-ro',
-                '--dhcp-script=/usr/share/pvc/pvcd/dnsmasq-zookeeper-leases.py',
+                '--dhcp-script=./pvcd/dnsmasq-zookeeper-leases.py',
                 '--dhcp-range={},{},48h'.format(self.dhcp_start, self.dhcp_end),
                 '--dhcp-hostsdir={}'.format(self.dnsmasq_hostsdir),
                 '--log-facility=-',
@@ -419,6 +424,7 @@ add rule inet filter input meta iifname {bridgenic} counter drop
                 prefix='VNI {}'.format(self.vni),
                 state='o'
             )
-            self.dhcp_server_daemon.signal('int')
+            # Terminate, then kill
-            time.sleep(0.2)
             self.dhcp_server_daemon.signal('term')
+            time.sleep(0.2)
+            self.dhcp_server_daemon.signal('kill')

node-daemon/pvcd/common.py

@@ -50,7 +50,8 @@ class OSDaemon(object):
         signal_map = {
             'hup': signal.SIGHUP,
             'int': signal.SIGINT,
-            'term': signal.SIGTERM
+            'term': signal.SIGTERM,
+            'kill': signal.SIGKILL
         }
         self.proc.send_signal(signal_map[sent_signal])