Add floating IPs and better termination of daemons

This commit is contained in:
Joshua Boniface 2018-10-17 00:23:43 -04:00
parent c726865b89
commit 87d1c7513e
6 changed files with 143 additions and 23 deletions

View File

@ -31,6 +31,13 @@
# flush action; can be "mem", "load", "vcpus", or "vms" (defaults # flush action; can be "mem", "load", "vcpus", or "vms" (defaults
# to "mem"); the best choice based on this field is selected for # to "mem"); the best choice based on this field is selected for
# each VM to be migrated # each VM to be migrated
# vni_floating_ip: the IP address (in CIDR format) for the floating IP on the VNI network,
# used to provide a consistent view of the dynamic primary node to other
# machines in the VNI network, e.g. for slaving DNS or sending in routes.
# upstream_floating_ip: the IP address (in CIDR format) for the floating IP on the upstream
# network, used to provide a consistent view of the dynamic primary
# node to machines in the upstream network, e.g. for slaving DNS or
# sending in routes.
# The following values are required for each node specifically (usually node-unique): # The following values are required for each node specifically (usually node-unique):
# vni_dev: the lower-level network device to bind VNI traffic to # vni_dev: the lower-level network device to bind VNI traffic to
# vni_dev_ip: the IP address (in CIDR format) of the lower-level network device, used by frr # vni_dev_ip: the IP address (in CIDR format) of the lower-level network device, used by frr
@ -58,6 +65,8 @@ suicide_intervals = 0
successful_fence = migrate successful_fence = migrate
failed_fence = none failed_fence = none
migration_target_selector = mem migration_target_selector = mem
vni_floating_ip = 10.255.0.254/24
upstream_floating_ip = 10.101.0.30/24
[pvc-hv1] [pvc-hv1]
vni_dev = ens4 vni_dev = ens4

View File

@ -40,7 +40,12 @@ class DNSAggregatorInstance(object):
self.logger = logger self.logger = logger
self.d_network = d_network self.d_network = d_network
self.active = False # Floating upstreams
self.vni_dev = self.config['vni_dev']
self.vni_ipaddr, self.vni_cidrnetmask = self.config['vni_floating_ip'].split('/')
self.upstream_dev = self.config['upstream_dev']
self.upstream_ipaddr, self.upstream_cidrnetmask = self.config['upstream_floating_ip'].split('/')
self.database_file = self.config['pdns_dynamic_directory'] + '/pdns-aggregator.sqlite3' self.database_file = self.config['pdns_dynamic_directory'] + '/pdns-aggregator.sqlite3'
self.dns_server_daemon = None self.dns_server_daemon = None
@ -98,7 +103,7 @@ class DNSAggregatorInstance(object):
if write_domain: if write_domain:
sql_curs.execute( sql_curs.execute(
'insert into domains (name, master, type, account) values (?, ?, "SLAVE", "internal")', 'insert into domains (name, master, type, account) values (?, ?, "MASTER", "internal")',
(network_domain, network_gateway) (network_domain, network_gateway)
) )
sql_conn.commit() sql_conn.commit()
@ -146,22 +151,30 @@ class DNSAggregatorInstance(object):
) )
# Define the PowerDNS config # Define the PowerDNS config
dns_configuration = [ dns_configuration = [
# Option # Explanation
'--no-config', '--no-config',
'--daemon=no', '--daemon=no', # Start directly
'--disable-syslog=yes', '--guardian=yes', # Use a guardian
'--disable-axfr=no', '--disable-syslog=yes', # Log only to stdout (which is then captured)
'--guardian=yes', '--disable-axfr=no', # Allow AXFRs
'--local-address=0.0.0.0', '--allow-axfr-ips=0.0.0.0/0', # Allow AXFRs to anywhere
'--local-port=10053', '--also-notify=10.101.0.60', # Notify upstreams
'--log-dns-details=on', '--local-address={},{}'.format(self.vni_ipaddr, self.upstream_ipaddr),
'--loglevel=3', # Listen on floating IPs
'--master=no', '--local-port=10053', # On port 10053
'--slave=yes', '--log-dns-details=on', # Log details
'--version-string=powerdns', '--loglevel=3', # Log info
'--master=yes', # Enable master mode
'--slave=yes', # Enable slave mode
'--slave-renotify=yes', # Renotify out for our slaved zones
'--version-string=powerdns', # Set the version string
'--default-soa-name=dns.pvc.local', # Override dnsmasq's invalid name
'--socket-dir={}'.format(self.config['pdns_dynamic_directory']), '--socket-dir={}'.format(self.config['pdns_dynamic_directory']),
'--launch=gsqlite3', # Standard socket directory
'--launch=gsqlite3', # Use the sqlite3 backend
'--gsqlite3-database={}'.format(self.database_file), '--gsqlite3-database={}'.format(self.database_file),
'--gsqlite3-dnssec=no' # Database file
'--gsqlite3-dnssec=no' # Don't do DNSSEC here
] ]
# Start the pdns process in a thread # Start the pdns process in a thread
self.dns_server_daemon = common.run_os_daemon( self.dns_server_daemon = common.run_os_daemon(

View File

@ -123,10 +123,12 @@ config_values = [
'migration_target_selector', 'migration_target_selector',
'vni_dev', 'vni_dev',
'vni_dev_ip', 'vni_dev_ip',
'vni_floating_ip',
'storage_dev', 'storage_dev',
'storage_dev_ip', 'storage_dev_ip',
'upstream_dev', 'upstream_dev',
'upstream_dev_ip', 'upstream_dev_ip',
'upstream_floating_ip',
'ipmi_hostname', 'ipmi_hostname',
'ipmi_username', 'ipmi_username',
'ipmi_password' 'ipmi_password'
@ -549,9 +551,9 @@ def update_networks(new_network_list):
for network in new_network_list: for network in new_network_list:
if not network in network_list: if not network in network_list:
d_network[network] = VXNetworkInstance.VXNetworkInstance(network, zk_conn, config, logger, this_node) d_network[network] = VXNetworkInstance.VXNetworkInstance(network, zk_conn, config, logger, this_node)
dns_aggregator.add_client_network(network)
# Start primary functionality # Start primary functionality
if this_node.router_state == 'primary': if this_node.router_state == 'primary':
dns_aggregator.add_client_network(network)
d_network[network].createGatewayAddress() d_network[network].createGatewayAddress()
d_network[network].startDHCPServer() d_network[network].startDHCPServer()
@ -562,10 +564,10 @@ def update_networks(new_network_list):
if this_node.router_state == 'primary': if this_node.router_state == 'primary':
d_network[network].stopDHCPServer() d_network[network].stopDHCPServer()
d_network[network].removeGatewayAddress() d_network[network].removeGatewayAddress()
dns_aggregator.remove_client_network(network)
# Stop general functionality # Stop general functionality
d_network[network].removeFirewall() d_network[network].removeFirewall()
d_network[network].removeNetwork() d_network[network].removeNetwork()
dns_aggregator.remove_client_network(network)
# Delete the object # Delete the object
del(d_network[network]) del(d_network[network])

View File

@ -69,6 +69,11 @@ class NodeInstance(object):
self.memfree = 0 self.memfree = 0
self.memalloc = 0 self.memalloc = 0
self.vcpualloc = 0 self.vcpualloc = 0
# Floating upstreams
self.vni_dev = self.config['vni_dev']
self.vni_ipaddr, self.vni_cidrnetmask = self.config['vni_floating_ip'].split('/')
self.upstream_dev = self.config['upstream_dev']
self.upstream_ipaddr, self.upstream_cidrnetmask = self.config['upstream_floating_ip'].split('/')
# Flags # Flags
self.inflush = False self.inflush = False
@ -295,10 +300,12 @@ class NodeInstance(object):
self.d_network[network].stopDHCPServer() self.d_network[network].stopDHCPServer()
self.d_network[network].removeGatewayAddress() self.d_network[network].removeGatewayAddress()
self.dns_aggregator.stop_aggregator() self.dns_aggregator.stop_aggregator()
self.removeFloatingAddresses()
def become_primary(self): def become_primary(self):
self.logger.out('Setting router {} to primary state.'.format(self.name), state='i') self.logger.out('Setting router {} to primary state.'.format(self.name), state='i')
self.logger.out('Network list: {}'.format(', '.join(self.network_list))) self.logger.out('Network list: {}'.format(', '.join(self.network_list)))
self.createFloatingAddresses()
self.dns_aggregator.start_aggregator() self.dns_aggregator.start_aggregator()
time.sleep(0.5) time.sleep(0.5)
# Start up the gateways and DHCP servers # Start up the gateways and DHCP servers
@ -310,6 +317,88 @@ class NodeInstance(object):
for network in self.d_network: for network in self.d_network:
self.dns_aggregator.get_axfr(network) self.dns_aggregator.get_axfr(network)
def createFloatingAddresses(self):
# VNI floating IP
self.logger.out(
'Creating floating management IP {}/{} on interface {}'.format(
self.vni_ipaddr,
self.vni_cidrnetmask,
self.vni_dev
),
state='o'
)
common.run_os_command(
'ip address add {}/{} dev {}'.format(
self.vni_ipaddr,
self.vni_cidrnetmask,
self.vni_dev
)
)
common.run_os_command(
'arping -A -c2 -I {} {}'.format(
self.vni_dev,
self.vni_ipaddr
),
background=True
)
# Upstream floating IP
self.logger.out(
'Creating floating upstream IP {}/{} on interface {}'.format(
self.upstream_ipaddr,
self.upstream_cidrnetmask,
self.upstream_dev
),
state='o'
)
common.run_os_command(
'ip address add {}/{} dev {}'.format(
self.upstream_ipaddr,
self.upstream_cidrnetmask,
self.upstream_dev
)
)
common.run_os_command(
'arping -A -c2 -I {} {}'.format(
self.upstream_dev,
self.upstream_ipaddr
),
background=True
)
def removeFloatingAddresses(self):
# VNI floating IP
self.logger.out(
'Removing floating management IP {}/{} from interface {}'.format(
self.vni_ipaddr,
self.vni_cidrnetmask,
self.vni_dev
),
state='o'
)
common.run_os_command(
'ip address delete {}/{} dev {}'.format(
self.vni_ipaddr,
self.vni_cidrnetmask,
self.vni_dev
)
)
# Upstream floating IP
self.logger.out(
'Removing floating upstream IP {}/{} from interface {}'.format(
self.upstream_ipaddr,
self.upstream_cidrnetmask,
self.upstream_dev
),
state='o'
)
common.run_os_command(
'ip address delete {}/{} dev {}'.format(
self.upstream_ipaddr,
self.upstream_cidrnetmask,
self.upstream_dev
)
)
# Flush all VMs on the host # Flush all VMs on the host
def flush(self): def flush(self):
self.inflush = True self.inflush = True

View File

@ -22,6 +22,7 @@
import os import os
import sys import sys
import time
from textwrap import dedent from textwrap import dedent
import pvcd.log as log import pvcd.log as log
@ -153,6 +154,7 @@ class VXNetworkInstance(object):
if self.dhcp_reservations != new_reservations: if self.dhcp_reservations != new_reservations:
old_reservations = self.dhcp_reservations old_reservations = self.dhcp_reservations
self.dhcp_reservations = new_reservations self.dhcp_reservations = new_reservations
if self.this_node.router_state == 'primary':
self.updateDHCPReservations(old_reservations, new_reservations) self.updateDHCPReservations(old_reservations, new_reservations)
@self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules'.format(self.vni)) @self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules'.format(self.vni))
@ -165,6 +167,7 @@ class VXNetworkInstance(object):
if self.firewall_rules != new_rules: if self.firewall_rules != new_rules:
old_rules = self.firewall_rules old_rules = self.firewall_rules
self.firewall_rules = new_rules self.firewall_rules = new_rules
if self.this_node.router_state == 'primary':
self.updateFirewallRules(old_rules, new_rules) self.updateFirewallRules(old_rules, new_rules)
self.createNetwork() self.createNetwork()
@ -204,10 +207,12 @@ class VXNetworkInstance(object):
for rule in new_rules_list: for rule in new_rules_list:
if rule not in old_rules_list: if rule not in old_rules_list:
# Add new rule entry # Add new rule entry
print(rule)
pass pass
for rule in old_rules_list: for rule in old_rules_list:
if rule not in new_rules_list: if rule not in new_rules_list:
print(rule)
pass pass
def createNetwork(self): def createNetwork(self):
@ -337,7 +342,7 @@ add rule inet filter input meta iifname {bridgenic} counter drop
'--listen-address={}'.format(self.ip_gateway), '--listen-address={}'.format(self.ip_gateway),
'--bind-interfaces', '--bind-interfaces',
'--leasefile-ro', '--leasefile-ro',
'--dhcp-script=/usr/share/pvc/pvcd/dnsmasq-zookeeper-leases.py', '--dhcp-script=./pvcd/dnsmasq-zookeeper-leases.py',
'--dhcp-range={},{},48h'.format(self.dhcp_start, self.dhcp_end), '--dhcp-range={},{},48h'.format(self.dhcp_start, self.dhcp_end),
'--dhcp-hostsdir={}'.format(self.dnsmasq_hostsdir), '--dhcp-hostsdir={}'.format(self.dnsmasq_hostsdir),
'--log-facility=-', '--log-facility=-',
@ -419,6 +424,7 @@ add rule inet filter input meta iifname {bridgenic} counter drop
prefix='VNI {}'.format(self.vni), prefix='VNI {}'.format(self.vni),
state='o' state='o'
) )
self.dhcp_server_daemon.signal('int') # Terminate, then kill
time.sleep(0.2)
self.dhcp_server_daemon.signal('term') self.dhcp_server_daemon.signal('term')
time.sleep(0.2)
self.dhcp_server_daemon.signal('kill')

View File

@ -50,7 +50,8 @@ class OSDaemon(object):
signal_map = { signal_map = {
'hup': signal.SIGHUP, 'hup': signal.SIGHUP,
'int': signal.SIGINT, 'int': signal.SIGINT,
'term': signal.SIGTERM 'term': signal.SIGTERM,
'kill': signal.SIGKILL
} }
self.proc.send_signal(signal_map[sent_signal]) self.proc.send_signal(signal_map[sent_signal])