From 0d2e22a1119c1674103ce1c61caa6a91f327037a Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Sun, 12 Jan 2020 19:04:31 -0500 Subject: [PATCH] Normalize all static networks with bridges Modifies the storage and upstream networks to mirror the cluster network, with a bridge on top of the underlying specified dev, and all IPs bound to the bridge. Allows creating VMs in the storage or upstream networks, as well as the cluster network, should the administrator choose to do so (manually). --- node-daemon/pvcd/DNSAggregatorInstance.py | 2 - node-daemon/pvcd/Daemon.py | 58 ++++++++++++++++------- node-daemon/pvcd/NodeInstance.py | 8 ++-- 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/node-daemon/pvcd/DNSAggregatorInstance.py b/node-daemon/pvcd/DNSAggregatorInstance.py index a33635b3..f0e85776 100644 --- a/node-daemon/pvcd/DNSAggregatorInstance.py +++ b/node-daemon/pvcd/DNSAggregatorInstance.py @@ -77,9 +77,7 @@ class PowerDNSInstance(object): self.dns_server_daemon = None # Floating upstreams - self.vni_dev = self.config['vni_dev'] self.vni_ipaddr, self.vni_cidrnetmask = self.config['vni_floating_ip'].split('/') - self.upstream_dev = self.config['upstream_dev'] self.upstream_ipaddr, self.upstream_cidrnetmask = self.config['upstream_floating_ip'].split('/') def start(self): diff --git a/node-daemon/pvcd/Daemon.py b/node-daemon/pvcd/Daemon.py index 583a922b..bb7b3abc 100644 --- a/node-daemon/pvcd/Daemon.py +++ b/node-daemon/pvcd/Daemon.py @@ -365,7 +365,7 @@ if enable_networking: vni_dev = config['vni_dev'] vni_mtu = config['vni_mtu'] vni_dev_ip = config['vni_dev_ip'] - logger.out('Setting up VNI network interface {}'.format(vni_dev, vni_dev_ip), state='i') + logger.out('Setting up VNI network interface {} with MTU {}'.format(vni_dev, vni_mtu), state='i') common.run_os_command('ip link set {} mtu {} up'.format(vni_dev, vni_mtu)) # Cluster bridge configuration @@ -378,25 +378,47 @@ if enable_networking: # Storage configuration storage_dev = config['storage_dev'] storage_mtu = config['storage_mtu'] - if storage_dev == vni_dev: - storage_dev = 'brcluster' - storage_mtu = vni_mtu storage_dev_ip = config['storage_dev_ip'] - logger.out('Setting up Storage network on interface {} with IP {}'.format(storage_dev, storage_dev_ip), state='i') + logger.out('Setting up Storage network interface {} with MTU {}'.format(storage_dev, vni_mtu), state='i') common.run_os_command('ip link set {} mtu {} up'.format(storage_dev, storage_mtu)) - common.run_os_command('ip address add {} dev {}'.format(storage_dev_ip, storage_dev)) + + # Storage bridge configuration + if storage_dev == vni_dev: + logger.out('Adding Storage network IP {} to VNI Cluster bridge brcluster'.format(storage_dev_ip), state='i') + common.run_os_command('ip address add {} dev {}'.format(storage_dev_ip, 'brcluster')) + else: + logger.out('Setting up Storage network bridge on interface {} with IP {}'.format(vni_dev, vni_dev_ip), state='i') + common.run_os_command('brctl addbr brstorage') + common.run_os_command('brctl addif brstorage {}'.format(storage_dev)) + common.run_os_command('ip link set brstorage mtu {} up'.format(storage_mtu)) + common.run_os_command('ip address add {} dev {}'.format(storage_dev_ip, 'brstorage')) # Upstream configuration - if config['upstream_dev']: - upstream_dev = config['upstream_dev'] - upstream_mtu = config['upstream_mtu'] - upstream_dev_ip = config['upstream_dev_ip'] - upstream_dev_gateway = config['upstream_gateway'] - logger.out('Setting up Upstream network on interface {} with IP {}'.format(upstream_dev, upstream_dev_ip), state='i') - common.run_os_command('ip link set {} mtu {} up'.format(upstream_dev, upstream_mtu)) - common.run_os_command('ip address add {} dev {}'.format(upstream_dev_ip, upstream_dev)) - if upstream_dev_gateway: - common.run_os_command('ip route add default via {} dev {}'.format(upstream_dev_gateway, upstream_dev)) + upstream_dev = config['upstream_dev'] + upstream_mtu = config['upstream_mtu'] + upstream_dev_ip = config['upstream_dev_ip'] + logger.out('Setting up Upstream network interface {} with MTU {}'.format(upstream_dev, upstream_mtu), state='i') + common.run_os_command('ip link set {} mtu {} up'.format(upstream_dev, upstream_mtu)) + + # Upstream bridge configuration + if upstream_dev == vni_dev: + logger.out('Adding Upstream network IP {} to VNI Cluster bridge brcluster'.format(upstream_dev_ip), state='i') + common.run_os_command('ip address add {} dev {}'.format(upstream_dev_ip, 'brcluster')) + else: + logger.out('Setting up Upstream network bridge on interface {} with IP {}'.format(vni_dev, vni_dev_ip), state='i') + common.run_os_command('brctl addbr brupstream') + common.run_os_command('brctl addif brupstream {}'.format(upstream_dev)) + common.run_os_command('ip link set brupstream mtu {} up'.format(upstream_mtu)) + common.run_os_command('ip address add {} dev {}'.format(upstream_dev_ip, 'brupstream')) + + # Add upstream default gateway + upstream_gateway = config.get('upstream_gateway', None) + if upstream_gateway: + logger.out('Setting up Upstream default gateway IP {}'.format(upstream_gateway), state='i') + if upstream_dev == vni_dev: + common.run_os_command('ip route add default via {} dev {}'.format(upstream_gateway, 'brcluster')) + else: + common.run_os_command('ip route add default via {} dev {}'.format(upstream_gateway, 'brupstream')) ############################################################################### # PHASE 2b - Prepare sysctl for pvcd @@ -419,13 +441,15 @@ if enable_networking: common.run_os_command('sysctl net.ipv6.conf.all.accept_source_route=1') common.run_os_command('sysctl net.ipv6.conf.default.accept_source_route=1') - # Disable RP filtering on the VNI dev and bridge interfaces (to allow traffic pivoting) + # Disable RP filtering on the VNI Cluster and Upstream interfaces (to allow traffic pivoting) common.run_os_command('sysctl net.ipv4.conf.{}.rp_filter=0'.format(config['vni_dev'])) common.run_os_command('sysctl net.ipv4.conf.{}.rp_filter=0'.format(config['upstream_dev'])) common.run_os_command('sysctl net.ipv4.conf.brcluster.rp_filter=0') + common.run_os_command('sysctl net.ipv4.conf.brupstream.rp_filter=0') common.run_os_command('sysctl net.ipv6.conf.{}.rp_filter=0'.format(config['vni_dev'])) common.run_os_command('sysctl net.ipv6.conf.{}.rp_filter=0'.format(config['upstream_dev'])) common.run_os_command('sysctl net.ipv6.conf.brcluster.rp_filter=0') + common.run_os_command('sysctl net.ipv6.conf.brupstream.rp_filter=0') ############################################################################### # PHASE 3a - Determine coordinator mode diff --git a/node-daemon/pvcd/NodeInstance.py b/node-daemon/pvcd/NodeInstance.py index a88401ee..20a701d0 100644 --- a/node-daemon/pvcd/NodeInstance.py +++ b/node-daemon/pvcd/NodeInstance.py @@ -352,11 +352,11 @@ class NodeInstance(object): 'Creating floating upstream IP {}/{} on interface {}'.format( self.upstream_ipaddr, self.upstream_cidrnetmask, - self.upstream_dev + 'brupstream' ), state='o' ) - common.createIPAddress(self.upstream_ipaddr, self.upstream_cidrnetmask, self.upstream_dev) + common.createIPAddress(self.upstream_ipaddr, self.upstream_cidrnetmask, 'brupstream') self.logger.out('Releasing write lock for synchronization C', state='i') zkhandler.writedata(self.zk_conn, {'/locks/primary_node': ''}) lock.release() @@ -540,11 +540,11 @@ class NodeInstance(object): 'Removing floating upstream IP {}/{} from interface {}'.format( self.upstream_ipaddr, self.upstream_cidrnetmask, - self.upstream_dev + 'brupstream' ), state='o' ) - common.removeIPAddress(self.upstream_ipaddr, self.upstream_cidrnetmask, self.upstream_dev) + common.removeIPAddress(self.upstream_ipaddr, self.upstream_cidrnetmask, 'brupstream') self.logger.out('Releasing read lock for synchronization C', state='i') lock.release() self.logger.out('Released read lock for synchronization C', state='o')