Move to YAML config and allow split functions

1. Move to a YAML-based configuration format instead of the original
   INI-based configuration to facilitate better organization and
   readability.
2. Modify the daemon to be able to operate in several modes based
   on configuration flags. Either networking or storage functions
   can be disabled using the configuration, allowing the PVC system
   to be used only for hypervisor management if required.
This commit is contained in:
Joshua Boniface 2019-03-11 01:44:26 -04:00
parent 994315afa3
commit d90fb07240
11 changed files with 570 additions and 545 deletions

View File

@ -1 +1,2 @@
client-cli/pvc.py usr/share/pvc
client-cli/pvc_init.py usr/share/pvc

View File

@ -1,4 +1,4 @@
node-daemon/pvcd.py usr/share/pvc
node-daemon/pvcd.service lib/systemd/system
node-daemon/pvcd.conf.sample etc/pvc
node-daemon/pvcd.sample.yaml etc/pvc
node-daemon/pvcd usr/share/pvc

2
debian/rules vendored
View File

@ -6,6 +6,8 @@
%:
dh $@
override_dh_auto_clean:
find . -name "__pycache__" -exec rm -r {} \; || true
# If you need to rebuild the Sphinx documentation
# Add spinxdoc to the dh --with line

View File

@ -1 +1 @@
3.0 (quilt)
1.0

View File

@ -1,98 +0,0 @@
# pvcd cluster configuration file example
#
# This configuration file specifies details for this node in PVC. Multiple node
# blocks can be added but only the one matching the current system nodename will
# be used by the local daemon. Default values are not supported; the values in
# this sample configuration are considered defaults and, with adjustment of the
# nodename section and coordinators list, can be used as-is on a Debian system.
#
# The following values are required for each node or in a default section:
# coordinators: a CSV list of the short hostnames of the coordinator nodes; these nodes become
# members of the Zookeeper cluster, can act as routers, and perform additional
# special functions in a cluster; ideally there are 3 coordinators, though 5
# coordinators are supported
# cluster_domain: the node cluster domain, set during bootstrap
# storage_domain: the node storage domain, set during bootstrap
# dynamic_directory: the ramdisk directory for PVC to store its dynamic configurations,
# usually under /run or /var/run
# log_directory: the logging directory, usually under /var/log
# file_logging = whether to log daemon to a file (pvc.log under log_directory) in addition to
# normal stdout printing
# keepalive_interval: the interval between keepalives and for dead node timeout (defaults to 5)
# fence_intervals: the number of keepalive_intervals without Zookeeper contact before this node
# will consider another node dead and fence it (defaults to 6, i.e. 30s)
# suicide_intervals: the number of keepalive_intervals without Zookeeper contact before this
# node will consider itself failed and terminate all running VMs (defaults
# to 0, i.e. disabled); should be less than "fence_intervals"
# successful_fence: the action to take on a successful fencing operation; can be "none" or
# "migrate" (defaults to "migrate")
# failed_fence: the action to take on a failed fencing operation; can be "none" or "migrate"
# (defaults to "none"); "migrate" requires "suicide_intervals" to be set)
# NOTE: POTENTIALLY DANGEROUS - see README for details
# migration_target_selector: the method to use to select target nodes during a virtual machine
# flush action; can be "mem", "load", "vcpus", or "vms" (defaults
# to "mem"); the best choice based on this field is selected for
# each VM to be migrated
# pdns_mysql_host: the host address (usually "localhost") of the PowerDNS zone aggregator
# backend database
# pdns_mysql_port: the port (usually "3306") of the PowerDNS zone aggregator backend database
# pdns_mysql_dbname: the database name (usually "pvcdns") of the PowerDNS zone aggregator
# backend database
# pdns_mysql_user: the client username (usually "pvcdns") of the PowerDNS zone aggregator
# backend database
# pdns_mysql_password: the client user password (randomly generated at cluster bootstrap)
# of the PowerDNS zone aggregator backend database
# vni_floating_ip: the IP address (in CIDR format) for the floating IP on the VNI network,
# used to provide a consistent view of the dynamic primary node to other
# machines in the VNI network, e.g. for slaving DNS or sending in routes.
# upstream_floating_ip: the IP address (in CIDR format) for the floating IP on the upstream
# network, used to provide a consistent view of the dynamic primary
# node to machines in the upstream network, e.g. for slaving DNS or
# sending in routes.
# The following values are required for each node specifically (usually node-unique):
# vni_dev: the lower-level network device to bind VNI traffic to
# vni_dev_ip: the IP address (in CIDR format) of the lower-level network device, used by frr
# to communicate between nodes and pass routes between them.
# storage_dev: the lower-level network device to bind storage traffic to
# storage_dev_ip: the IP address (in CIDR format) of the lower-level network device, used by
# Ceph for storage traffic (both monitor and OSD).
# upstream_dev: the lower-level network device to bind coordinator upstream traffic to
# upstream_dev_ip: the IP address (in CIDR format) of the upstream network device, used by
# the system for upstream traffic flow.
# ipmi_hostname: the IPMI hostname for fencing (defaults to <shortname>-lom.<domain>)
# ipmi_username: username to connect to IPMI
# ipmi_password: password to connect to IPMI
#
# Copy this example to /etc/pvc/pvcd.conf and edit to your needs
[default]
coordinators = pvc-hv1,pvc-hv2,pvc-hv3
cluster_domain = i.bonilan.net
storage_domain = sx.bonilan.net
dynamic_directory = /run/pvc
log_directory = /var/log/pvc
file_logging = True
keepalive_interval = 5
fence_intervals = 6
suicide_intervals = 0
successful_fence = migrate
failed_fence = none
migration_target_selector = mem
pdns_mysql_host = localhost
pdns_mysql_port = 3306
pdns_mysql_dbname = pvcdns
pdns_mysql_user = pvcdns
pdns_mysql_password = pvcdns
vni_floating_ip = 10.255.0.254/24
upstream_floating_ip = 10.101.0.30/24
[pvc-hv1]
vni_dev = ens4
vni_dev_ip = 10.255.0.1/24
storage_dev = ens4
storage_dev_ip = 10.254.0.1/24
upstream_dev = ens2
upstream_dev_ip = 10.101.0.31/24
ipmi_username = admin
ipmi_password = Passw0rd
ipmi_hostname = pvc-hv1-lom

View File

@ -11,6 +11,15 @@
pvc:
# node: The (short) hostname of the node, set during provisioning
node: pvc-hv1
# functions: The daemon functions to enable
functions:
# enable_hypervisor: Enable or disable hypervisor functionality
# This should never be False except in very advanced usecases
enable_hypervisor: True
# enable_networking: Enable or disable virtual networking and routing functionality
enable_networking: True
# enable_storage: Enable or disable Ceph storage management functionality
enable_storage: True
# cluster: Cluster-level configuration
cluster:
# coordinators: The list of cluster coordinator hostnames
@ -19,6 +28,7 @@ pvc:
- pvc-hv2
- pvc-hv3
# networks: Cluster-level network configuration
# OPTIONAL if enable_networking: False
networks:
# upstream: Upstream routed network for in- and out-bound upstream networking
upstream:
@ -43,8 +53,9 @@ pvc:
# network: Cluster storage network block
network: "10.254.0.0/24"
# floating_ip: Cluster storage floating IP address for the primary coordinator
floating_ip: "10.255.0.254/24"
floating_ip: "10.254.0.254/24"
# coordinator: Coordinator-specific configuration
# OPTIONAL if enable_networking: False
coordinator:
# dns: DNS aggregator subsystem
dns:
@ -105,6 +116,7 @@ pvc:
# stdout_logging: Enable or disable logging to stdout (i.e. journald)
stdout_logging: True
# networking: PVC networking configuration
# OPTIONAL if enable_networking: False
networking:
# devices: Interface devices configuration
devices:

View File

@ -7,7 +7,7 @@ After = network-online.target libvirtd.service zookeeper.service mariadb.service
Type = simple
WorkingDirectory = /usr/share/pvc
Environment = PYTHONUNBUFFERED=true
Environment = PVCD_CONFIG_FILE=/etc/pvc/pvcd.conf
Environment = PVCD_CONFIG_FILE=/etc/pvc/pvcd.yaml
ExecStart = /usr/share/pvc/pvcd.py
Restart = never

File diff suppressed because it is too large Load Diff

View File

@ -66,10 +66,18 @@ class NodeInstance(object):
self.memalloc = 0
self.vcpualloc = 0
# Floating upstreams
self.vni_dev = self.config['vni_dev']
self.vni_ipaddr, self.vni_cidrnetmask = self.config['vni_floating_ip'].split('/')
self.upstream_dev = self.config['upstream_dev']
self.upstream_ipaddr, self.upstream_cidrnetmask = self.config['upstream_floating_ip'].split('/')
if self.config['enable_networking']:
self.vni_dev = self.config['vni_dev']
self.vni_ipaddr, self.vni_cidrnetmask = self.config['vni_floating_ip'].split('/')
self.upstream_dev = self.config['upstream_dev']
self.upstream_ipaddr, self.upstream_cidrnetmask = self.config['upstream_floating_ip'].split('/')
else:
self.vni_dev = None
self.vni_ipaddr = None
self.vni_cidrnetmask = None
self.upstream_dev = None
self.upstream_ipaddr = None
self.upstream_cidrnetmask = None
# Flags
self.inflush = False
@ -240,25 +248,27 @@ class NodeInstance(object):
# Routing primary/secondary states
def become_secondary(self):
self.logger.out('Setting router {} to secondary state'.format(self.name), state='i')
self.logger.out('Network list: {}'.format(', '.join(self.network_list)))
time.sleep(1)
for network in self.d_network:
self.d_network[network].stopDHCPServer()
self.d_network[network].removeGateways()
self.removeFloatingAddresses()
self.dns_aggregator.stop_aggregator()
if self.config['enable_networking']:
self.logger.out('Setting router {} to secondary state'.format(self.name), state='i')
self.logger.out('Network list: {}'.format(', '.join(self.network_list)))
time.sleep(1)
for network in self.d_network:
self.d_network[network].stopDHCPServer()
self.d_network[network].removeGateways()
self.removeFloatingAddresses()
self.dns_aggregator.stop_aggregator()
def become_primary(self):
self.logger.out('Setting router {} to primary state'.format(self.name), state='i')
self.logger.out('Network list: {}'.format(', '.join(self.network_list)))
self.createFloatingAddresses()
# Start up the gateways and DHCP servers
for network in self.d_network:
self.d_network[network].createGateways()
self.d_network[network].startDHCPServer()
time.sleep(0.5)
self.dns_aggregator.start_aggregator()
if self.config['enable_networking']:
self.logger.out('Setting router {} to primary state'.format(self.name), state='i')
self.logger.out('Network list: {}'.format(', '.join(self.network_list)))
self.createFloatingAddresses()
# Start up the gateways and DHCP servers
for network in self.d_network:
self.d_network[network].createGateways()
self.d_network[network].startDHCPServer()
time.sleep(0.5)
self.dns_aggregator.start_aggregator()
def createFloatingAddresses(self):
# VNI floating IP

13
node-daemon/pvcd/fixrbdlocks Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
for disk in $( sudo rbd list ${BLSE_STORAGE_POOL_VM} | grep "^${vm}" ); do
echo -e " Disk: $disk"
locks="$( sudo rbd lock list ${BLSE_STORAGE_POOL_VM}/${disk} | grep '^client' )"
echo "${locks}"
if [[ -n "${locks}" ]]; then
echo -e " LOCK FOUND! Clearing."
locker="$( awk '{ print $1 }' <<<"${locks}" )"
id="$( awk '{ print $2" "$3 }' <<<"${locks}" )"
sudo rbd lock remove ${BLSE_STORAGE_POOL_VM}/${disk} "${id}" "${locker}"
fi
done

View File

@ -45,6 +45,8 @@ class Logger(object):
# We open the logfile for the duration of our session, but have a hup function
self.writer = open(self.logfile, 'a', buffering=1)
self.last_colour = self.fmt_cyan
else:
self.last_colour = ""
# Provide a hup function to close and reopen the writer
def hup(self):