Convert fencing to new zkhandler
This commit is contained in:
parent
1c9a7a6479
commit
6beea0693c
|
@ -21,7 +21,6 @@
|
||||||
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import pvcnoded.zkhandler as zkhandler
|
|
||||||
import pvcnoded.common as common
|
import pvcnoded.common as common
|
||||||
import pvcnoded.VMInstance as VMInstance
|
import pvcnoded.VMInstance as VMInstance
|
||||||
|
|
||||||
|
@ -29,7 +28,7 @@ import pvcnoded.VMInstance as VMInstance
|
||||||
#
|
#
|
||||||
# Fence thread entry function
|
# Fence thread entry function
|
||||||
#
|
#
|
||||||
def fenceNode(node_name, zk_conn, config, logger):
|
def fenceNode(node_name, zkhandler, config, logger):
|
||||||
# We allow exactly 6 saving throws (30 seconds) for the host to come back online or we kill it
|
# We allow exactly 6 saving throws (30 seconds) for the host to come back online or we kill it
|
||||||
failcount_limit = 6
|
failcount_limit = 6
|
||||||
failcount = 0
|
failcount = 0
|
||||||
|
@ -37,7 +36,7 @@ def fenceNode(node_name, zk_conn, config, logger):
|
||||||
# Wait 5 seconds
|
# Wait 5 seconds
|
||||||
time.sleep(config['keepalive_interval'])
|
time.sleep(config['keepalive_interval'])
|
||||||
# Get the state
|
# Get the state
|
||||||
node_daemon_state = zkhandler.readdata(zk_conn, '/nodes/{}/daemonstate'.format(node_name))
|
node_daemon_state = zkhandler.read('/nodes/{}/daemonstate'.format(node_name))
|
||||||
# Is it still 'dead'
|
# Is it still 'dead'
|
||||||
if node_daemon_state == 'dead':
|
if node_daemon_state == 'dead':
|
||||||
failcount += 1
|
failcount += 1
|
||||||
|
@ -50,9 +49,9 @@ def fenceNode(node_name, zk_conn, config, logger):
|
||||||
logger.out('Fencing node "{}" via IPMI reboot signal'.format(node_name), state='w')
|
logger.out('Fencing node "{}" via IPMI reboot signal'.format(node_name), state='w')
|
||||||
|
|
||||||
# Get IPMI information
|
# Get IPMI information
|
||||||
ipmi_hostname = zkhandler.readdata(zk_conn, '/nodes/{}/ipmihostname'.format(node_name))
|
ipmi_hostname = zkhandler.read('/nodes/{}/ipmihostname'.format(node_name))
|
||||||
ipmi_username = zkhandler.readdata(zk_conn, '/nodes/{}/ipmiusername'.format(node_name))
|
ipmi_username = zkhandler.read('/nodes/{}/ipmiusername'.format(node_name))
|
||||||
ipmi_password = zkhandler.readdata(zk_conn, '/nodes/{}/ipmipassword'.format(node_name))
|
ipmi_password = zkhandler.read('/nodes/{}/ipmipassword'.format(node_name))
|
||||||
|
|
||||||
# Shoot it in the head
|
# Shoot it in the head
|
||||||
fence_status = rebootViaIPMI(ipmi_hostname, ipmi_username, ipmi_password, logger)
|
fence_status = rebootViaIPMI(ipmi_hostname, ipmi_username, ipmi_password, logger)
|
||||||
|
@ -62,47 +61,53 @@ def fenceNode(node_name, zk_conn, config, logger):
|
||||||
# Force into secondary network state if needed
|
# Force into secondary network state if needed
|
||||||
if node_name in config['coordinators']:
|
if node_name in config['coordinators']:
|
||||||
logger.out('Forcing secondary status for node "{}"'.format(node_name), state='i')
|
logger.out('Forcing secondary status for node "{}"'.format(node_name), state='i')
|
||||||
zkhandler.writedata(zk_conn, {'/nodes/{}/routerstate'.format(node_name): 'secondary'})
|
zkhandler.write([
|
||||||
if zkhandler.readdata(zk_conn, '/config/primary_node') == node_name:
|
('/nodes/{}/routerstate'.format(node_name), 'secondary')
|
||||||
zkhandler.writedata(zk_conn, {'/config/primary_node': 'none'})
|
])
|
||||||
|
if zkhandler.read('/config/primary_node') == node_name:
|
||||||
|
zkhandler.write([
|
||||||
|
('/config/primary_node', 'none')
|
||||||
|
])
|
||||||
|
|
||||||
# If the fence succeeded and successful_fence is migrate
|
# If the fence succeeded and successful_fence is migrate
|
||||||
if fence_status and config['successful_fence'] == 'migrate':
|
if fence_status and config['successful_fence'] == 'migrate':
|
||||||
migrateFromFencedNode(zk_conn, node_name, config, logger)
|
migrateFromFencedNode(zkhandler, node_name, config, logger)
|
||||||
|
|
||||||
# If the fence failed and failed_fence is migrate
|
# If the fence failed and failed_fence is migrate
|
||||||
if not fence_status and config['failed_fence'] == 'migrate' and config['suicide_intervals'] != '0':
|
if not fence_status and config['failed_fence'] == 'migrate' and config['suicide_intervals'] != '0':
|
||||||
migrateFromFencedNode(zk_conn, node_name, config, logger)
|
migrateFromFencedNode(zkhandler, node_name, config, logger)
|
||||||
|
|
||||||
|
|
||||||
# Migrate hosts away from a fenced node
|
# Migrate hosts away from a fenced node
|
||||||
def migrateFromFencedNode(zk_conn, node_name, config, logger):
|
def migrateFromFencedNode(zkhandler, node_name, config, logger):
|
||||||
logger.out('Migrating VMs from dead node "{}" to new hosts'.format(node_name), state='i')
|
logger.out('Migrating VMs from dead node "{}" to new hosts'.format(node_name), state='i')
|
||||||
|
|
||||||
# Get the list of VMs
|
# Get the list of VMs
|
||||||
dead_node_running_domains = zkhandler.readdata(zk_conn, '/nodes/{}/runningdomains'.format(node_name)).split()
|
dead_node_running_domains = zkhandler.read('/nodes/{}/runningdomains'.format(node_name)).split()
|
||||||
|
|
||||||
# Set the node to a custom domainstate so we know what's happening
|
# Set the node to a custom domainstate so we know what's happening
|
||||||
zkhandler.writedata(zk_conn, {'/nodes/{}/domainstate'.format(node_name): 'fence-flush'})
|
zkhandler.write([
|
||||||
|
('/nodes/{}/domainstate'.format(node_name), 'fence-flush')
|
||||||
|
])
|
||||||
|
|
||||||
# Migrate a VM after a flush
|
# Migrate a VM after a flush
|
||||||
def fence_migrate_vm(dom_uuid):
|
def fence_migrate_vm(dom_uuid):
|
||||||
VMInstance.flush_locks(zk_conn, logger, dom_uuid)
|
VMInstance.flush_locks(zkhandler, logger, dom_uuid)
|
||||||
|
|
||||||
target_node = common.findTargetNode(zk_conn, config, logger, dom_uuid)
|
target_node = common.findTargetNode(zkhandler, config, logger, dom_uuid)
|
||||||
|
|
||||||
if target_node is not None:
|
if target_node is not None:
|
||||||
logger.out('Migrating VM "{}" to node "{}"'.format(dom_uuid, target_node), state='i')
|
logger.out('Migrating VM "{}" to node "{}"'.format(dom_uuid, target_node), state='i')
|
||||||
zkhandler.writedata(zk_conn, {
|
zkhandler.write([
|
||||||
'/domains/{}/state'.format(dom_uuid): 'start',
|
('/domains/{}/state'.format(dom_uuid), 'start'),
|
||||||
'/domains/{}/node'.format(dom_uuid): target_node,
|
('/domains/{}/node'.format(dom_uuid), target_node),
|
||||||
'/domains/{}/lastnode'.format(dom_uuid): node_name
|
('/domains/{}/lastnode'.format(dom_uuid), node_name)
|
||||||
})
|
])
|
||||||
else:
|
else:
|
||||||
logger.out('No target node found for VM "{}"; VM will autostart on next unflush/ready of current node'.format(dom_uuid), state='i')
|
logger.out('No target node found for VM "{}"; VM will autostart on next unflush/ready of current node'.format(dom_uuid), state='i')
|
||||||
zkhandler.writedata(zk_conn, {
|
zkhandler.write({
|
||||||
'/domains/{}/state'.format(dom_uuid): 'stopped',
|
('/domains/{}/state'.format(dom_uuid), 'stopped'),
|
||||||
'/domains/{}/node_autostart'.format(dom_uuid): 'True'
|
('/domains/{}/node_autostart'.format(dom_uuid), 'True')
|
||||||
})
|
})
|
||||||
|
|
||||||
# Loop through the VMs
|
# Loop through the VMs
|
||||||
|
@ -110,7 +115,9 @@ def migrateFromFencedNode(zk_conn, node_name, config, logger):
|
||||||
fence_migrate_vm(dom_uuid)
|
fence_migrate_vm(dom_uuid)
|
||||||
|
|
||||||
# Set node in flushed state for easy remigrating when it comes back
|
# Set node in flushed state for easy remigrating when it comes back
|
||||||
zkhandler.writedata(zk_conn, {'/nodes/{}/domainstate'.format(node_name): 'flushed'})
|
zkhandler.write([
|
||||||
|
('/nodes/{}/domainstate'.format(node_name), 'flushed')
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
#
|
#
|
||||||
|
|
Loading…
Reference in New Issue