parallelvirtualcluster
/
pvc
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 111 Wiki Activity

Add support for SR-IOV NICs to VMs

This commit is contained in:
Joshua Boniface 2021-06-21 22:21:54 -04:00
parent 93c2fdec93
commit eeb83da97d
7 changed files with 245 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
client-cli/cli_lib/network.py
View File

@ -1034,6 +1034,12 @@ def format_info_sriov_vf(config, vf_information, node):
ainformation.append('{}VF User Trust:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), getColour(vf_information['config']['trust']), vf_information['config']['trust'], ansiprint.end()))
ainformation.append('{}Query RSS Config:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), getColour(vf_information['config']['query_rss']), vf_information['config']['query_rss'], ansiprint.end()))
ainformation.append('')
# PCIe bus information
ainformation.append('{}PCIe domain:{} {}'.format(ansiprint.purple(), ansiprint.end(), vf_information['pci']['domain']))
ainformation.append('{}PCIe bus:{} {}'.format(ansiprint.purple(), ansiprint.end(), vf_information['pci']['bus']))
ainformation.append('{}PCIe slot:{} {}'.format(ansiprint.purple(), ansiprint.end(), vf_information['pci']['slot']))
ainformation.append('{}PCIe function:{} {}'.format(ansiprint.purple(), ansiprint.end(), vf_information['pci']['function']))
ainformation.append('')
# Usage information
ainformation.append('{}VF Used:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), getColour(vf_information['usage']['used']), vf_information['usage']['used'], ansiprint.end()))
if vf_information['usage']['used'] == 'True' and vm_information is not None:

125
client-cli/cli_lib/vm.py
View File

@ -501,7 +501,7 @@ def format_vm_memory(config, name, memory):
return '\n'.join(output_list)
def vm_networks_add(config, vm, network, macaddr, model, restart):
def vm_networks_add(config, vm, network, macaddr, model, sriov, sriov_mode, restart):
"""
Add a new network to the VM
@ -514,13 +514,15 @@ def vm_networks_add(config, vm, network, macaddr, model, restart):
from random import randint
import cli_lib.network as pvc_network
# Verify that the provided network is valid
# Verify that the provided network is valid (not in SR-IOV mode)
if not sriov:
retcode, retdata = pvc_network.net_info(config, network)
if not retcode:
# Ignore the three special networks
if network not in ['upstream', 'cluster', 'storage']:
return False, "Network {} is not present in the cluster.".format(network)
# Set the bridge prefix
if network in ['upstream', 'cluster', 'storage']:
br_prefix = 'br'
else:
@ -551,24 +553,73 @@ def vm_networks_add(config, vm, network, macaddr, model, restart):
octetC=random_octet_C
)
# Add an SR-IOV network
if sriov:
valid, sriov_vf_information = pvc_network.net_sriov_vf_info(config, domain_information['node'], network)
if not valid:
return False, 'Specified SR-IOV VF "{}" does not exist on VM node "{}".'.format(network, domain_information['node'])
# Add a hostdev (direct PCIe) SR-IOV network
if sriov_mode == 'hostdev':
bus_address = 'domain="0x{pci_domain}" bus="0x{pci_bus}" slot="0x{pci_slot}" function="0x{pci_function}"'.format(
pci_domain=sriov_vf_information['pci']['domain'],
pci_bus=sriov_vf_information['pci']['bus'],
pci_slot=sriov_vf_information['pci']['slot'],
pci_function=sriov_vf_information['pci']['function'],
)
device_string = '<interface type="hostdev"><mac address="{macaddr}"/><source><address type="pci" {bus_address}/></source><sriov_device>{network}</sriov_device></interface>'.format(
macaddr=macaddr,
bus_address=bus_address,
network=network
)
# Add a macvtap SR-IOV network
elif sriov_mode == 'macvtap':
device_string = '<interface type="direct"><mac address="{macaddr}"/><source dev="{network}" mode="passthrough"/></interface>'.format(
macaddr=macaddr,
network=network
)
else:
return False, "ERROR: Invalid SR-IOV mode specified."
# Add a normal bridged PVC network
else:
device_string = '<interface type="bridge"><mac address="{macaddr}"/><source bridge="{bridge}"/><model type="{model}"/></interface>'.format(
macaddr=macaddr,
bridge="{}{}".format(br_prefix, network),
model=model
)
device_xml = fromstring(device_string)
last_interface = None
all_interfaces = parsed_xml.devices.find('interface')
if all_interfaces is None:
all_interfaces = []
for interface in all_interfaces:
last_interface = re.match(r'[vm]*br([0-9a-z]+)', interface.source.attrib.get('bridge')).group(1)
if last_interface == network:
return False, 'Network {} is already configured for VM {}.'.format(network, vm)
if last_interface is not None:
for interface in parsed_xml.devices.find('interface'):
if last_interface == re.match(r'[vm]*br([0-9a-z]+)', interface.source.attrib.get('bridge')).group(1):
if sriov:
if sriov_mode == 'hostdev':
if interface.attrib.get('type') == 'hostdev':
interface_address = 'domain="{pci_domain}" bus="{pci_bus}" slot="{pci_slot}" function="{pci_function}"'.format(
interface.source.address.attrib.get('domain'),
interface.source.address.attrib.get('bus'),
interface.source.address.attrib.get('slot'),
interface.source.address.attrib.get('function')
)
if interface_address == bus_address:
return False, 'Network "{}" is already configured for VM "{}".'.format(network, vm)
elif sriov_mode == 'macvtap':
if interface.attrib.get('type') == 'direct':
interface_dev = interface.source.attrib.get('dev')
if interface_dev == network:
return False, 'Network "{}" is already configured for VM "{}".'.format(network, vm)
else:
if interface.attrib.get('type') == 'bridge':
interface_vni = re.match(r'[vm]*br([0-9a-z]+)', interface.source.attrib.get('bridge')).group(1)
if interface_vni == network:
return False, 'Network "{}" is already configured for VM "{}".'.format(network, vm)
# Add the interface at the end of the list (or, right above emulator)
if len(all_interfaces) > 0:
for idx, interface in enumerate(parsed_xml.devices.find('interface')):
if idx == len(all_interfaces) - 1:
interface.addnext(device_xml)
else:
parsed_xml.devices.find('emulator').addprevious(device_xml)
@ -581,7 +632,7 @@ def vm_networks_add(config, vm, network, macaddr, model, restart):
return vm_modify(config, vm, new_xml, restart)
def vm_networks_remove(config, vm, network, restart):
def vm_networks_remove(config, vm, network, sriov, restart):
"""
Remove a network to the VM
@ -605,17 +656,33 @@ def vm_networks_remove(config, vm, network, restart):
except Exception:
return False, 'ERROR: Failed to parse XML data.'
changed = False
for interface in parsed_xml.devices.find('interface'):
if sriov:
if interface.attrib.get('type') == 'hostdev':
if_dev = str(interface.sriov_device)
if network == if_dev:
interface.getparent().remove(interface)
changed = True
elif interface.attrib.get('type') == 'direct':
if_dev = str(interface.source.attrib.get('dev'))
if network == if_dev:
interface.getparent().remove(interface)
changed = True
else:
if_vni = re.match(r'[vm]*br([0-9a-z]+)', interface.source.attrib.get('bridge')).group(1)
if network == if_vni:
interface.getparent().remove(interface)
changed = True
if changed:
try:
new_xml = tostring(parsed_xml, pretty_print=True)
except Exception:
return False, 'ERROR: Failed to dump XML data.'
return vm_modify(config, vm, new_xml, restart)
else:
return False, 'ERROR: Network "{}" does not exist on VM.'.format(network)
def vm_networks_get(config, vm):
@ -1178,7 +1245,7 @@ def format_info(config, domain_information, long_output):
cluster_net_list = call_api(config, 'get', '/network').json()
for net in domain_information['networks']:
net_vni = net['vni']
if net_vni not in ['cluster', 'storage', 'upstream'] and not re.match(r'^e.*', net_vni):
if net_vni not in ['cluster', 'storage', 'upstream'] and not re.match(r'^macvtap:.*', net_vni) and not re.match(r'^hostdev:.*', net_vni):
if int(net_vni) not in [net['vni'] for net in cluster_net_list]:
net_list.append(ansiprint.red() + net_vni + ansiprint.end() + ' [invalid]')
else:
@ -1212,15 +1279,29 @@ def format_info(config, domain_information, long_output):
ainformation.append('')
ainformation.append('{}Interfaces:{} {}ID Type Source Model MAC Data (r/w) Packets (r/w) Errors (r/w){}'.format(ansiprint.purple(), ansiprint.end(), ansiprint.bold(), ansiprint.end()))
for net in domain_information['networks']:
ainformation.append(' {0: <3} {1: <7} {2: <10} {3: <8} {4: <18} {5: <12} {6: <15} {7: <12}'.format(
net_type = net['type']
net_source = net['source']
net_mac = net['mac']
if net_type in ['direct', 'hostdev']:
net_model = 'N/A'
net_bytes = 'N/A'
net_packets = 'N/A'
net_errors = 'N/A'
elif net_type in ['bridge']:
net_model = net['model']
net_bytes = '/'.join([str(format_bytes(net.get('rd_bytes', 0))), str(format_bytes(net.get('wr_bytes', 0)))])
net_packets = '/'.join([str(format_metric(net.get('rd_packets', 0))), str(format_metric(net.get('wr_packets', 0)))])
net_errors = '/'.join([str(format_metric(net.get('rd_errors', 0))), str(format_metric(net.get('wr_errors', 0)))])
ainformation.append(' {0: <3} {1: <8} {2: <12} {3: <8} {4: <18} {5: <12} {6: <15} {7: <12}'.format(
domain_information['networks'].index(net),
net['type'],
net['source'],
net['model'],
net['mac'],
'/'.join([str(format_bytes(net.get('rd_bytes', 0))), str(format_bytes(net.get('wr_bytes', 0)))]),
'/'.join([str(format_metric(net.get('rd_packets', 0))), str(format_metric(net.get('wr_packets', 0)))]),
'/'.join([str(format_metric(net.get('rd_errors', 0))), str(format_metric(net.get('wr_errors', 0)))]),
net_type,
net_source,
net_model,
net_mac,
net_bytes,
net_packets,
net_errors
))
# Controller list
ainformation.append('')
@ -1259,7 +1340,7 @@ def format_list(config, vm_list, raw):
vm_nets_length = 9
vm_ram_length = 8
vm_vcpu_length = 6
vm_node_length = 8
vm_node_length = 5
vm_migrated_length = 10
for domain_information in vm_list:
net_list = getNiceNetID(domain_information)
@ -1335,7 +1416,7 @@ def format_list(config, vm_list, raw):
cluster_net_list = call_api(config, 'get', '/network').json()
vm_net_colour = ''
for net_vni in net_list:
if net_vni not in ['cluster', 'storage', 'upstream'] and not re.match(r'^e.*', net_vni):
if net_vni not in ['cluster', 'storage', 'upstream'] and not re.match(r'^macvtap:.*', net_vni) and not re.match(r'^hostdev:.*', net_vni):
if int(net_vni) not in [net['vni'] for net in cluster_net_list]:
vm_net_colour = ansiprint.red()

44
client-cli/pvc.py
View File

@ -1309,15 +1309,24 @@ def vm_network_get(domain, raw):
'domain'
)
@click.argument(
'vni'
'net'
)
@click.option(
'-a', '--macaddr', 'macaddr', default=None,
help='Use this MAC address instead of random generation; must be a valid MAC address in colon-deliniated format.'
help='Use this MAC address instead of random generation; must be a valid MAC address in colon-delimited format.'
)
@click.option(
'-m', '--model', 'model', default='virtio',
help='The model for the interface; must be a valid libvirt model.'
help='The model for the interface; must be a valid libvirt model. Not used for SR-IOV NETs.'
)
@click.option(
'-s', '--sriov', 'sriov', is_flag=True, default=False,
help='Identify that NET is an SR-IOV device name and not a VNI. Required for adding SR-IOV NETs.'
)
@click.option(
'-d', '--sriov-mode', 'sriov_mode', default='hostdev',
type=click.Choice(['hostdev', 'macvtap']),
help='For SR-IOV NETs, the SR-IOV network device mode.'
)
@click.option(
'-r', '--restart', 'restart', is_flag=True, default=False,
@ -1329,9 +1338,18 @@ def vm_network_get(domain, raw):
help='Confirm the restart'
)
@cluster_req
def vm_network_add(domain, vni, macaddr, model, restart, confirm_flag):
def vm_network_add(domain, net, macaddr, model, sriov, sriov_mode, restart, confirm_flag):
"""
Add the network VNI to the virtual machine DOMAIN. Networks are always addded to the end of the current list of networks in the virtual machine.
Add the network NET to the virtual machine DOMAIN. Networks are always addded to the end of the current list of networks in the virtual machine.
NET may be a PVC network VNI, which is added as a bridged device, or a SR-IOV VF device connected in the given mode.
NOTE: Adding a SR-IOV network device in the "hostdev" mode has the following caveats:
1. The VM will not be able to be live migrated; it must be shut down to migrate between nodes. The VM metadata will be updated to force this.
2. If an identical SR-IOV VF device is not present on the target node, post-migration startup will fail. It may be prudent to use a node limit here.
"""
if restart and not confirm_flag and not config['unsafe']:
try:
@ -1339,7 +1357,7 @@ def vm_network_add(domain, vni, macaddr, model, restart, confirm_flag):
except Exception:
restart = False
retcode, retmsg = pvc_vm.vm_networks_add(config, domain, vni, macaddr, model, restart)
retcode, retmsg = pvc_vm.vm_networks_add(config, domain, net, macaddr, model, sriov, sriov_mode, restart)
if retcode and not restart:
retmsg = retmsg + " Changes will be applied on next VM start/restart."
cleanup(retcode, retmsg)
@ -1353,7 +1371,11 @@ def vm_network_add(domain, vni, macaddr, model, restart, confirm_flag):
'domain'
)
@click.argument(
'vni'
'net'
)
@click.option(
'-s', '--sriov', 'sriov', is_flag=True, default=False,
help='Identify that NET is an SR-IOV device name and not a VNI. Required for removing SR-IOV NETs.'
)
@click.option(
'-r', '--restart', 'restart', is_flag=True, default=False,
@ -1365,9 +1387,11 @@ def vm_network_add(domain, vni, macaddr, model, restart, confirm_flag):
help='Confirm the restart'
)
@cluster_req
def vm_network_remove(domain, vni, restart, confirm_flag):
def vm_network_remove(domain, net, sriov, restart, confirm_flag):
"""
Remove the network VNI to the virtual machine DOMAIN.
Remove the network NET from the virtual machine DOMAIN.
NET may be a PVC network VNI, which is added as a bridged device, or a SR-IOV VF device connected in the given mode.
"""
if restart and not confirm_flag and not config['unsafe']:
try:
@ -1375,7 +1399,7 @@ def vm_network_remove(domain, vni, restart, confirm_flag):
except Exception:
restart = False
retcode, retmsg = pvc_vm.vm_networks_remove(config, domain, vni, restart)
retcode, retmsg = pvc_vm.vm_networks_remove(config, domain, net, sriov, restart)
if retcode and not restart:
retmsg = retmsg + " Changes will be applied on next VM start/restart."
cleanup(retcode, retmsg)

15
daemon-common/common.py
View File

@ -373,23 +373,28 @@ def getDomainNetworks(parsed_xml, stats_data):
net_type = device.attrib.get('type')
except Exception:
net_type = None
try:
net_mac = device.mac.attrib.get('address')
except Exception:
net_mac = None
try:
net_bridge = device.source.attrib.get(net_type)
except Exception:
net_bridge = None
try:
net_model = device.model.attrib.get('type')
except Exception:
net_model = None
try:
net_stats_list = [x for x in stats_data.get('net_stats', []) if x.get('bridge') == net_bridge]
net_stats = net_stats_list[0]
except Exception:
net_stats = {}
net_rd_bytes = net_stats.get('rd_bytes', 0)
net_rd_packets = net_stats.get('rd_packets', 0)
net_rd_errors = net_stats.get('rd_errors', 0)
@ -398,10 +403,16 @@ def getDomainNetworks(parsed_xml, stats_data):
net_wr_packets = net_stats.get('wr_packets', 0)
net_wr_errors = net_stats.get('wr_errors', 0)
net_wr_drops = net_stats.get('wr_drops', 0)
if net_type in ['direct', 'hostdev']:
net_vni = device.source.attrib.get('dev')
if net_type == 'direct':
net_vni = 'macvtap:' + device.source.attrib.get('dev')
net_bridge = device.source.attrib.get('dev')
elif net_type == 'hostdev':
net_vni = 'hostdev:' + str(device.sriov_device)
net_bridge = str(device.sriov_device)
else:
net_vni = re_match(r'[vm]*br([0-9a-z]+)', net_bridge).group(1)
net_obj = {
'type': net_type,
'vni': net_vni,

24
daemon-common/network.py
View File

@ -806,7 +806,7 @@ def set_sriov_vf_config(zkhandler, node, vf, vlan_id=None, vlan_qos=None, tx_rat
return False, 'Failed to modify configuration of SR-IOV VF "{}" on node "{}".'.format(vf, node)
def set_sriov_vf_vm(zkhandler, node, vf, vm_name, vm_macaddr):
def set_sriov_vf_vm(zkhandler, vm_uuid, node, vf, vf_macaddr, vf_type):
# Verify node is valid
valid_node = common.verifyNode(zkhandler, node)
if not valid_node:
@ -817,11 +817,19 @@ def set_sriov_vf_vm(zkhandler, node, vf, vm_name, vm_macaddr):
if not vf_information:
return False
zkhandler.write([
update_list = [
(('node.sriov.vf', node, 'sriov_vf.used', vf), 'True'),
(('node.sriov.vf', node, 'sriov_vf.used_by', vf), vm_name),
(('node.sriov.vf', node, 'sriov_vf.mac', vf), vm_macaddr),
])
(('node.sriov.vf', node, 'sriov_vf.used_by', vf), vm_uuid),
(('node.sriov.vf', node, 'sriov_vf.mac', vf), vf_macaddr),
]
# Hostdev type SR-IOV prevents the guest from live migrating
if vf_type == 'hostdev':
update_list.append(
(('domain.meta.migrate_method', vm_uuid), 'shutdown')
)
zkhandler.write(update_list)
return True
@ -837,9 +845,11 @@ def unset_sriov_vf_vm(zkhandler, node, vf):
if not vf_information:
return False
zkhandler.write([
update_list = [
(('node.sriov.vf', node, 'sriov_vf.used', vf), 'False'),
(('node.sriov.vf', node, 'sriov_vf.used_by', vf), ''),
])
]
zkhandler.write(update_list)
return True

52
daemon-common/vm.py
View File

@ -27,6 +27,7 @@ import lxml.etree
import daemon_lib.common as common
import daemon_lib.ceph as ceph
from daemon_lib.network import set_sriov_vf_vm, unset_sriov_vf_vm
#
@ -191,6 +192,21 @@ def define_vm(zkhandler, config_data, target_node, node_limit, node_selector, no
if not valid_node:
return False, 'ERROR: Specified node "{}" is invalid.'.format(target_node)
# If a SR-IOV network device is being added, set its used state
dnetworks = common.getDomainNetworks(parsed_xml, {})
for network in dnetworks:
if network['type'] in ['direct', 'hostdev']:
dom_node = zkhandler.read(('domain.node', dom_uuid))
# Check if the network is already in use
is_used = zkhandler.read(('node.sriov.vf', dom_node, 'sriov_vf.used', network['source']))
if is_used == 'True':
used_by_name = searchClusterByUUID(zkhandler, zkhandler.read(('node.sriov.vf', dom_node, 'sriov_vf.used_by', network['source'])))
return False, 'ERROR: Attempted to use SR-IOV network "{}" which is already used by VM "{}" on node "{}".'.format(network['source'], used_by_name, dom_node)
# We must update the "used" section
set_sriov_vf_vm(zkhandler, dom_uuid, dom_node, network['source'], network['mac'], network['type'])
# Obtain the RBD disk list using the common functions
ddisks = common.getDomainDisks(parsed_xml, {})
rbd_list = []
@ -211,7 +227,7 @@ def define_vm(zkhandler, config_data, target_node, node_limit, node_selector, no
formatted_rbd_list = ''
# Add the new domain to Zookeeper
result = zkhandler.write([
zkhandler.write([
(('domain', dom_uuid), dom_name),
(('domain.xml', dom_uuid), config_data),
(('domain.state', dom_uuid), initial_state),
@ -230,10 +246,7 @@ def define_vm(zkhandler, config_data, target_node, node_limit, node_selector, no
(('domain.migrate.sync_lock', dom_uuid), ''),
])
if result:
return True, 'Added new VM with Name "{}" and UUID "{}" to database.'.format(dom_name, dom_uuid)
else:
return False, 'ERROR: Failed to add new VM.'
def modify_vm_metadata(zkhandler, domain, node_limit, node_selector, node_autostart, provisioner_profile, migration_method):
@ -276,7 +289,36 @@ def modify_vm(zkhandler, domain, restart, new_vm_config):
try:
parsed_xml = lxml.objectify.fromstring(new_vm_config)
except Exception:
return False, 'ERROR: Failed to parse XML data.'
return False, 'ERROR: Failed to parse new XML data.'
# If a SR-IOV network device is being added, set its used state
dnetworks = common.getDomainNetworks(parsed_xml, {})
for network in dnetworks:
if network['type'] in ['direct', 'hostdev']:
dom_node = zkhandler.read(('domain.node', dom_uuid))
# Check if the network is already in use
is_used = zkhandler.read(('node.sriov.vf', dom_node, 'sriov_vf.used', network['source']))
if is_used == 'True':
used_by_name = searchClusterByUUID(zkhandler, zkhandler.read(('node.sriov.vf', dom_node, 'sriov_vf.used_by', network['source'])))
return False, 'ERROR: Attempted to use SR-IOV network "{}" which is already used by VM "{}" on node "{}".'.format(network['source'], used_by_name, dom_node)
# We must update the "used" section
set_sriov_vf_vm(zkhandler, dom_uuid, dom_node, network['source'], network['mac'], network['type'])
# If a SR-IOV network device is being removed, unset its used state
old_vm_config = zkhandler.read(('domain.xml', dom_uuid))
try:
old_parsed_xml = lxml.objectify.fromstring(old_vm_config)
except Exception:
return False, 'ERROR: Failed to parse old XML data.'
old_dnetworks = common.getDomainNetworks(old_parsed_xml, {})
for network in old_dnetworks:
if network['type'] in ['direct', 'hostdev']:
if network['mac'] not in [n['mac'] for n in dnetworks]:
dom_node = zkhandler.read(('domain.node', dom_uuid))
# We must update the "used" section
unset_sriov_vf_vm(zkhandler, dom_node, network['source'])
# Obtain the RBD disk list using the common functions
ddisks = common.getDomainDisks(parsed_xml, {})

2
node-daemon/pvcnoded/Daemon.py
View File

@ -1129,7 +1129,7 @@ if enable_networking:
dev_uevent = vfh.readlines()
for line in dev_uevent:
if re.match(r'^PCI_SLOT_NAME=.*', line):
dev_pcie_path = line.split('=')[-1]
dev_pcie_path = line.rstrip().split('=')[-1]
except FileNotFoundError:
# Something must already be using the PCIe device
pass