parallelvirtualcluster/pvc
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 115 Wiki Activity

Add support for SR-IOV NICs to VMs

Browse Source
This commit is contained in:
Joshua Boniface
2021-06-21 22:21:54 -04:00
parent 93c2fdec93
commit eeb83da97d
7 changed files with 245 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
client-cli/cli_lib/network.py
View File

@@ -1034,6 +1034,12 @@ def format_info_sriov_vf(config, vf_information, node):
ainformation.append('{}VF User Trust:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), getColour(vf_information['config']['trust']), vf_information['config']['trust'], ansiprint.end()))
ainformation.append('{}Query RSS Config:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), getColour(vf_information['config']['query_rss']), vf_information['config']['query_rss'], ansiprint.end()))
ainformation.append('')
# PCIe bus information
ainformation.append('{}PCIe domain:{} {}'.format(ansiprint.purple(), ansiprint.end(), vf_information['pci']['domain']))
ainformation.append('{}PCIe bus:{} {}'.format(ansiprint.purple(), ansiprint.end(), vf_information['pci']['bus']))
ainformation.append('{}PCIe slot:{} {}'.format(ansiprint.purple(), ansiprint.end(), vf_information['pci']['slot']))
ainformation.append('{}PCIe function:{} {}'.format(ansiprint.purple(), ansiprint.end(), vf_information['pci']['function']))
ainformation.append('')
# Usage information
ainformation.append('{}VF Used:{} {}{}{}'.format(ansiprint.purple(), ansiprint.end(), getColour(vf_information['usage']['used']), vf_information['usage']['used'], ansiprint.end()))
if vf_information['usage']['used'] == 'True' and vm_information is not None:

171
client-cli/cli_lib/vm.py
View File

@@ -501,7 +501,7 @@ def format_vm_memory(config, name, memory):
return '\n'.join(output_list)
def vm_networks_add(config, vm, network, macaddr, model, restart):
def vm_networks_add(config, vm, network, macaddr, model, sriov, sriov_mode, restart):
"""
Add a new network to the VM
@@ -514,17 +514,19 @@ def vm_networks_add(config, vm, network, macaddr, model, restart):
from random import randint
import cli_lib.network as pvc_network
# Verify that the provided network is valid
retcode, retdata = pvc_network.net_info(config, network)
if not retcode:
# Ignore the three special networks
if network not in ['upstream', 'cluster', 'storage']:
return False, "Network {} is not present in the cluster.".format(network)
# Verify that the provided network is valid (not in SR-IOV mode)
if not sriov:
retcode, retdata = pvc_network.net_info(config, network)
if not retcode:
# Ignore the three special networks
if network not in ['upstream', 'cluster', 'storage']:
return False, "Network {} is not present in the cluster.".format(network)
if network in ['upstream', 'cluster', 'storage']:
br_prefix = 'br'
else:
br_prefix = 'vmbr'
# Set the bridge prefix
if network in ['upstream', 'cluster', 'storage']:
br_prefix = 'br'
else:
br_prefix = 'vmbr'
status, domain_information = vm_info(config, vm)
if not status:
@@ -551,24 +553,73 @@ def vm_networks_add(config, vm, network, macaddr, model, restart):
octetC=random_octet_C
)
device_string = '<interface type="bridge"><mac address="{macaddr}"/><source bridge="{bridge}"/><model type="{model}"/></interface>'.format(
macaddr=macaddr,
bridge="{}{}".format(br_prefix, network),
model=model
)
# Add an SR-IOV network
if sriov:
valid, sriov_vf_information = pvc_network.net_sriov_vf_info(config, domain_information['node'], network)
if not valid:
return False, 'Specified SR-IOV VF "{}" does not exist on VM node "{}".'.format(network, domain_information['node'])
# Add a hostdev (direct PCIe) SR-IOV network
if sriov_mode == 'hostdev':
bus_address = 'domain="0x{pci_domain}" bus="0x{pci_bus}" slot="0x{pci_slot}" function="0x{pci_function}"'.format(
pci_domain=sriov_vf_information['pci']['domain'],
pci_bus=sriov_vf_information['pci']['bus'],
pci_slot=sriov_vf_information['pci']['slot'],
pci_function=sriov_vf_information['pci']['function'],
)
device_string = '<interface type="hostdev"><mac address="{macaddr}"/><source><address type="pci" {bus_address}/></source><sriov_device>{network}</sriov_device></interface>'.format(
macaddr=macaddr,
bus_address=bus_address,
network=network
)
# Add a macvtap SR-IOV network
elif sriov_mode == 'macvtap':
device_string = '<interface type="direct"><mac address="{macaddr}"/><source dev="{network}" mode="passthrough"/></interface>'.format(
macaddr=macaddr,
network=network
)
else:
return False, "ERROR: Invalid SR-IOV mode specified."
# Add a normal bridged PVC network
else:
device_string = '<interface type="bridge"><mac address="{macaddr}"/><source bridge="{bridge}"/><model type="{model}"/></interface>'.format(
macaddr=macaddr,
bridge="{}{}".format(br_prefix, network),
model=model
)
device_xml = fromstring(device_string)
last_interface = None
all_interfaces = parsed_xml.devices.find('interface')
if all_interfaces is None:
all_interfaces = []
for interface in all_interfaces:
last_interface = re.match(r'[vm]*br([0-9a-z]+)', interface.source.attrib.get('bridge')).group(1)
if last_interface == network:
return False, 'Network {} is already configured for VM {}.'.format(network, vm)
if last_interface is not None:
for interface in parsed_xml.devices.find('interface'):
if last_interface == re.match(r'[vm]*br([0-9a-z]+)', interface.source.attrib.get('bridge')).group(1):
if sriov:
if sriov_mode == 'hostdev':
if interface.attrib.get('type') == 'hostdev':
interface_address = 'domain="{pci_domain}" bus="{pci_bus}" slot="{pci_slot}" function="{pci_function}"'.format(
interface.source.address.attrib.get('domain'),
interface.source.address.attrib.get('bus'),
interface.source.address.attrib.get('slot'),
interface.source.address.attrib.get('function')
)
if interface_address == bus_address:
return False, 'Network "{}" is already configured for VM "{}".'.format(network, vm)
elif sriov_mode == 'macvtap':
if interface.attrib.get('type') == 'direct':
interface_dev = interface.source.attrib.get('dev')
if interface_dev == network:
return False, 'Network "{}" is already configured for VM "{}".'.format(network, vm)
else:
if interface.attrib.get('type') == 'bridge':
interface_vni = re.match(r'[vm]*br([0-9a-z]+)', interface.source.attrib.get('bridge')).group(1)
if interface_vni == network:
return False, 'Network "{}" is already configured for VM "{}".'.format(network, vm)
# Add the interface at the end of the list (or, right above emulator)
if len(all_interfaces) > 0:
for idx, interface in enumerate(parsed_xml.devices.find('interface')):
if idx == len(all_interfaces) - 1:
interface.addnext(device_xml)
else:
parsed_xml.devices.find('emulator').addprevious(device_xml)
@@ -581,7 +632,7 @@ def vm_networks_add(config, vm, network, macaddr, model, restart):
return vm_modify(config, vm, new_xml, restart)
def vm_networks_remove(config, vm, network, restart):
def vm_networks_remove(config, vm, network, sriov, restart):
"""
Remove a network to the VM
@@ -605,17 +656,33 @@ def vm_networks_remove(config, vm, network, restart):
except Exception:
return False, 'ERROR: Failed to parse XML data.'
changed = False
for interface in parsed_xml.devices.find('interface'):
if_vni = re.match(r'[vm]*br([0-9a-z]+)', interface.source.attrib.get('bridge')).group(1)
if network == if_vni:
interface.getparent().remove(interface)
if sriov:
if interface.attrib.get('type') == 'hostdev':
if_dev = str(interface.sriov_device)
if network == if_dev:
interface.getparent().remove(interface)
changed = True
elif interface.attrib.get('type') == 'direct':
if_dev = str(interface.source.attrib.get('dev'))
if network == if_dev:
interface.getparent().remove(interface)
changed = True
else:
if_vni = re.match(r'[vm]*br([0-9a-z]+)', interface.source.attrib.get('bridge')).group(1)
if network == if_vni:
interface.getparent().remove(interface)
changed = True
if changed:
try:
new_xml = tostring(parsed_xml, pretty_print=True)
except Exception:
return False, 'ERROR: Failed to dump XML data.'
try:
new_xml = tostring(parsed_xml, pretty_print=True)
except Exception:
return False, 'ERROR: Failed to dump XML data.'
return vm_modify(config, vm, new_xml, restart)
return vm_modify(config, vm, new_xml, restart)
else:
return False, 'ERROR: Network "{}" does not exist on VM.'.format(network)
def vm_networks_get(config, vm):
@@ -1178,7 +1245,7 @@ def format_info(config, domain_information, long_output):
cluster_net_list = call_api(config, 'get', '/network').json()
for net in domain_information['networks']:
net_vni = net['vni']
if net_vni not in ['cluster', 'storage', 'upstream'] and not re.match(r'^e.*', net_vni):
if net_vni not in ['cluster', 'storage', 'upstream'] and not re.match(r'^macvtap:.*', net_vni) and not re.match(r'^hostdev:.*', net_vni):
if int(net_vni) not in [net['vni'] for net in cluster_net_list]:
net_list.append(ansiprint.red() + net_vni + ansiprint.end() + ' [invalid]')
else:
@@ -1210,17 +1277,31 @@ def format_info(config, domain_information, long_output):
width=name_length
))
ainformation.append('')
ainformation.append('{}Interfaces:{} {}ID Type Source Model MAC Data (r/w) Packets (r/w) Errors (r/w){}'.format(ansiprint.purple(), ansiprint.end(), ansiprint.bold(), ansiprint.end()))
ainformation.append('{}Interfaces:{} {}ID Type Source Model MAC Data (r/w) Packets (r/w) Errors (r/w){}'.format(ansiprint.purple(), ansiprint.end(), ansiprint.bold(), ansiprint.end()))
for net in domain_information['networks']:
ainformation.append(' {0: <3} {1: <7} {2: <10} {3: <8} {4: <18} {5: <12} {6: <15} {7: <12}'.format(
net_type = net['type']
net_source = net['source']
net_mac = net['mac']
if net_type in ['direct', 'hostdev']:
net_model = 'N/A'
net_bytes = 'N/A'
net_packets = 'N/A'
net_errors = 'N/A'
elif net_type in ['bridge']:
net_model = net['model']
net_bytes = '/'.join([str(format_bytes(net.get('rd_bytes', 0))), str(format_bytes(net.get('wr_bytes', 0)))])
net_packets = '/'.join([str(format_metric(net.get('rd_packets', 0))), str(format_metric(net.get('wr_packets', 0)))])
net_errors = '/'.join([str(format_metric(net.get('rd_errors', 0))), str(format_metric(net.get('wr_errors', 0)))])
ainformation.append(' {0: <3} {1: <8} {2: <12} {3: <8} {4: <18} {5: <12} {6: <15} {7: <12}'.format(
domain_information['networks'].index(net),
net['type'],
net['source'],
net['model'],
net['mac'],
'/'.join([str(format_bytes(net.get('rd_bytes', 0))), str(format_bytes(net.get('wr_bytes', 0)))]),
'/'.join([str(format_metric(net.get('rd_packets', 0))), str(format_metric(net.get('wr_packets', 0)))]),
'/'.join([str(format_metric(net.get('rd_errors', 0))), str(format_metric(net.get('wr_errors', 0)))]),
net_type,
net_source,
net_model,
net_mac,
net_bytes,
net_packets,
net_errors
))
# Controller list
ainformation.append('')
@@ -1259,7 +1340,7 @@ def format_list(config, vm_list, raw):
vm_nets_length = 9
vm_ram_length = 8
vm_vcpu_length = 6
vm_node_length = 8
vm_node_length = 5
vm_migrated_length = 10
for domain_information in vm_list:
net_list = getNiceNetID(domain_information)
@@ -1335,7 +1416,7 @@ def format_list(config, vm_list, raw):
cluster_net_list = call_api(config, 'get', '/network').json()
vm_net_colour = ''
for net_vni in net_list:
if net_vni not in ['cluster', 'storage', 'upstream'] and not re.match(r'^e.*', net_vni):
if net_vni not in ['cluster', 'storage', 'upstream'] and not re.match(r'^macvtap:.*', net_vni) and not re.match(r'^hostdev:.*', net_vni):
if int(net_vni) not in [net['vni'] for net in cluster_net_list]:
vm_net_colour = ansiprint.red()

44
client-cli/pvc.py
View File

@@ -1309,15 +1309,24 @@ def vm_network_get(domain, raw):
'domain'
)
@click.argument(
'vni'
'net'
)
@click.option(
'-a', '--macaddr', 'macaddr', default=None,
help='Use this MAC address instead of random generation; must be a valid MAC address in colon-deliniated format.'
help='Use this MAC address instead of random generation; must be a valid MAC address in colon-delimited format.'
)
@click.option(
'-m', '--model', 'model', default='virtio',
help='The model for the interface; must be a valid libvirt model.'
help='The model for the interface; must be a valid libvirt model. Not used for SR-IOV NETs.'
)
@click.option(
'-s', '--sriov', 'sriov', is_flag=True, default=False,
help='Identify that NET is an SR-IOV device name and not a VNI. Required for adding SR-IOV NETs.'
)
@click.option(
'-d', '--sriov-mode', 'sriov_mode', default='hostdev',
type=click.Choice(['hostdev', 'macvtap']),
help='For SR-IOV NETs, the SR-IOV network device mode.'
)
@click.option(
'-r', '--restart', 'restart', is_flag=True, default=False,
@@ -1329,9 +1338,18 @@ def vm_network_get(domain, raw):
help='Confirm the restart'
)
@cluster_req
def vm_network_add(domain, vni, macaddr, model, restart, confirm_flag):
def vm_network_add(domain, net, macaddr, model, sriov, sriov_mode, restart, confirm_flag):
"""
Add the network VNI to the virtual machine DOMAIN. Networks are always addded to the end of the current list of networks in the virtual machine.
Add the network NET to the virtual machine DOMAIN. Networks are always addded to the end of the current list of networks in the virtual machine.
NET may be a PVC network VNI, which is added as a bridged device, or a SR-IOV VF device connected in the given mode.
NOTE: Adding a SR-IOV network device in the "hostdev" mode has the following caveats:
1. The VM will not be able to be live migrated; it must be shut down to migrate between nodes. The VM metadata will be updated to force this.
2. If an identical SR-IOV VF device is not present on the target node, post-migration startup will fail. It may be prudent to use a node limit here.
"""
if restart and not confirm_flag and not config['unsafe']:
try:
@@ -1339,7 +1357,7 @@ def vm_network_add(domain, vni, macaddr, model, restart, confirm_flag):
except Exception:
restart = False
retcode, retmsg = pvc_vm.vm_networks_add(config, domain, vni, macaddr, model, restart)
retcode, retmsg = pvc_vm.vm_networks_add(config, domain, net, macaddr, model, sriov, sriov_mode, restart)
if retcode and not restart:
retmsg = retmsg + " Changes will be applied on next VM start/restart."
cleanup(retcode, retmsg)
@@ -1353,7 +1371,11 @@ def vm_network_add(domain, vni, macaddr, model, restart, confirm_flag):
'domain'
)
@click.argument(
'vni'
'net'
)
@click.option(
'-s', '--sriov', 'sriov', is_flag=True, default=False,
help='Identify that NET is an SR-IOV device name and not a VNI. Required for removing SR-IOV NETs.'
)
@click.option(
'-r', '--restart', 'restart', is_flag=True, default=False,
@@ -1365,9 +1387,11 @@ def vm_network_add(domain, vni, macaddr, model, restart, confirm_flag):
help='Confirm the restart'
)
@cluster_req
def vm_network_remove(domain, vni, restart, confirm_flag):
def vm_network_remove(domain, net, sriov, restart, confirm_flag):
"""
Remove the network VNI to the virtual machine DOMAIN.
Remove the network NET from the virtual machine DOMAIN.
NET may be a PVC network VNI, which is added as a bridged device, or a SR-IOV VF device connected in the given mode.
"""
if restart and not confirm_flag and not config['unsafe']:
try:
@@ -1375,7 +1399,7 @@ def vm_network_remove(domain, vni, restart, confirm_flag):
except Exception:
restart = False
retcode, retmsg = pvc_vm.vm_networks_remove(config, domain, vni, restart)
retcode, retmsg = pvc_vm.vm_networks_remove(config, domain, net, sriov, restart)
if retcode and not restart:
retmsg = retmsg + " Changes will be applied on next VM start/restart."
cleanup(retcode, retmsg)