From 64d1a37b3cc3be62a94d310776ee978d3f0fe1fd Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Mon, 21 Jun 2021 20:49:45 -0400 Subject: [PATCH] Add PCIe device paths to SR-IOV VF information This will be used when adding VM network interfaces of type hostdev. --- daemon-common/migrations/versions/1.json | 2 +- daemon-common/network.py | 10 ++++++++ daemon-common/zkhandler.py | 5 ++++ node-daemon/pvcnoded/Daemon.py | 30 +++++++++++++++++++++++- 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/daemon-common/migrations/versions/1.json b/daemon-common/migrations/versions/1.json index 70f5e23f..5ccc1bf0 100644 --- a/daemon-common/migrations/versions/1.json +++ b/daemon-common/migrations/versions/1.json @@ -1 +1 @@ -{"version": "1", "root": "", "base": {"root": "", "schema": "/schema", "schema.version": "/schema/version", "config": "/config", "config.maintenance": "/config/maintenance", "config.primary_node": "/config/primary_node", "config.primary_node.sync_lock": "/config/primary_node/sync_lock", "config.upstream_ip": "/config/upstream_ip", "config.migration_target_selector": "/config/migration_target_selector", "cmd": "/cmd", "cmd.node": "/cmd/nodes", "cmd.domain": "/cmd/domains", "cmd.ceph": "/cmd/ceph", "node": "/nodes", "domain": "/domains", "network": "/networks", "storage": "/ceph", "storage.util": "/ceph/util", "osd": "/ceph/osds", "pool": "/ceph/pools", "volume": "/ceph/volumes", "snapshot": "/ceph/snapshots"}, "node": {"name": "", "keepalive": "/keepalive", "mode": "/daemonmode", "data.active_schema": "/activeschema", "data.latest_schema": "/latestschema", "data.static": "/staticdata", "running_domains": "/runningdomains", "count.provisioned_domains": "/domainscount", "count.networks": "/networkscount", "state.daemon": "/daemonstate", "state.router": "/routerstate", "state.domain": "/domainstate", "cpu.load": "/cpuload", "vcpu.allocated": "/vcpualloc", "memory.total": "/memtotal", "memory.used": "/memused", "memory.free": "/memfree", "memory.allocated": "/memalloc", "memory.provisioned": "/memprov", "ipmi.hostname": "/ipmihostname", "ipmi.username": "/ipmiusername", "ipmi.password": "/ipmipassword", "sriov": "/sriov", "sriov.pf": "/sriov/pf", "sriov.vf": "/sriov/vf"}, "sriov_pf": {"phy": "", "mtu": "/mtu", "vfcount": "/vfcount"}, "sriov_vf": {"phy": "", "pf": "/pf", "mtu": "/mtu", "mac": "/mac", "config": "/config", "config.vlan_id": "/config/vlan_id", "config.vlan_qos": "/config/vlan_qos", "config.tx_rate_min": "/config/tx_rate_min", "config.tx_rate_max": "/config/tx_rate_max", "config.spoof_check": "/config/spoof_check", "config.link_state": "/config/link_state", "config.trust": "/config/trust", "config.query_rss": "/config/query_rss", "used": "/used", "used_by": "/used_by"}, "domain": {"name": "", "xml": "/xml", "state": "/state", "profile": "/profile", "stats": "/stats", "node": "/node", "last_node": "/lastnode", "failed_reason": "/failedreason", "storage.volumes": "/rbdlist", "console.log": "/consolelog", "console.vnc": "/vnc", "meta.autostart": "/node_autostart", "meta.migrate_method": "/migration_method", "meta.node_selector": "/node_selector", "meta.node_limit": "/node_limit", "migrate.sync_lock": "/migrate_sync_lock"}, "network": {"vni": "", "type": "/nettype", "rule": "/firewall_rules", "rule.in": "/firewall_rules/in", "rule.out": "/firewall_rules/out", "nameservers": "/name_servers", "domain": "/domain", "reservation": "/dhcp4_reservations", "lease": "/dhcp4_leases", "ip4.gateway": "/ip4_gateway", "ip4.network": "/ip4_network", "ip4.dhcp": "/dhcp4_flag", "ip4.dhcp_start": "/dhcp4_start", "ip4.dhcp_end": "/dhcp4_end", "ip6.gateway": "/ip6_gateway", "ip6.network": "/ip6_network", "ip6.dhcp": "/dhcp6_flag"}, "reservation": {"mac": "", "ip": "/ipaddr", "hostname": "/hostname"}, "lease": {"mac": "", "ip": "/ipaddr", "hostname": "/hostname", "expiry": "/expiry", "client_id": "/clientid"}, "rule": {"description": "", "rule": "/rule", "order": "/order"}, "osd": {"id": "", "node": "/node", "device": "/device", "stats": "/stats"}, "pool": {"name": "", "pgs": "/pgs", "stats": "/stats"}, "volume": {"name": "", "stats": "/stats"}, "snapshot": {"name": "", "stats": "/stats"}} \ No newline at end of file +{"version": "1", "root": "", "base": {"root": "", "schema": "/schema", "schema.version": "/schema/version", "config": "/config", "config.maintenance": "/config/maintenance", "config.primary_node": "/config/primary_node", "config.primary_node.sync_lock": "/config/primary_node/sync_lock", "config.upstream_ip": "/config/upstream_ip", "config.migration_target_selector": "/config/migration_target_selector", "cmd": "/cmd", "cmd.node": "/cmd/nodes", "cmd.domain": "/cmd/domains", "cmd.ceph": "/cmd/ceph", "node": "/nodes", "domain": "/domains", "network": "/networks", "storage": "/ceph", "storage.util": "/ceph/util", "osd": "/ceph/osds", "pool": "/ceph/pools", "volume": "/ceph/volumes", "snapshot": "/ceph/snapshots"}, "node": {"name": "", "keepalive": "/keepalive", "mode": "/daemonmode", "data.active_schema": "/activeschema", "data.latest_schema": "/latestschema", "data.static": "/staticdata", "running_domains": "/runningdomains", "count.provisioned_domains": "/domainscount", "count.networks": "/networkscount", "state.daemon": "/daemonstate", "state.router": "/routerstate", "state.domain": "/domainstate", "cpu.load": "/cpuload", "vcpu.allocated": "/vcpualloc", "memory.total": "/memtotal", "memory.used": "/memused", "memory.free": "/memfree", "memory.allocated": "/memalloc", "memory.provisioned": "/memprov", "ipmi.hostname": "/ipmihostname", "ipmi.username": "/ipmiusername", "ipmi.password": "/ipmipassword", "sriov": "/sriov", "sriov.pf": "/sriov/pf", "sriov.vf": "/sriov/vf"}, "sriov_pf": {"phy": "", "mtu": "/mtu", "vfcount": "/vfcount"}, "sriov_vf": {"phy": "", "pf": "/pf", "mtu": "/mtu", "mac": "/mac", "config": "/config", "config.vlan_id": "/config/vlan_id", "config.vlan_qos": "/config/vlan_qos", "config.tx_rate_min": "/config/tx_rate_min", "config.tx_rate_max": "/config/tx_rate_max", "config.spoof_check": "/config/spoof_check", "config.link_state": "/config/link_state", "config.trust": "/config/trust", "config.query_rss": "/config/query_rss", "pci": "/pci", "pci.domain": "/pci/domain", "pci.bus": "/pci/bus", "pci.slot": "/pci/slot", "pci.function": "/pci/function", "used": "/used", "used_by": "/used_by"}, "domain": {"name": "", "xml": "/xml", "state": "/state", "profile": "/profile", "stats": "/stats", "node": "/node", "last_node": "/lastnode", "failed_reason": "/failedreason", "storage.volumes": "/rbdlist", "console.log": "/consolelog", "console.vnc": "/vnc", "meta.autostart": "/node_autostart", "meta.migrate_method": "/migration_method", "meta.node_selector": "/node_selector", "meta.node_limit": "/node_limit", "migrate.sync_lock": "/migrate_sync_lock"}, "network": {"vni": "", "type": "/nettype", "rule": "/firewall_rules", "rule.in": "/firewall_rules/in", "rule.out": "/firewall_rules/out", "nameservers": "/name_servers", "domain": "/domain", "reservation": "/dhcp4_reservations", "lease": "/dhcp4_leases", "ip4.gateway": "/ip4_gateway", "ip4.network": "/ip4_network", "ip4.dhcp": "/dhcp4_flag", "ip4.dhcp_start": "/dhcp4_start", "ip4.dhcp_end": "/dhcp4_end", "ip6.gateway": "/ip6_gateway", "ip6.network": "/ip6_network", "ip6.dhcp": "/dhcp6_flag"}, "reservation": {"mac": "", "ip": "/ipaddr", "hostname": "/hostname"}, "lease": {"mac": "", "ip": "/ipaddr", "hostname": "/hostname", "expiry": "/expiry", "client_id": "/clientid"}, "rule": {"description": "", "rule": "/rule", "order": "/order"}, "osd": {"id": "", "node": "/node", "device": "/device", "stats": "/stats"}, "pool": {"name": "", "pgs": "/pgs", "stats": "/stats"}, "volume": {"name": "", "stats": "/stats"}, "snapshot": {"name": "", "stats": "/stats"}} \ No newline at end of file diff --git a/daemon-common/network.py b/daemon-common/network.py index aedadb6a..7a4e48d9 100644 --- a/daemon-common/network.py +++ b/daemon-common/network.py @@ -690,6 +690,10 @@ def getSRIOVVFInformation(zkhandler, node, vf): spoof_check = zkhandler.read(('node.sriov.vf', node, 'sriov_vf.config.spoof_check', vf)) trust = zkhandler.read(('node.sriov.vf', node, 'sriov_vf.config.trust', vf)) query_rss = zkhandler.read(('node.sriov.vf', node, 'sriov_vf.config.query_rss', vf)) + pci_domain = zkhandler.read(('node.sriov.vf', node, 'sriov_vf.pci.domain', vf)) + pci_bus = zkhandler.read(('node.sriov.vf', node, 'sriov_vf.pci.bus', vf)) + pci_slot = zkhandler.read(('node.sriov.vf', node, 'sriov_vf.pci.slot', vf)) + pci_function = zkhandler.read(('node.sriov.vf', node, 'sriov_vf.pci.function', vf)) used = zkhandler.read(('node.sriov.vf', node, 'sriov_vf.used', vf)) used_by_domain = zkhandler.read(('node.sriov.vf', node, 'sriov_vf.used_by', vf)) @@ -708,6 +712,12 @@ def getSRIOVVFInformation(zkhandler, node, vf): 'trust': trust, 'query_rss': query_rss, }, + 'pci': { + 'domain': pci_domain, + 'bus': pci_bus, + 'slot': pci_slot, + 'function': pci_function, + }, 'usage': { 'used': used, 'domain': used_by_domain, diff --git a/daemon-common/zkhandler.py b/daemon-common/zkhandler.py index 5bcffa6f..5ecb1cbf 100644 --- a/daemon-common/zkhandler.py +++ b/daemon-common/zkhandler.py @@ -509,6 +509,11 @@ class ZKSchema(object): 'config.link_state': '/config/link_state', 'config.trust': '/config/trust', 'config.query_rss': '/config/query_rss', + 'pci': '/pci', + 'pci.domain': '/pci/domain', + 'pci.bus': '/pci/bus', + 'pci.slot': '/pci/slot', + 'pci.function': '/pci/function', 'used': '/used', 'used_by': '/used_by' }, diff --git a/node-daemon/pvcnoded/Daemon.py b/node-daemon/pvcnoded/Daemon.py index 14a45791..57c6ab24 100644 --- a/node-daemon/pvcnoded/Daemon.py +++ b/node-daemon/pvcnoded/Daemon.py @@ -1121,8 +1121,30 @@ if enable_networking: # 'query_rss_en': False # } vfphy = '{}v{}'.format(pf, vf['vf']) + + # Get the PCIe bus information + dev_pcie_path = None + try: + with open('/sys/class/net/{}/device/uevent'.format(vfphy)) as vfh: + dev_uevent = vfh.readlines() + for line in dev_uevent: + if re.match(r'^PCI_SLOT_NAME=.*', line): + dev_pcie_path = line.split('=')[-1] + except FileNotFoundError: + # Something must already be using the PCIe device + pass + # Add the VF to Zookeeper if it does not yet exist if not zkhandler.exists(('node.sriov.vf', myhostname, 'sriov_vf', vfphy)): + if dev_pcie_path is not None: + pcie_domain, pcie_bus, pcie_slot, pcie_function = re.split(r':|\.', dev_pcie_path) + else: + # We can't add the device - for some reason we can't get any information on its PCIe bus path, + # so just ignore this one, and continue. + # This shouldn't happen under any real circumstances, unless the admin tries to attach a non-existent + # VF to a VM manually, then goes ahead and adds that VF to the system with the VM running. + continue + zkhandler.write([ (('node.sriov.vf', myhostname, 'sriov_vf', vfphy), ''), (('node.sriov.vf', myhostname, 'sriov_vf.pf', vfphy), pf), @@ -1137,9 +1159,15 @@ if enable_networking: (('node.sriov.vf', myhostname, 'sriov_vf.config.link_state', vfphy), vf['link_state']), (('node.sriov.vf', myhostname, 'sriov_vf.config.trust', vfphy), vf['trust']), (('node.sriov.vf', myhostname, 'sriov_vf.config.query_rss', vfphy), vf['query_rss_en']), + (('node.sriov.vf', myhostname, 'sriov_vf.pci', vfphy), ''), + (('node.sriov.vf', myhostname, 'sriov_vf.pci.domain', vfphy), pcie_domain), + (('node.sriov.vf', myhostname, 'sriov_vf.pci.bus', vfphy), pcie_bus), + (('node.sriov.vf', myhostname, 'sriov_vf.pci.slot', vfphy), pcie_slot), + (('node.sriov.vf', myhostname, 'sriov_vf.pci.function', vfphy), pcie_function), (('node.sriov.vf', myhostname, 'sriov_vf.used', vfphy), False), (('node.sriov.vf', myhostname, 'sriov_vf.used_by', vfphy), ''), ]) + # Append the device to the list of VFs sriov_vf_list.append(vfphy) @@ -1660,7 +1688,7 @@ def collect_vm_stats(queue): domain_network_stats = [] for interface in tree.findall('devices/interface'): interface_type = interface.get('type') - if interface_type in ['hostdev']: + if interface_type not in ['bridge']: continue interface_name = interface.find('target').get('dev') interface_bridge = interface.find('source').get('bridge')