parallelvirtualcluster
/
pvc-bootstrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix various delays and bugs

This commit is contained in:
Joshua Boniface 2023-01-16 19:37:19 +00:00
parent 83118331a5
commit 090e39694c
2 changed files with 46 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bootstrap-daemon/pvcbootstrapd/lib/db.py
View File

@ -67,7 +67,7 @@ def init_database(config):
(id INTEGER PRIMARY KEY AUTOINCREMENT, (id INTEGER PRIMARY KEY AUTOINCREMENT,
cluster INTEGER NOT NULL, cluster INTEGER NOT NULL,
state TEXT NOT NULL, state TEXT NOT NULL,
name TEXT UNIQUE NOT NULL, name TEXT NOT NULL,
nodeid INTEGER NOT NULL, nodeid INTEGER NOT NULL,
bmc_macaddr TEXT NOT NULL, bmc_macaddr TEXT NOT NULL,
bmc_ipaddr TEXT NOT NULL, bmc_ipaddr TEXT NOT NULL,

85
bootstrap-daemon/pvcbootstrapd/lib/redfish.py
View File

@ -748,8 +748,8 @@ def redfish_init(config, cspec, data):
return return
notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: Logged in to Redfish for host {cspec_fqdn} at {bmc_host}") notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: Logged in to Redfish for host {cspec_fqdn} at {bmc_host}")
logger.info("Waiting 5 seconds for system normalization") logger.info("Waiting 30 seconds for system normalization")
sleep(5) sleep(30)
logger.info("Characterizing node...") logger.info("Characterizing node...")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Beginning Redfish characterization of host {cspec_fqdn} at {bmc_host}") notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Beginning Redfish characterization of host {cspec_fqdn} at {bmc_host}")
@ -792,24 +792,29 @@ def redfish_init(config, cspec, data):
try: try:
ethernet_root = system_detail["EthernetInterfaces"]["@odata.id"].rstrip("/") ethernet_root = system_detail["EthernetInterfaces"]["@odata.id"].rstrip("/")
ethernet_detail = session.get(ethernet_root) ethernet_detail = session.get(ethernet_root)
logger.debug(f"Found Ethernet detail: {ethernet_detail}")
embedded_ethernet_detail_members = [e for e in ethernet_detail["Members"] if "Embedded" in e["@odata.id"]] embedded_ethernet_detail_members = [e for e in ethernet_detail["Members"] if "Embedded" in e["@odata.id"]]
embedded_ethernet_detail_members.sort(key = lambda k: k["@odata.id"]) embedded_ethernet_detail_members.sort(key = lambda k: k["@odata.id"])
logger.debug(f"Found Ethernet members: {embedded_ethernet_detail_members}")
first_interface_root = embedded_ethernet_detail_members[0]["@odata.id"].rstrip("/") first_interface_root = embedded_ethernet_detail_members[0]["@odata.id"].rstrip("/")
first_interface_detail = session.get(first_interface_root) first_interface_detail = session.get(first_interface_root)
# Something went wrong, so fall back # Something went wrong, so fall back
except KeyError: except Exception:
first_interface_detail = dict() first_interface_detail = dict()
logger.debug(f"First interface detail: {first_interface_detail}")
logger.debug(f"HostCorrelation detail: {system_detail.get('HostCorrelation', {})}")
# Try to get the MAC address directly from the interface detail (Redfish standard) # Try to get the MAC address directly from the interface detail (Redfish standard)
logger.debug("Try to get the MAC address directly from the interface detail (Redfish standard)")
if first_interface_detail.get("MACAddress") is not None: if first_interface_detail.get("MACAddress") is not None:
logger.debug("Try to get the MAC address directly from the interface detail (Redfish standard)")
bootstrap_mac_address = first_interface_detail["MACAddress"].strip().lower() bootstrap_mac_address = first_interface_detail["MACAddress"].strip().lower()
# Try to get the MAC address from the HostCorrelation->HostMACAddress (HP DL360x G8) # Try to get the MAC address from the HostCorrelation->HostMACAddress (HP DL360x G8)
elif len(system_detail.get("HostCorrelation", {}).get("HostMACAddress", [])) > 0: elif len(system_detail.get("HostCorrelation", {}).get("HostMACAddress", [])) > 0:
logger.debug("Try to get the MAC address from the HostCorrelation (HP iLO)")
bootstrap_mac_address = ( bootstrap_mac_address = (
system_detail["HostCorrelation"]["HostMACAddress"][0].strip().lower() system_detail["HostCorrelation"]["HostMACAddress"][0].strip().lower()
) )
# We can't find it, so use a dummy value # We can't find it, so abort
else: else:
logger.error("Could not find a valid MAC address for the bootstrap interface.") logger.error("Could not find a valid MAC address for the bootstrap interface.")
return return
@ -878,43 +883,43 @@ def redfish_init(config, cspec, data):
return return
# Adjust any BIOS settings # Adjust any BIOS settings
logger.info("Adjusting BIOS settings...") if len(cspec_node["bmc"].get("bios_settings", {}).items()) > 0:
try: logger.info("Adjusting BIOS settings...")
bios_root = system_detail.get("Bios", {}).get("@odata.id") try:
if bios_root is not None: bios_root = system_detail.get("Bios", {}).get("@odata.id")
bios_detail = session.get(bios_root) if bios_root is not None:
bios_attributes = list(bios_detail["Attributes"].keys()) bios_detail = session.get(bios_root)
for setting, value in cspec_node["bmc"].get("bios_settings", {}).items(): bios_attributes = list(bios_detail["Attributes"].keys())
if setting not in bios_attributes: for setting, value in cspec_node["bmc"].get("bios_settings", {}).items():
continue if setting not in bios_attributes:
continue
payload = {"Attributes": {setting: value}} payload = {"Attributes": {setting: value}}
session.patch(f"{bios_root}/Settings", payload) session.patch(f"{bios_root}/Settings", payload)
except Exception as e: except Exception as e:
notifications.send_webhook(config, "failure", f"Cluster {cspec_cluster}: Failed to set BIOS settings for host {cspec_fqdn} at {bmc_host}. Check pvcbootstrapd logs and reset this host's BMC to retry.") notifications.send_webhook(config, "failure", f"Cluster {cspec_cluster}: Failed to set BIOS settings for host {cspec_fqdn} at {bmc_host}. Check pvcbootstrapd logs and reset this host's BMC to retry.")
logger.error(f"Cluster {cspec_cluster}: Failed to set BIOS settings for host {cspec_fqdn} at {bmc_host}: {e}") logger.error(f"Cluster {cspec_cluster}: Failed to set BIOS settings for host {cspec_fqdn} at {bmc_host}: {e}")
logger.error("Aborting Redfish configuration; reset BMC to retry.") logger.error("Aborting Redfish configuration; reset BMC to retry.")
del session del session
return return
# Adjust any Manager settings # Adjust any Manager settings
logger.info("Adjusting Manager settings...") if len(cspec_node["bmc"].get("manager_settings", {}).items()) > 0:
try: logger.info("Adjusting Manager settings...")
mgrattribute_root = f"{manager_root}/Attributes" try:
mgrattribute_detail = session.get(mgrattribute_root) mgrattribute_root = f"{manager_root}/Attributes"
mgrattribute_attributes = list(mgrattribute_detail["Attributes"].keys()) mgrattribute_detail = session.get(mgrattribute_root)
for setting, value in cspec_node["bmc"].get("manager_settings", {}).items(): mgrattribute_attributes = list(mgrattribute_detail["Attributes"].keys())
if setting not in mgrattribute_attributes: for setting, value in cspec_node["bmc"].get("manager_settings", {}).items():
continue if setting not in mgrattribute_attributes:
continue
payload = {"Attributes": {setting: value}} payload = {"Attributes": {setting: value}}
session.patch(mgrattribute_root, payload) session.patch(mgrattribute_root, payload)
except Exception as e: except Exception as e:
notifications.send_webhook(config, "failure", f"Cluster {cspec_cluster}: Failed to set BMC settings for host {cspec_fqdn} at {bmc_host}. Check pvcbootstrapd logs and reset this host's BMC to retry.") notifications.send_webhook(config, "failure", f"Cluster {cspec_cluster}: Failed to set BMC settings for host {cspec_fqdn} at {bmc_host}. Check pvcbootstrapd logs and reset this host's BMC to retry.")
logger.error(f"Cluster {cspec_cluster}: Failed to set BMC settings for host {cspec_fqdn} at {bmc_host}: {e}") logger.error(f"Cluster {cspec_cluster}: Failed to set BMC settings for host {cspec_fqdn} at {bmc_host}: {e}")
logger.error("Aborting Redfish configuration; reset BMC to retry.") logger.error("Aborting Redfish configuration; reset BMC to retry.")
del session del session
return return
# Set boot override to Pxe for the installer boot # Set boot override to Pxe for the installer boot
logger.info("Setting temporary PXE boot...") logger.info("Setting temporary PXE boot...")