parallelvirtualcluster/pvc-bootstrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add note message type and improve messages

Browse Source
This commit is contained in:
Joshua Boniface
2022-07-07 15:52:39 +00:00
parent 9a9caae923
commit fe6f34a3eb
8 changed files with 34 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
bootstrap-daemon/pvcbootstrapd.yaml.sample
View File

@@ -101,6 +101,7 @@ pvc:
action: post action: post
# Icons to use for various status types; embedded in the message with `{icon}` # Icons to use for various status types; embedded in the message with `{icon}`
icons: icons:
info: "❕" # A note about an event
begin: "🤞" # A task is beginning begin: "🤞" # A task is beginning
success: "✅" # A task succeeded success: "✅" # A task succeeded
failure: "❌" # A task failed failure: "❌" # A task failed

1
bootstrap-daemon/pvcbootstrapd.yaml.template
View File

@@ -36,6 +36,7 @@ pvc:
uri: https://mattermost.domain.tld/hooks/asecretstring uri: https://mattermost.domain.tld/hooks/asecretstring
action: post action: post
icons: icons:
info: "❕" # A note about an event
begin: "🤞" # A task is beginning begin: "🤞" # A task is beginning
success: "✅" # A task succeeded success: "✅" # A task succeeded
failure: "❌" # A task failed failure: "❌" # A task failed

8
bootstrap-daemon/pvcbootstrapd/Daemon.py
View File

@@ -248,7 +248,7 @@ def entrypoint():
print("|----------------------------------------------------------|") print("|----------------------------------------------------------|")
print("") print("")
notifications.send_webhook(config, "begin", "Starting up pvcbootstrapd") notifications.send_webhook(config, "info", "Initializing pvcbootstrapd")
# Initialize the database # Initialize the database
db.init_database(config) db.init_database(config)
@@ -261,7 +261,7 @@ def entrypoint():
if "--init-only" in argv: if "--init-only" in argv:
print("Successfully initialized pvcbootstrapd; exiting.") print("Successfully initialized pvcbootstrapd; exiting.")
notifications.send_webhook(config, "success", "Successfully initialized pvcbootstrapd") notifications.send_webhook(config, "info", "Successfully initialized pvcbootstrapd")
exit(0) exit(0)
# Start DNSMasq # Start DNSMasq
@@ -274,14 +274,14 @@ def entrypoint():
def term(signum="", frame=""): def term(signum="", frame=""):
print("Received TERM, exiting.") print("Received TERM, exiting.")
notifications.send_webhook(config, "completed", "Received TERM, exiting pvcbootstrapd") notifications.send_webhook(config, "info", "Received TERM, exiting pvcbootstrapd")
cleanup(0) cleanup(0)
signal.signal(signal.SIGTERM, term) signal.signal(signal.SIGTERM, term)
signal.signal(signal.SIGINT, term) signal.signal(signal.SIGINT, term)
signal.signal(signal.SIGQUIT, term) signal.signal(signal.SIGQUIT, term)
notifications.send_webhook(config, "success", "Started up pvcbootstrapd") notifications.send_webhook(config, "info", "Starting up pvcbootstrapd")
# Start Flask # Start Flask
pvcbootstrapd.app.run( pvcbootstrapd.app.run(

8
bootstrap-daemon/pvcbootstrapd/lib/ansible.py
View File

@@ -55,7 +55,7 @@ def run_bootstrap(config, cspec, cluster, nodes):
sleep(60) sleep(60)
logger.info("Starting Ansible bootstrap of cluster {cluster.name}") logger.info("Starting Ansible bootstrap of cluster {cluster.name}")
notifications.send_webhook(config, "begin", f"Starting Ansible bootstrap of cluster {cluster.name}") notifications.send_webhook(config, "begin", f"Cluster {cluster.name}: Starting Ansible bootstrap")
# Run the Ansible playbooks # Run the Ansible playbooks
with tempfile.TemporaryDirectory(prefix="pvc-ansible-bootstrap_") as pdir: with tempfile.TemporaryDirectory(prefix="pvc-ansible-bootstrap_") as pdir:
@@ -78,9 +78,9 @@ def run_bootstrap(config, cspec, cluster, nodes):
if r.rc == 0: if r.rc == 0:
git.commit_repository(config) git.commit_repository(config)
git.push_repository(config) git.push_repository(config)
notifications.send_webhook(config, "success", f"Completed Ansible bootstrap of cluster {cluster.name}") notifications.send_webhook(config, "success", f"Cluster {cluster.name}: Completed Ansible bootstrap")
else: else:
notifications.send_webhook(config, "failure", f"Failed Ansible bootstrap of cluster {cluster.name}; check pvcbootstrapd logs") notifications.send_webhook(config, "failure", f"Cluster {cluster.name}: Failed Ansible bootstrap; check pvcbootstrapd logs")
except Exception as e: except Exception as e:
logger.warning(f"Error: {e}") logger.warning(f"Error: {e}")
notifications.send_webhook(config, "failure", f"Failed Ansible bootstrap of cluster {cluster.name} with error {e}; check pvcbootstrapd logs") notifications.send_webhook(config, "failure", f"Cluster {cluster.name}: Failed Ansible bootstrap with error {e}; check pvcbootstrapd logs")

2
bootstrap-daemon/pvcbootstrapd/lib/git.py
View File

@@ -52,7 +52,7 @@ def init_repository(config):
g = git.cmd.Git(f"{config['ansible_path']}") g = git.cmd.Git(f"{config['ansible_path']}")
g.checkout(config["ansible_branch"]) g.checkout(config["ansible_branch"])
g.submodule("update", "--init", env=dict(GIT_SSH_COMMAND=git_ssh_cmd)) g.submodule("update", "--init", env=dict(GIT_SSH_COMMAND=git_ssh_cmd))
notifications.send_webhook(config, "success", "Successfully initialized Git repository") notifications.send_webhook(config, "success", "Successfully updated Git repository")
except Exception as e: except Exception as e:
print(f"Error: {e}") print(f"Error: {e}")

4
bootstrap-daemon/pvcbootstrapd/lib/hooks.py
View File

@@ -319,7 +319,7 @@ def run_hooks(config, cspec, cluster, nodes):
logger.info("Waiting 300s before starting hook run.") logger.info("Waiting 300s before starting hook run.")
sleep(300) sleep(300)
notifications.send_webhook(config, "begin", f"Running hook tasks for cluster {cluster.name}") notifications.send_webhook(config, "begin", f"Cluster {cluster.name}: Running post-setup hook tasks")
cluster_hooks = cspec["hooks"][cluster.name] cluster_hooks = cspec["hooks"][cluster.name]
@@ -363,4 +363,4 @@ def run_hooks(config, cspec, cluster, nodes):
}, },
) )
notifications.send_webhook(config, "success", f"Completed hook tasks for cluster {cluster.name}") notifications.send_webhook(config, "success", f"Cluster {cluster.name}: Completed post-setup hook tasks")

31
bootstrap-daemon/pvcbootstrapd/lib/lib.py
View File

@@ -51,7 +51,7 @@ def dnsmasq_checkin(config, data):
cspec = git.load_cspec_yaml(config) cspec = git.load_cspec_yaml(config)
is_in_bootstrap_map = True if data["macaddr"] in cspec["bootstrap"] else False is_in_bootstrap_map = True if data["macaddr"] in cspec["bootstrap"] else False
if is_in_bootstrap_map: if is_in_bootstrap_map:
notifications.send_webhook(config, "begin", f"New host checkin from MAC '{data['macaddr']}' as host {cspec['bootstrap'][data['macaddr']]['node']['fqdn']}") notifications.send_webhook(config, "info", f"New host checkin from MAC {data['macaddr']} as host {cspec['bootstrap'][data['macaddr']]['node']['fqdn']} in cluster {cspec['bootstrap'][data['macaddr']]['node']['cluster']}")
if ( if (
cspec["bootstrap"][data["macaddr"]]["bmc"].get("redfish", None) cspec["bootstrap"][data["macaddr"]]["bmc"].get("redfish", None)
is not None is not None
@@ -83,28 +83,29 @@ def host_checkin(config, data):
""" """
Handle checkins from the PVC node Handle checkins from the PVC node
""" """
logger.info(f"Registering checkin for host {data['hostname']}") logger.info(f"Registering checkin for {data['bmc_macaddr']}")
logger.debug(f"data = {data}") logger.debug(f"data = {data}")
cspec = git.load_cspec_yaml(config) cspec = git.load_cspec_yaml(config)
bmc_macaddr = data["bmc_macaddr"] bmc_macaddr = data["bmc_macaddr"]
cspec_cluster = cspec["bootstrap"][bmc_macaddr]["node"]["cluster"] cspec_cluster = cspec["bootstrap"][bmc_macaddr]["node"]["cluster"]
cspec_fqdn = cspec["bootstrap"][bmc_macaddr]["node"]["fqdn"]
if data["action"] in ["install-start"]: if data["action"] in ["install-start"]:
# Node install has started # Node install has started
logger.info(f"Registering install start for host {data['hostname']}") logger.info(f"Registering install start for host {cspec_fqdn}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering install start for host {data['hostname']}") notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: starting starting for host {cspec_fqdn}")
host.installer_init(config, cspec, data) host.installer_init(config, cspec, data)
elif data["action"] in ["install-complete"]: elif data["action"] in ["install-complete"]:
# Node install has finished # Node install has finished
logger.info(f"Registering install complete for host {data['hostname']}") logger.info(f"Registering install complete for host {cspec_fqdn}")
notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: Registering install complete for host {data['hostname']}") notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: install completed for host {cspec_fqdn}")
host.installer_complete(config, cspec, data) host.installer_complete(config, cspec, data)
elif data["action"] in ["system-boot_initial"]: elif data["action"] in ["system-boot_initial"]:
# Node has booted for the first time and can begin Ansible runs once all nodes up # Node has booted for the first time and can begin Ansible runs once all nodes up
logger.info(f"Registering first boot for host {data['hostname']}") logger.info(f"Registering first boot for host {cspec_fqdn}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering first boot for host {data['hostname']}") notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Registering first boot for host {cspec_fqdn}")
target_state = "booted-initial" target_state = "booted-initial"
host.set_boot_state(config, cspec, data, target_state) host.set_boot_state(config, cspec, data, target_state)
@@ -122,8 +123,8 @@ def host_checkin(config, data):
elif data["action"] in ["system-boot_configured"]: elif data["action"] in ["system-boot_configured"]:
# Node has been booted after Ansible run and can begin hook runs # Node has been booted after Ansible run and can begin hook runs
logger.info(f"Registering post-Ansible boot for host {data['hostname']}") logger.info(f"Registering post-Ansible boot for host {cspec_fqdn}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering post-Ansible boot for host {data['hostname']}") notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Registering post-Ansible boot for host {cspec_fqdn}")
target_state = "booted-configured" target_state = "booted-configured"
host.set_boot_state(config, cspec, data, target_state) host.set_boot_state(config, cspec, data, target_state)
@@ -141,8 +142,8 @@ def host_checkin(config, data):
elif data["action"] in ["system-boot_completed"]: elif data["action"] in ["system-boot_completed"]:
# Node has been fully configured and can be shut down for the final time # Node has been fully configured and can be shut down for the final time
logger.info(f"Registering post-hooks boot for host {data['hostname']}") logger.info(f"Registering post-hooks boot for host {cspec_fqdn}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering post-hooks boot for host {data['hostname']}") notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Registering post-hooks boot for host {cspec_fqdn}")
target_state = "booted-completed" target_state = "booted-completed"
host.set_boot_state(config, cspec, data, target_state) host.set_boot_state(config, cspec, data, target_state)
@@ -153,7 +154,7 @@ def host_checkin(config, data):
logger.info(f"Ready: {len(ready_nodes)} All: {len(all_nodes)}") logger.info(f"Ready: {len(ready_nodes)} All: {len(all_nodes)}")
if len(ready_nodes) >= len(all_nodes): if len(ready_nodes) >= len(all_nodes):
cluster = db.update_cluster_state(config, cspec_cluster, "completed")
notifications.send_webhook(config, "completed", f"Cluster {cspec_cluster} deployment completed")
# Hosts will now power down ready for real activation in production # Hosts will now power down ready for real activation in production
sleep(30)
cluster = db.update_cluster_state(config, cspec_cluster, "completed")
notifications.send_webhook(config, "completed", f"Cluster {cspec_cluster}: Deployment completed")

10
bootstrap-daemon/pvcbootstrapd/lib/redfish.py
View File

@@ -688,8 +688,9 @@ def redfish_init(config, cspec, data):
cspec_cluster = cspec_node["node"]["cluster"] cspec_cluster = cspec_node["node"]["cluster"]
cspec_hostname = cspec_node["node"]["hostname"] cspec_hostname = cspec_node["node"]["hostname"]
cspec_fqdn = cspec_node["node"]["fqdn"]
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Beginning Redfish initialization of host {cspec_hostname}") notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Beginning Redfish initialization of host {cspec_fqdn}")
cluster = db.get_cluster(config, name=cspec_cluster) cluster = db.get_cluster(config, name=cspec_cluster)
if cluster is None: if cluster is None:
@@ -849,12 +850,12 @@ def redfish_init(config, cspec, data):
logger.info("Setting temporary PXE boot...") logger.info("Setting temporary PXE boot...")
set_boot_override(session, system_root, redfish_vendor, "Pxe") set_boot_override(session, system_root, redfish_vendor, "Pxe")
notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: Completed Redfish initialization of host {cspec_hostname}") notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: Completed Redfish initialization of host {cspec_fqdn}")
# Turn on the system # Turn on the system
logger.info("Powering on node...") logger.info("Powering on node...")
set_power_state(session, system_root, redfish_vendor, "on") set_power_state(session, system_root, redfish_vendor, "on")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Powering on host {cspec_hostname}") notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Powering on host {cspec_fqdn}")
node = db.update_node_state(config, cspec_cluster, cspec_hostname, "pxe-booting") node = db.update_node_state(config, cspec_cluster, cspec_hostname, "pxe-booting")
@@ -868,7 +869,7 @@ def redfish_init(config, cspec, data):
node = db.get_node(config, cspec_cluster, name=cspec_hostname) node = db.get_node(config, cspec_cluster, name=cspec_hostname)
# Graceful shutdown of the machine # Graceful shutdown of the machine
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Powering off host {cspec_hostname}") notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Powering off host {cspec_fqdn}")
set_power_state(session, system_root, redfish_vendor, "GracefulShutdown") set_power_state(session, system_root, redfish_vendor, "GracefulShutdown")
system_power_state = "On" system_power_state = "On"
while system_power_state != "Off": while system_power_state != "Off":
@@ -879,7 +880,6 @@ def redfish_init(config, cspec, data):
# Turn off the indicator to indicate bootstrap has completed # Turn off the indicator to indicate bootstrap has completed
set_indicator_state(session, system_root, redfish_vendor, "off") set_indicator_state(session, system_root, redfish_vendor, "off")
notifications.send_webhook(config, "completed", f"Cluster {cspec_cluster}: Powered off host {cspec_hostname}")
# We must delete the session # We must delete the session
del session del session