From fe6f34a3eb9e6bb2ef92de3940c65fe5afb0d2e2 Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Thu, 7 Jul 2022 15:52:39 +0000 Subject: [PATCH] Add note message type and improve messages --- bootstrap-daemon/pvcbootstrapd.yaml.sample | 1 + bootstrap-daemon/pvcbootstrapd.yaml.template | 1 + bootstrap-daemon/pvcbootstrapd/Daemon.py | 8 ++--- bootstrap-daemon/pvcbootstrapd/lib/ansible.py | 8 ++--- bootstrap-daemon/pvcbootstrapd/lib/git.py | 2 +- bootstrap-daemon/pvcbootstrapd/lib/hooks.py | 4 +-- bootstrap-daemon/pvcbootstrapd/lib/lib.py | 31 ++++++++++--------- bootstrap-daemon/pvcbootstrapd/lib/redfish.py | 10 +++--- 8 files changed, 34 insertions(+), 31 deletions(-) diff --git a/bootstrap-daemon/pvcbootstrapd.yaml.sample b/bootstrap-daemon/pvcbootstrapd.yaml.sample index 8c9eb46..a97e825 100644 --- a/bootstrap-daemon/pvcbootstrapd.yaml.sample +++ b/bootstrap-daemon/pvcbootstrapd.yaml.sample @@ -101,6 +101,7 @@ pvc: action: post # Icons to use for various status types; embedded in the message with `{icon}` icons: + info: "❕" # A note about an event begin: "🤞" # A task is beginning success: "✅" # A task succeeded failure: "❌" # A task failed diff --git a/bootstrap-daemon/pvcbootstrapd.yaml.template b/bootstrap-daemon/pvcbootstrapd.yaml.template index 9c00646..af12c43 100644 --- a/bootstrap-daemon/pvcbootstrapd.yaml.template +++ b/bootstrap-daemon/pvcbootstrapd.yaml.template @@ -36,6 +36,7 @@ pvc: uri: https://mattermost.domain.tld/hooks/asecretstring action: post icons: + info: "❕" # A note about an event begin: "🤞" # A task is beginning success: "✅" # A task succeeded failure: "❌" # A task failed diff --git a/bootstrap-daemon/pvcbootstrapd/Daemon.py b/bootstrap-daemon/pvcbootstrapd/Daemon.py index 01559b6..6e5ad6b 100755 --- a/bootstrap-daemon/pvcbootstrapd/Daemon.py +++ b/bootstrap-daemon/pvcbootstrapd/Daemon.py @@ -248,7 +248,7 @@ def entrypoint(): print("|----------------------------------------------------------|") print("") - notifications.send_webhook(config, "begin", "Starting up pvcbootstrapd") + notifications.send_webhook(config, "info", "Initializing pvcbootstrapd") # Initialize the database db.init_database(config) @@ -261,7 +261,7 @@ def entrypoint(): if "--init-only" in argv: print("Successfully initialized pvcbootstrapd; exiting.") - notifications.send_webhook(config, "success", "Successfully initialized pvcbootstrapd") + notifications.send_webhook(config, "info", "Successfully initialized pvcbootstrapd") exit(0) # Start DNSMasq @@ -274,14 +274,14 @@ def entrypoint(): def term(signum="", frame=""): print("Received TERM, exiting.") - notifications.send_webhook(config, "completed", "Received TERM, exiting pvcbootstrapd") + notifications.send_webhook(config, "info", "Received TERM, exiting pvcbootstrapd") cleanup(0) signal.signal(signal.SIGTERM, term) signal.signal(signal.SIGINT, term) signal.signal(signal.SIGQUIT, term) - notifications.send_webhook(config, "success", "Started up pvcbootstrapd") + notifications.send_webhook(config, "info", "Starting up pvcbootstrapd") # Start Flask pvcbootstrapd.app.run( diff --git a/bootstrap-daemon/pvcbootstrapd/lib/ansible.py b/bootstrap-daemon/pvcbootstrapd/lib/ansible.py index d428425..7e27aa7 100755 --- a/bootstrap-daemon/pvcbootstrapd/lib/ansible.py +++ b/bootstrap-daemon/pvcbootstrapd/lib/ansible.py @@ -55,7 +55,7 @@ def run_bootstrap(config, cspec, cluster, nodes): sleep(60) logger.info("Starting Ansible bootstrap of cluster {cluster.name}") - notifications.send_webhook(config, "begin", f"Starting Ansible bootstrap of cluster {cluster.name}") + notifications.send_webhook(config, "begin", f"Cluster {cluster.name}: Starting Ansible bootstrap") # Run the Ansible playbooks with tempfile.TemporaryDirectory(prefix="pvc-ansible-bootstrap_") as pdir: @@ -78,9 +78,9 @@ def run_bootstrap(config, cspec, cluster, nodes): if r.rc == 0: git.commit_repository(config) git.push_repository(config) - notifications.send_webhook(config, "success", f"Completed Ansible bootstrap of cluster {cluster.name}") + notifications.send_webhook(config, "success", f"Cluster {cluster.name}: Completed Ansible bootstrap") else: - notifications.send_webhook(config, "failure", f"Failed Ansible bootstrap of cluster {cluster.name}; check pvcbootstrapd logs") + notifications.send_webhook(config, "failure", f"Cluster {cluster.name}: Failed Ansible bootstrap; check pvcbootstrapd logs") except Exception as e: logger.warning(f"Error: {e}") - notifications.send_webhook(config, "failure", f"Failed Ansible bootstrap of cluster {cluster.name} with error {e}; check pvcbootstrapd logs") + notifications.send_webhook(config, "failure", f"Cluster {cluster.name}: Failed Ansible bootstrap with error {e}; check pvcbootstrapd logs") diff --git a/bootstrap-daemon/pvcbootstrapd/lib/git.py b/bootstrap-daemon/pvcbootstrapd/lib/git.py index 021be49..b1f7328 100755 --- a/bootstrap-daemon/pvcbootstrapd/lib/git.py +++ b/bootstrap-daemon/pvcbootstrapd/lib/git.py @@ -52,7 +52,7 @@ def init_repository(config): g = git.cmd.Git(f"{config['ansible_path']}") g.checkout(config["ansible_branch"]) g.submodule("update", "--init", env=dict(GIT_SSH_COMMAND=git_ssh_cmd)) - notifications.send_webhook(config, "success", "Successfully initialized Git repository") + notifications.send_webhook(config, "success", "Successfully updated Git repository") except Exception as e: print(f"Error: {e}") diff --git a/bootstrap-daemon/pvcbootstrapd/lib/hooks.py b/bootstrap-daemon/pvcbootstrapd/lib/hooks.py index 3ebdc7b..3367a83 100755 --- a/bootstrap-daemon/pvcbootstrapd/lib/hooks.py +++ b/bootstrap-daemon/pvcbootstrapd/lib/hooks.py @@ -319,7 +319,7 @@ def run_hooks(config, cspec, cluster, nodes): logger.info("Waiting 300s before starting hook run.") sleep(300) - notifications.send_webhook(config, "begin", f"Running hook tasks for cluster {cluster.name}") + notifications.send_webhook(config, "begin", f"Cluster {cluster.name}: Running post-setup hook tasks") cluster_hooks = cspec["hooks"][cluster.name] @@ -363,4 +363,4 @@ def run_hooks(config, cspec, cluster, nodes): }, ) - notifications.send_webhook(config, "success", f"Completed hook tasks for cluster {cluster.name}") + notifications.send_webhook(config, "success", f"Cluster {cluster.name}: Completed post-setup hook tasks") diff --git a/bootstrap-daemon/pvcbootstrapd/lib/lib.py b/bootstrap-daemon/pvcbootstrapd/lib/lib.py index 275fd75..a6f30e3 100755 --- a/bootstrap-daemon/pvcbootstrapd/lib/lib.py +++ b/bootstrap-daemon/pvcbootstrapd/lib/lib.py @@ -51,7 +51,7 @@ def dnsmasq_checkin(config, data): cspec = git.load_cspec_yaml(config) is_in_bootstrap_map = True if data["macaddr"] in cspec["bootstrap"] else False if is_in_bootstrap_map: - notifications.send_webhook(config, "begin", f"New host checkin from MAC '{data['macaddr']}' as host {cspec['bootstrap'][data['macaddr']]['node']['fqdn']}") + notifications.send_webhook(config, "info", f"New host checkin from MAC {data['macaddr']} as host {cspec['bootstrap'][data['macaddr']]['node']['fqdn']} in cluster {cspec['bootstrap'][data['macaddr']]['node']['cluster']}") if ( cspec["bootstrap"][data["macaddr"]]["bmc"].get("redfish", None) is not None @@ -83,28 +83,29 @@ def host_checkin(config, data): """ Handle checkins from the PVC node """ - logger.info(f"Registering checkin for host {data['hostname']}") + logger.info(f"Registering checkin for {data['bmc_macaddr']}") logger.debug(f"data = {data}") cspec = git.load_cspec_yaml(config) bmc_macaddr = data["bmc_macaddr"] cspec_cluster = cspec["bootstrap"][bmc_macaddr]["node"]["cluster"] + cspec_fqdn = cspec["bootstrap"][bmc_macaddr]["node"]["fqdn"] if data["action"] in ["install-start"]: # Node install has started - logger.info(f"Registering install start for host {data['hostname']}") - notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering install start for host {data['hostname']}") + logger.info(f"Registering install start for host {cspec_fqdn}") + notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: starting starting for host {cspec_fqdn}") host.installer_init(config, cspec, data) elif data["action"] in ["install-complete"]: # Node install has finished - logger.info(f"Registering install complete for host {data['hostname']}") - notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: Registering install complete for host {data['hostname']}") + logger.info(f"Registering install complete for host {cspec_fqdn}") + notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: install completed for host {cspec_fqdn}") host.installer_complete(config, cspec, data) elif data["action"] in ["system-boot_initial"]: # Node has booted for the first time and can begin Ansible runs once all nodes up - logger.info(f"Registering first boot for host {data['hostname']}") - notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering first boot for host {data['hostname']}") + logger.info(f"Registering first boot for host {cspec_fqdn}") + notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Registering first boot for host {cspec_fqdn}") target_state = "booted-initial" host.set_boot_state(config, cspec, data, target_state) @@ -122,8 +123,8 @@ def host_checkin(config, data): elif data["action"] in ["system-boot_configured"]: # Node has been booted after Ansible run and can begin hook runs - logger.info(f"Registering post-Ansible boot for host {data['hostname']}") - notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering post-Ansible boot for host {data['hostname']}") + logger.info(f"Registering post-Ansible boot for host {cspec_fqdn}") + notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Registering post-Ansible boot for host {cspec_fqdn}") target_state = "booted-configured" host.set_boot_state(config, cspec, data, target_state) @@ -141,8 +142,8 @@ def host_checkin(config, data): elif data["action"] in ["system-boot_completed"]: # Node has been fully configured and can be shut down for the final time - logger.info(f"Registering post-hooks boot for host {data['hostname']}") - notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering post-hooks boot for host {data['hostname']}") + logger.info(f"Registering post-hooks boot for host {cspec_fqdn}") + notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Registering post-hooks boot for host {cspec_fqdn}") target_state = "booted-completed" host.set_boot_state(config, cspec, data, target_state) @@ -153,7 +154,7 @@ def host_checkin(config, data): logger.info(f"Ready: {len(ready_nodes)} All: {len(all_nodes)}") if len(ready_nodes) >= len(all_nodes): - cluster = db.update_cluster_state(config, cspec_cluster, "completed") - - notifications.send_webhook(config, "completed", f"Cluster {cspec_cluster} deployment completed") # Hosts will now power down ready for real activation in production + sleep(30) + cluster = db.update_cluster_state(config, cspec_cluster, "completed") + notifications.send_webhook(config, "completed", f"Cluster {cspec_cluster}: Deployment completed") diff --git a/bootstrap-daemon/pvcbootstrapd/lib/redfish.py b/bootstrap-daemon/pvcbootstrapd/lib/redfish.py index 95af734..71ca9bf 100755 --- a/bootstrap-daemon/pvcbootstrapd/lib/redfish.py +++ b/bootstrap-daemon/pvcbootstrapd/lib/redfish.py @@ -688,8 +688,9 @@ def redfish_init(config, cspec, data): cspec_cluster = cspec_node["node"]["cluster"] cspec_hostname = cspec_node["node"]["hostname"] + cspec_fqdn = cspec_node["node"]["fqdn"] - notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Beginning Redfish initialization of host {cspec_hostname}") + notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Beginning Redfish initialization of host {cspec_fqdn}") cluster = db.get_cluster(config, name=cspec_cluster) if cluster is None: @@ -849,12 +850,12 @@ def redfish_init(config, cspec, data): logger.info("Setting temporary PXE boot...") set_boot_override(session, system_root, redfish_vendor, "Pxe") - notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: Completed Redfish initialization of host {cspec_hostname}") + notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: Completed Redfish initialization of host {cspec_fqdn}") # Turn on the system logger.info("Powering on node...") set_power_state(session, system_root, redfish_vendor, "on") - notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Powering on host {cspec_hostname}") + notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Powering on host {cspec_fqdn}") node = db.update_node_state(config, cspec_cluster, cspec_hostname, "pxe-booting") @@ -868,7 +869,7 @@ def redfish_init(config, cspec, data): node = db.get_node(config, cspec_cluster, name=cspec_hostname) # Graceful shutdown of the machine - notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Powering off host {cspec_hostname}") + notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Powering off host {cspec_fqdn}") set_power_state(session, system_root, redfish_vendor, "GracefulShutdown") system_power_state = "On" while system_power_state != "Off": @@ -879,7 +880,6 @@ def redfish_init(config, cspec, data): # Turn off the indicator to indicate bootstrap has completed set_indicator_state(session, system_root, redfish_vendor, "off") - notifications.send_webhook(config, "completed", f"Cluster {cspec_cluster}: Powered off host {cspec_hostname}") # We must delete the session del session