Add remainder of notification webhooks

bootstrap-daemon/pvcbootstrapd.yaml.sample

@@ -104,6 +104,7 @@ pvc:
       begin: "🤞"       # A task is beginning
       success: "✅"     # A task succeeded
       failure: "❌"     # A task failed
+      completed: "👌"	# A task is completed
     # The webhook body elements; this is specific to the webhook target, and is converted into raw
     # JSON before sending.
     # Two special variables are used: "{icon}" displays one of the above icons, and "{message}" displays

bootstrap-daemon/pvcbootstrapd.yaml.template

@@ -39,6 +39,7 @@ pvc:
       begin: "🤞"       # A task is beginning
       success: "✅"     # A task succeeded
       failure: "❌"     # A task failed
+      completed: "👌"	# A task is completed
     body:
       channel: "mychannel"
       username: "pvcbootstrapd"

bootstrap-daemon/pvcbootstrapd/Daemon.py

@@ -122,6 +122,7 @@ def read_config():
         o_dhcp = o_base["dhcp"]
         o_tftp = o_base["tftp"]
         o_ansible = o_base["ansible"]
+        o_notifications = o_base["notifications"]
     except KeyError as k:
         raise MalformedConfigurationError(f"Missing first-level category {k}")
 
@@ -203,6 +204,15 @@ def read_config():
                 f"Missing third-level key '{key}' under 'ansible/cspec_files'"
             )
 
+    # Get the Notifications configuration
+    for key in ["enabled", "uri", "action", "icons", "body"]:
+        try:
+            config[f"notifications_{key}"] = o_notifications[key]
+        except Exception:
+            raise MalformedConfigurationError(
+                f"Missing second-level key '{key}' under 'notifications'"
+            )
+
     return config
 
 
@@ -240,6 +250,9 @@ def entrypoint():
 
     notifications.send_webhook(config, "begin", "Starting up pvcbootstrapd")
 
+    cspec = git.load_cspec_yaml(config)
+    print(cspec)
+
     # Initialize the database
     db.init_database(config)
 
@@ -251,6 +264,7 @@ def entrypoint():
 
     if "--init-only" in argv:
         print("Successfully initialized pvcbootstrapd; exiting.")
+        notifications.send_webhook(config, "success", "Successfully initialized pvcbootstrapd")
         exit(0)
 
     # Start DNSMasq
@@ -263,12 +277,15 @@ def entrypoint():
 
     def term(signum="", frame=""):
         print("Received TERM, exiting.")
+        notifications.send_webhook(config, "begin", "Received TERM, exiting pvcbootstrapd")
         cleanup(0)
 
     signal.signal(signal.SIGTERM, term)
     signal.signal(signal.SIGINT, term)
     signal.signal(signal.SIGQUIT, term)
 
+    notifications.send_webhook(config, "success", "Started up pvcbootstrapd")
+
     # Start Flask
     pvcbootstrapd.app.run(
         config["api_address"],

bootstrap-daemon/pvcbootstrapd/lib/ansible.py

@@ -19,6 +19,7 @@
 #
 ###############################################################################
 
+import pvcbootstrapd.lib.notifications as notifications
 import pvcbootstrapd.lib.git as git
 
 import ansible_runner
@@ -53,6 +54,9 @@ def run_bootstrap(config, cspec, cluster, nodes):
     logger.info("Waiting 60s before starting Ansible bootstrap.")
     sleep(60)
 
+    logger.info("Starting Ansible bootstrap of cluster {cluster.name}")
+    notifications.send_webhook(config, "begin", f"Starting Ansible bootstrap of cluster {cluster.name}")
+
     # Run the Ansible playbooks
     with tempfile.TemporaryDirectory(prefix="pvc-ansible-bootstrap_") as pdir:
         try:
@@ -74,5 +78,9 @@ def run_bootstrap(config, cspec, cluster, nodes):
             if r.rc == 0:
                 git.commit_repository(config)
                 git.push_repository(config)
+                notifications.send_webhook(config, "success", f"Completed Ansible bootstrap of cluster {cluster.name}")
+            else:
+                notifications.send_webhook(config, "failure", f"Failed Ansible bootstrap of cluster {cluster.name}; check pvcbootstrapd logs")
         except Exception as e:
             logger.warning(f"Error: {e}")
+            notifications.send_webhook(config, "failure", f"Failed Ansible bootstrap of cluster {cluster.name} with error {e}; check pvcbootstrapd logs")

bootstrap-daemon/pvcbootstrapd/lib/db.py

@@ -23,6 +23,8 @@ import os
 import sqlite3
 import contextlib
 
+import pvcbootstrapd.lib.notifications as notifications
+
 from pvcbootstrapd.lib.dataclasses import Cluster, Node
 
 from celery.utils.log import get_task_logger
@@ -48,6 +50,7 @@ def init_database(config):
     db_path = config["database_path"]
     if not os.path.isfile(db_path):
         print("First run: initializing database.")
+        notifications.send_webhook(config, "begin", "First run: initializing database")
         # Initializing the database
         with dbconn(db_path) as cur:
             # Table listing all clusters
@@ -73,6 +76,8 @@ def init_database(config):
                             CONSTRAINT cluster_col FOREIGN KEY (cluster) REFERENCES clusters(id) ON DELETE CASCADE )"""
             )
 
+        notifications.send_webhook(config, "success", "First run: successfully initialized database")
+
 
 #
 # Cluster functions

bootstrap-daemon/pvcbootstrapd/lib/git.py

@@ -23,6 +23,8 @@ import os.path
 import git
 import yaml
 
+import pvcbootstrapd.lib.notifications as notifications
+
 from celery.utils.log import get_task_logger
 
 
@@ -39,6 +41,7 @@ def init_repository(config):
             print(
                 f"First run: cloning repository {config['ansible_remote']} branch {config['ansible_branch']} to {config['ansible_path']}"
             )
+            notifications.send_webhook(config, "begin", f"First run: cloning repository {config['ansible_remote']} branch {config['ansible_branch']} to {config['ansible_path']}")
             git.Repo.clone_from(
                 config["ansible_remote"],
                 config["ansible_path"],
@@ -49,6 +52,7 @@ def init_repository(config):
         g = git.cmd.Git(f"{config['ansible_path']}")
         g.checkout(config["ansible_branch"])
         g.submodule("update", "--init", env=dict(GIT_SSH_COMMAND=git_ssh_cmd))
+        notifications.send_webhook(config, "success", "First run: successfully initialized Git repository")
     except Exception as e:
         print(f"Error: {e}")
 

bootstrap-daemon/pvcbootstrapd/lib/hooks.py

@@ -19,6 +19,7 @@
 #
 ###############################################################################
 
+import pvcbootstrapd.lib.notifications as notifications
 import pvcbootstrapd.lib.db as db
 
 import json
@@ -311,6 +312,8 @@ def run_hooks(config, cspec, cluster, nodes):
     logger.info("Waiting 300s before starting hook run.")
     sleep(300)
 
+    notifications.send_webhook(config, "begin", f"Running hook tasks for cluster {cluster.name}")
+
     cluster_hooks = cspec["hooks"][cluster.name]
 
     cluster_nodes = db.get_nodes_in_cluster(config, cluster.name)
@@ -334,9 +337,12 @@ def run_hooks(config, cspec, cluster, nodes):
 
         # Run the hook function
         try:
+            notifications.send_webhook(config, "begin", f"Cluster {cluster.name}: Running hook task '{hook_name}'")
             hook_functions[hook_type](config, target_nodes, hook_args)
+            notifications.send_webhook(config, "success", f"Cluster {cluster.name}: Completed hook task '{hook_name}'")
         except Exception as e:
             logger.warning(f"Error running hook: {e}")
+            notifications.send_webhook(config, "failure", f"Cluster {cluster.name}: Failed hook task '{hook_name}' with error {e}")
 
         # Wait 5s between hooks
         sleep(5)
@@ -349,3 +355,5 @@ def run_hooks(config, cspec, cluster, nodes):
             "script": "#!/usr/bin/env bash\necho bootstrapped | sudo tee /etc/pvc-install.hooks\nsudo reboot"
         },
     )
+
+    notifications.send_webhook(config, "success", f"Completed hook tasks for cluster {cluster.name}")

bootstrap-daemon/pvcbootstrapd/lib/host.py

@@ -19,10 +19,10 @@
 #
 ###############################################################################
 
-from celery.utils.log import get_task_logger
-
 import pvcbootstrapd.lib.db as db
 
+from celery.utils.log import get_task_logger
+
 
 logger = get_task_logger(__name__)
 

bootstrap-daemon/pvcbootstrapd/lib/lib.py

@@ -19,6 +19,7 @@
 #
 ###############################################################################
 
+import pvcbootstrapd.lib.notifications as notifications
 import pvcbootstrapd.lib.db as db
 import pvcbootstrapd.lib.git as git
 import pvcbootstrapd.lib.redfish as redfish
@@ -50,6 +51,7 @@ def dnsmasq_checkin(config, data):
         cspec = git.load_cspec_yaml(config)
         is_in_bootstrap_map = True if data["macaddr"] in cspec["bootstrap"] else False
         if is_in_bootstrap_map:
+            notifications.send_webhook(config, "begin", f"New host checkin from MAC '{data['macaddr']}' as host {cspec['bootstrap'][data['macaddr']]['node']['fqdn']}")
             if (
                 cspec["bootstrap"][data["macaddr"]]["bmc"].get("redfish", None)
                 is not None
@@ -90,16 +92,19 @@ def host_checkin(config, data):
     if data["action"] in ["install-start"]:
         # Node install has started
         logger.info(f"Registering install start for host {data['hostname']}")
+        notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering install start for host {data['hostname']}")
         host.installer_init(config, cspec, data)
 
     elif data["action"] in ["install-complete"]:
         # Node install has finished
         logger.info(f"Registering install complete for host {data['hostname']}")
+        notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering install complete for host {data['hostname']}")
         host.installer_complete(config, cspec, data)
 
     elif data["action"] in ["system-boot_initial"]:
         # Node has booted for the first time and can begin Ansible runs once all nodes up
         logger.info(f"Registering first boot for host {data['hostname']}")
+        notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering first boot for host {data['hostname']}")
         target_state = "booted-initial"
 
         host.set_boot_state(config, cspec, data, target_state)
@@ -118,6 +123,7 @@ def host_checkin(config, data):
     elif data["action"] in ["system-boot_configured"]:
         # Node has been booted after Ansible run and can begin hook runs
         logger.info(f"Registering post-Ansible boot for host {data['hostname']}")
+        notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering post-Ansible boot for host {data['hostname']}")
         target_state = "booted-configured"
 
         host.set_boot_state(config, cspec, data, target_state)
@@ -136,6 +142,7 @@ def host_checkin(config, data):
     elif data["action"] in ["system-boot_completed"]:
         # Node has been fully configured and can be shut down for the final time
         logger.info(f"Registering post-hooks boot for host {data['hostname']}")
+        notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering post-hooks boot for host {data['hostname']}")
         target_state = "booted-completed"
 
         host.set_boot_state(config, cspec, data, target_state)
@@ -148,4 +155,5 @@ def host_checkin(config, data):
         if len(ready_nodes) >= len(all_nodes):
             cluster = db.update_cluster_state(config, cspec_cluster, "completed")
 
+            notifications.send_webhook(config, "completed", f"Cluster {cspec_cluster} deployment completed")
             # Hosts will now power down ready for real activation in production

bootstrap-daemon/pvcbootstrapd/lib/notifications.py

@@ -33,16 +33,20 @@ def send_webhook(config, status, message):
     Send an notification webhook
     """
 
-    if not config.get('notifications', None) or not config['notifications']['enabled']:
+    if not config['notifications_enabled']:
         return
 
-    logger.debug(f"Sending notification to {config['notifications']['uri']}")
+    logger.debug(f"Sending notification to {config['notifications_uri']}")
 
     # Get the body data
-    data = json.dumps(config['notifications']['body']).format(
+    body = config['notifications_body']
-        icon=config['notifications']['icons'][status],
+    formatted_body = dict()
+    for element, value in body.items():
+        formatted_body[element] = value.format(
+            icon=config['notifications_icons'][status],
             message=message
         )
+    data = json.dumps(formatted_body)
     headers = {"content-type": "application/json"}
 
     # Craft up a Requests endpoint set for this
@@ -54,8 +58,8 @@ def send_webhook(config, status, message):
         "delete": requests.delete,
         "options": requests.options,
     }
-    action = config['notifications']["action"]
+    action = config['notifications_action']
 
-    result = requests_actions[action](config['notifications']["uri"], headers=headers, data=data)
+    result = requests_actions[action](config['notifications_uri'], headers=headers, data=data)
 
     logger.debug(f"Result: {result}")

bootstrap-daemon/pvcbootstrapd/lib/redfish.py

@@ -31,6 +31,7 @@ import math
 from time import sleep
 from celery.utils.log import get_task_logger
 
+import pvcbootstrapd.lib.notifications as notifications
 import pvcbootstrapd.lib.installer as installer
 import pvcbootstrapd.lib.db as db
 
@@ -688,6 +689,8 @@ def redfish_init(config, cspec, data):
     cspec_cluster = cspec_node["node"]["cluster"]
     cspec_hostname = cspec_node["node"]["hostname"]
 
+    notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Beginning Redfish initialization of host {cspec_hostname}")
+
     cluster = db.get_cluster(config, name=cspec_cluster)
     if cluster is None:
         cluster = db.add_cluster(config, cspec, cspec_cluster, "provisioning")
@@ -852,6 +855,7 @@ def redfish_init(config, cspec, data):
 
     node = db.update_node_state(config, cspec_cluster, cspec_hostname, "pxe-booting")
 
+    notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: Completed Redfish initialization of host {cspec_hostname}")
     logger.info("Waiting for completion of node and cluster installation...")
     # Wait for the system to install and be configured
     while node.state != "booted-completed":
@@ -862,6 +866,7 @@ def redfish_init(config, cspec, data):
         node = db.get_node(config, cspec_cluster, name=cspec_hostname)
 
     # Graceful shutdown of the machine
+    notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Powering off host {cspec_hostname}")
     set_power_state(session, system_root, redfish_vendor, "GracefulShutdown")
     system_power_state = "On"
     while system_power_state != "Off":
@@ -872,6 +877,7 @@ def redfish_init(config, cspec, data):
 
     # Turn off the indicator to indicate bootstrap has completed
     set_indicator_state(session, system_root, redfish_vendor, "off")
+    notifications.send_webhook(config, "completed", f"Cluster {cspec_cluster}: Powered off host {cspec_hostname}")
 
     # We must delete the session
     del session

bootstrap-daemon/pvcbootstrapd/lib/tftp.py

@@ -22,11 +22,14 @@
 import os.path
 import shutil
 
+import pvcbootstrapd.lib.notifications as notifications
+
 
 def build_tftp_repository(config):
     # Generate an installer config
     build_cmd = f"{config['ansible_path']}/pvc-installer/buildpxe.sh -o {config['tftp_root_path']} -u {config['deploy_username']}"
     print(f"Building TFTP contents via pvc-installer command: {build_cmd}")
+    notifications.send_webhook(config, "begin", f"Building TFTP contents via pvc-installer command: {build_cmd}")
     os.system(build_cmd)
 
 
@@ -36,6 +39,7 @@ def init_tftp(config):
     """
     if not os.path.exists(config["tftp_root_path"]):
         print("First run: building TFTP root and contents - this will take some time!")
+        notifications.send_webhook(config, "begin", "First run: building TFTP root and contents")
         os.makedirs(config["tftp_root_path"])
         os.makedirs(config["tftp_host_path"])
         shutil.copyfile(
@@ -43,3 +47,4 @@ def init_tftp(config):
         )
 
         build_tftp_repository(config)
+        notifications.send_webhook(config, "success", "First run: successfully initialized TFTP root and contents")