parallelvirtualcluster/pvc-bootstrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add note message type and improve messages

Browse Source
This commit is contained in:
Joshua Boniface
2022-07-07 15:52:39 +00:00
parent 9a9caae923
commit fe6f34a3eb
8 changed files with 34 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
bootstrap-daemon/pvcbootstrapd.yaml.sample
View File

@@ -101,6 +101,7 @@ pvc:
action: post
# Icons to use for various status types; embedded in the message with `{icon}`
icons:
info: "❕" # A note about an event
begin: "🤞" # A task is beginning
success: "✅" # A task succeeded
failure: "❌" # A task failed

1
bootstrap-daemon/pvcbootstrapd.yaml.template
View File

@@ -36,6 +36,7 @@ pvc:
uri: https://mattermost.domain.tld/hooks/asecretstring
action: post
icons:
info: "❕" # A note about an event
begin: "🤞" # A task is beginning
success: "✅" # A task succeeded
failure: "❌" # A task failed

8
bootstrap-daemon/pvcbootstrapd/Daemon.py
View File

@@ -248,7 +248,7 @@ def entrypoint():
print("|----------------------------------------------------------|")
print("")
notifications.send_webhook(config, "begin", "Starting up pvcbootstrapd")
notifications.send_webhook(config, "info", "Initializing pvcbootstrapd")
# Initialize the database
db.init_database(config)
@@ -261,7 +261,7 @@ def entrypoint():
if "--init-only" in argv:
print("Successfully initialized pvcbootstrapd; exiting.")
notifications.send_webhook(config, "success", "Successfully initialized pvcbootstrapd")
notifications.send_webhook(config, "info", "Successfully initialized pvcbootstrapd")
exit(0)
# Start DNSMasq
@@ -274,14 +274,14 @@ def entrypoint():
def term(signum="", frame=""):
print("Received TERM, exiting.")
notifications.send_webhook(config, "completed", "Received TERM, exiting pvcbootstrapd")
notifications.send_webhook(config, "info", "Received TERM, exiting pvcbootstrapd")
cleanup(0)
signal.signal(signal.SIGTERM, term)
signal.signal(signal.SIGINT, term)
signal.signal(signal.SIGQUIT, term)
notifications.send_webhook(config, "success", "Started up pvcbootstrapd")
notifications.send_webhook(config, "info", "Starting up pvcbootstrapd")
# Start Flask
pvcbootstrapd.app.run(

8
bootstrap-daemon/pvcbootstrapd/lib/ansible.py
View File

@@ -55,7 +55,7 @@ def run_bootstrap(config, cspec, cluster, nodes):
sleep(60)
logger.info("Starting Ansible bootstrap of cluster {cluster.name}")
notifications.send_webhook(config, "begin", f"Starting Ansible bootstrap of cluster {cluster.name}")
notifications.send_webhook(config, "begin", f"Cluster {cluster.name}: Starting Ansible bootstrap")
# Run the Ansible playbooks
with tempfile.TemporaryDirectory(prefix="pvc-ansible-bootstrap_") as pdir:
@@ -78,9 +78,9 @@ def run_bootstrap(config, cspec, cluster, nodes):
if r.rc == 0:
git.commit_repository(config)
git.push_repository(config)
notifications.send_webhook(config, "success", f"Completed Ansible bootstrap of cluster {cluster.name}")
notifications.send_webhook(config, "success", f"Cluster {cluster.name}: Completed Ansible bootstrap")
else:
notifications.send_webhook(config, "failure", f"Failed Ansible bootstrap of cluster {cluster.name}; check pvcbootstrapd logs")
notifications.send_webhook(config, "failure", f"Cluster {cluster.name}: Failed Ansible bootstrap; check pvcbootstrapd logs")
except Exception as e:
logger.warning(f"Error: {e}")
notifications.send_webhook(config, "failure", f"Failed Ansible bootstrap of cluster {cluster.name} with error {e}; check pvcbootstrapd logs")
notifications.send_webhook(config, "failure", f"Cluster {cluster.name}: Failed Ansible bootstrap with error {e}; check pvcbootstrapd logs")

2
bootstrap-daemon/pvcbootstrapd/lib/git.py
View File

@@ -52,7 +52,7 @@ def init_repository(config):
g = git.cmd.Git(f"{config['ansible_path']}")
g.checkout(config["ansible_branch"])
g.submodule("update", "--init", env=dict(GIT_SSH_COMMAND=git_ssh_cmd))
notifications.send_webhook(config, "success", "Successfully initialized Git repository")
notifications.send_webhook(config, "success", "Successfully updated Git repository")
except Exception as e:
print(f"Error: {e}")

4
bootstrap-daemon/pvcbootstrapd/lib/hooks.py
View File

@@ -319,7 +319,7 @@ def run_hooks(config, cspec, cluster, nodes):
logger.info("Waiting 300s before starting hook run.")
sleep(300)
notifications.send_webhook(config, "begin", f"Running hook tasks for cluster {cluster.name}")
notifications.send_webhook(config, "begin", f"Cluster {cluster.name}: Running post-setup hook tasks")
cluster_hooks = cspec["hooks"][cluster.name]
@@ -363,4 +363,4 @@ def run_hooks(config, cspec, cluster, nodes):
},
)
notifications.send_webhook(config, "success", f"Completed hook tasks for cluster {cluster.name}")
notifications.send_webhook(config, "success", f"Cluster {cluster.name}: Completed post-setup hook tasks")

31
bootstrap-daemon/pvcbootstrapd/lib/lib.py
View File

@@ -51,7 +51,7 @@ def dnsmasq_checkin(config, data):
cspec = git.load_cspec_yaml(config)
is_in_bootstrap_map = True if data["macaddr"] in cspec["bootstrap"] else False
if is_in_bootstrap_map:
notifications.send_webhook(config, "begin", f"New host checkin from MAC '{data['macaddr']}' as host {cspec['bootstrap'][data['macaddr']]['node']['fqdn']}")
notifications.send_webhook(config, "info", f"New host checkin from MAC {data['macaddr']} as host {cspec['bootstrap'][data['macaddr']]['node']['fqdn']} in cluster {cspec['bootstrap'][data['macaddr']]['node']['cluster']}")
if (
cspec["bootstrap"][data["macaddr"]]["bmc"].get("redfish", None)
is not None
@@ -83,28 +83,29 @@ def host_checkin(config, data):
"""
Handle checkins from the PVC node
"""
logger.info(f"Registering checkin for host {data['hostname']}")
logger.info(f"Registering checkin for {data['bmc_macaddr']}")
logger.debug(f"data = {data}")
cspec = git.load_cspec_yaml(config)
bmc_macaddr = data["bmc_macaddr"]
cspec_cluster = cspec["bootstrap"][bmc_macaddr]["node"]["cluster"]
cspec_fqdn = cspec["bootstrap"][bmc_macaddr]["node"]["fqdn"]
if data["action"] in ["install-start"]:
# Node install has started
logger.info(f"Registering install start for host {data['hostname']}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering install start for host {data['hostname']}")
logger.info(f"Registering install start for host {cspec_fqdn}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: starting starting for host {cspec_fqdn}")
host.installer_init(config, cspec, data)
elif data["action"] in ["install-complete"]:
# Node install has finished
logger.info(f"Registering install complete for host {data['hostname']}")
notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: Registering install complete for host {data['hostname']}")
logger.info(f"Registering install complete for host {cspec_fqdn}")
notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: install completed for host {cspec_fqdn}")
host.installer_complete(config, cspec, data)
elif data["action"] in ["system-boot_initial"]:
# Node has booted for the first time and can begin Ansible runs once all nodes up
logger.info(f"Registering first boot for host {data['hostname']}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering first boot for host {data['hostname']}")
logger.info(f"Registering first boot for host {cspec_fqdn}")
notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Registering first boot for host {cspec_fqdn}")
target_state = "booted-initial"
host.set_boot_state(config, cspec, data, target_state)
@@ -122,8 +123,8 @@ def host_checkin(config, data):
elif data["action"] in ["system-boot_configured"]:
# Node has been booted after Ansible run and can begin hook runs
logger.info(f"Registering post-Ansible boot for host {data['hostname']}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering post-Ansible boot for host {data['hostname']}")
logger.info(f"Registering post-Ansible boot for host {cspec_fqdn}")
notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Registering post-Ansible boot for host {cspec_fqdn}")
target_state = "booted-configured"
host.set_boot_state(config, cspec, data, target_state)
@@ -141,8 +142,8 @@ def host_checkin(config, data):
elif data["action"] in ["system-boot_completed"]:
# Node has been fully configured and can be shut down for the final time
logger.info(f"Registering post-hooks boot for host {data['hostname']}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering post-hooks boot for host {data['hostname']}")
logger.info(f"Registering post-hooks boot for host {cspec_fqdn}")
notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Registering post-hooks boot for host {cspec_fqdn}")
target_state = "booted-completed"
host.set_boot_state(config, cspec, data, target_state)
@@ -153,7 +154,7 @@ def host_checkin(config, data):
logger.info(f"Ready: {len(ready_nodes)} All: {len(all_nodes)}")
if len(ready_nodes) >= len(all_nodes):
cluster = db.update_cluster_state(config, cspec_cluster, "completed")
notifications.send_webhook(config, "completed", f"Cluster {cspec_cluster} deployment completed")
# Hosts will now power down ready for real activation in production
sleep(30)
cluster = db.update_cluster_state(config, cspec_cluster, "completed")
notifications.send_webhook(config, "completed", f"Cluster {cspec_cluster}: Deployment completed")

10
bootstrap-daemon/pvcbootstrapd/lib/redfish.py
View File

@@ -688,8 +688,9 @@ def redfish_init(config, cspec, data):
cspec_cluster = cspec_node["node"]["cluster"]
cspec_hostname = cspec_node["node"]["hostname"]
cspec_fqdn = cspec_node["node"]["fqdn"]
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Beginning Redfish initialization of host {cspec_hostname}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Beginning Redfish initialization of host {cspec_fqdn}")
cluster = db.get_cluster(config, name=cspec_cluster)
if cluster is None:
@@ -849,12 +850,12 @@ def redfish_init(config, cspec, data):
logger.info("Setting temporary PXE boot...")
set_boot_override(session, system_root, redfish_vendor, "Pxe")
notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: Completed Redfish initialization of host {cspec_hostname}")
notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: Completed Redfish initialization of host {cspec_fqdn}")
# Turn on the system
logger.info("Powering on node...")
set_power_state(session, system_root, redfish_vendor, "on")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Powering on host {cspec_hostname}")
notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Powering on host {cspec_fqdn}")
node = db.update_node_state(config, cspec_cluster, cspec_hostname, "pxe-booting")
@@ -868,7 +869,7 @@ def redfish_init(config, cspec, data):
node = db.get_node(config, cspec_cluster, name=cspec_hostname)
# Graceful shutdown of the machine
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Powering off host {cspec_hostname}")
notifications.send_webhook(config, "info", f"Cluster {cspec_cluster}: Powering off host {cspec_fqdn}")
set_power_state(session, system_root, redfish_vendor, "GracefulShutdown")
system_power_state = "On"
while system_power_state != "Off":
@@ -879,7 +880,6 @@ def redfish_init(config, cspec, data):
# Turn off the indicator to indicate bootstrap has completed
set_indicator_state(session, system_root, redfish_vendor, "off")
notifications.send_webhook(config, "completed", f"Cluster {cspec_cluster}: Powered off host {cspec_hostname}")
# We must delete the session
del session