parallelvirtualcluster/pvc-bootstrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: parallelvirtualcluster:d773ded73df53518332b085583658175cf6f1fdd
Branches Tags
parallelvirtualcluster:master
..
compare: parallelvirtualcluster:5563f8921f3457c151b01dbe550c603eb2594349
Branches Tags
parallelvirtualcluster:master

No commits in common. "d773ded73df53518332b085583658175cf6f1fdd" and "5563f8921f3457c151b01dbe550c603eb2594349" have entirely different histories.
d773ded73d ... 5563f8921f

12 changed files with 10 additions and 77 deletions
Show Stats Expand all files Collapse all files

1
bootstrap-daemon/pvcbootstrapd.yaml.sample
View File

@ -104,7 +104,6 @@ pvc:
begin: "🤞" # A task is beginning
success: "✅" # A task succeeded
failure: "❌" # A task failed
completed: "👌" # A task is completed
# The webhook body elements; this is specific to the webhook target, and is converted into raw
# JSON before sending.
# Two special variables are used: "{icon}" displays one of the above icons, and "{message}" displays

1
bootstrap-daemon/pvcbootstrapd.yaml.template
View File

@ -39,7 +39,6 @@ pvc:
begin: "🤞" # A task is beginning
success: "✅" # A task succeeded
failure: "❌" # A task failed
completed: "👌" # A task is completed
body:
channel: "mychannel"
username: "pvcbootstrapd"

17
bootstrap-daemon/pvcbootstrapd/Daemon.py
View File

@ -122,7 +122,6 @@ def read_config():
o_dhcp = o_base["dhcp"]
o_tftp = o_base["tftp"]
o_ansible = o_base["ansible"]
o_notifications = o_base["notifications"]
except KeyError as k:
raise MalformedConfigurationError(f"Missing first-level category {k}")
@ -204,15 +203,6 @@ def read_config():
f"Missing third-level key '{key}' under 'ansible/cspec_files'"
)
# Get the Notifications configuration
for key in ["enabled", "uri", "action", "icons", "body"]:
try:
config[f"notifications_{key}"] = o_notifications[key]
except Exception:
raise MalformedConfigurationError(
f"Missing second-level key '{key}' under 'notifications'"
)
return config
@ -250,9 +240,6 @@ def entrypoint():
notifications.send_webhook(config, "begin", "Starting up pvcbootstrapd")
cspec = git.load_cspec_yaml(config)
print(cspec)
# Initialize the database
db.init_database(config)
@ -264,7 +251,6 @@ def entrypoint():
if "--init-only" in argv:
print("Successfully initialized pvcbootstrapd; exiting.")
notifications.send_webhook(config, "success", "Successfully initialized pvcbootstrapd")
exit(0)
# Start DNSMasq
@ -277,15 +263,12 @@ def entrypoint():
def term(signum="", frame=""):
print("Received TERM, exiting.")
notifications.send_webhook(config, "begin", "Received TERM, exiting pvcbootstrapd")
cleanup(0)
signal.signal(signal.SIGTERM, term)
signal.signal(signal.SIGINT, term)
signal.signal(signal.SIGQUIT, term)
notifications.send_webhook(config, "success", "Started up pvcbootstrapd")
# Start Flask
pvcbootstrapd.app.run(
config["api_address"],

8
bootstrap-daemon/pvcbootstrapd/lib/ansible.py
View File

@ -19,7 +19,6 @@
#
###############################################################################
import pvcbootstrapd.lib.notifications as notifications
import pvcbootstrapd.lib.git as git
import ansible_runner
@ -54,9 +53,6 @@ def run_bootstrap(config, cspec, cluster, nodes):
logger.info("Waiting 60s before starting Ansible bootstrap.")
sleep(60)
logger.info("Starting Ansible bootstrap of cluster {cluster.name}")
notifications.send_webhook(config, "begin", f"Starting Ansible bootstrap of cluster {cluster.name}")
# Run the Ansible playbooks
with tempfile.TemporaryDirectory(prefix="pvc-ansible-bootstrap_") as pdir:
try:
@ -78,9 +74,5 @@ def run_bootstrap(config, cspec, cluster, nodes):
if r.rc == 0:
git.commit_repository(config)
git.push_repository(config)
notifications.send_webhook(config, "success", f"Completed Ansible bootstrap of cluster {cluster.name}")
else:
notifications.send_webhook(config, "failure", f"Failed Ansible bootstrap of cluster {cluster.name}; check pvcbootstrapd logs")
except Exception as e:
logger.warning(f"Error: {e}")
notifications.send_webhook(config, "failure", f"Failed Ansible bootstrap of cluster {cluster.name} with error {e}; check pvcbootstrapd logs")

5
bootstrap-daemon/pvcbootstrapd/lib/db.py
View File

@ -23,8 +23,6 @@ import os
import sqlite3
import contextlib
import pvcbootstrapd.lib.notifications as notifications
from pvcbootstrapd.lib.dataclasses import Cluster, Node
from celery.utils.log import get_task_logger
@ -50,7 +48,6 @@ def init_database(config):
db_path = config["database_path"]
if not os.path.isfile(db_path):
print("First run: initializing database.")
notifications.send_webhook(config, "begin", "First run: initializing database")
# Initializing the database
with dbconn(db_path) as cur:
# Table listing all clusters
@ -76,8 +73,6 @@ def init_database(config):
CONSTRAINT cluster_col FOREIGN KEY (cluster) REFERENCES clusters(id) ON DELETE CASCADE )"""
)
notifications.send_webhook(config, "success", "First run: successfully initialized database")
#
# Cluster functions

4
bootstrap-daemon/pvcbootstrapd/lib/git.py
View File

@ -23,8 +23,6 @@ import os.path
import git
import yaml
import pvcbootstrapd.lib.notifications as notifications
from celery.utils.log import get_task_logger
@ -41,7 +39,6 @@ def init_repository(config):
print(
f"First run: cloning repository {config['ansible_remote']} branch {config['ansible_branch']} to {config['ansible_path']}"
)
notifications.send_webhook(config, "begin", f"First run: cloning repository {config['ansible_remote']} branch {config['ansible_branch']} to {config['ansible_path']}")
git.Repo.clone_from(
config["ansible_remote"],
config["ansible_path"],
@ -52,7 +49,6 @@ def init_repository(config):
g = git.cmd.Git(f"{config['ansible_path']}")
g.checkout(config["ansible_branch"])
g.submodule("update", "--init", env=dict(GIT_SSH_COMMAND=git_ssh_cmd))
notifications.send_webhook(config, "success", "First run: successfully initialized Git repository")
except Exception as e:
print(f"Error: {e}")

8
bootstrap-daemon/pvcbootstrapd/lib/hooks.py
View File

@ -19,7 +19,6 @@
#
###############################################################################
import pvcbootstrapd.lib.notifications as notifications
import pvcbootstrapd.lib.db as db
import json
@ -312,8 +311,6 @@ def run_hooks(config, cspec, cluster, nodes):
logger.info("Waiting 300s before starting hook run.")
sleep(300)
notifications.send_webhook(config, "begin", f"Running hook tasks for cluster {cluster.name}")
cluster_hooks = cspec["hooks"][cluster.name]
cluster_nodes = db.get_nodes_in_cluster(config, cluster.name)
@ -337,12 +334,9 @@ def run_hooks(config, cspec, cluster, nodes):
# Run the hook function
try:
notifications.send_webhook(config, "begin", f"Cluster {cluster.name}: Running hook task '{hook_name}'")
hook_functions[hook_type](config, target_nodes, hook_args)
notifications.send_webhook(config, "success", f"Cluster {cluster.name}: Completed hook task '{hook_name}'")
except Exception as e:
logger.warning(f"Error running hook: {e}")
notifications.send_webhook(config, "failure", f"Cluster {cluster.name}: Failed hook task '{hook_name}' with error {e}")
# Wait 5s between hooks
sleep(5)
@ -355,5 +349,3 @@ def run_hooks(config, cspec, cluster, nodes):
"script": "#!/usr/bin/env bash\necho bootstrapped | sudo tee /etc/pvc-install.hooks\nsudo reboot"
},
)
notifications.send_webhook(config, "success", f"Completed hook tasks for cluster {cluster.name}")

4
bootstrap-daemon/pvcbootstrapd/lib/host.py
View File

@ -19,10 +19,10 @@
#
###############################################################################
import pvcbootstrapd.lib.db as db
from celery.utils.log import get_task_logger
import pvcbootstrapd.lib.db as db
logger = get_task_logger(__name__)

8
bootstrap-daemon/pvcbootstrapd/lib/lib.py
View File

@ -19,7 +19,6 @@
#
###############################################################################
import pvcbootstrapd.lib.notifications as notifications
import pvcbootstrapd.lib.db as db
import pvcbootstrapd.lib.git as git
import pvcbootstrapd.lib.redfish as redfish
@ -51,7 +50,6 @@ def dnsmasq_checkin(config, data):
cspec = git.load_cspec_yaml(config)
is_in_bootstrap_map = True if data["macaddr"] in cspec["bootstrap"] else False
if is_in_bootstrap_map:
notifications.send_webhook(config, "begin", f"New host checkin from MAC '{data['macaddr']}' as host {cspec['bootstrap'][data['macaddr']]['node']['fqdn']}")
if (
cspec["bootstrap"][data["macaddr"]]["bmc"].get("redfish", None)
is not None
@ -92,19 +90,16 @@ def host_checkin(config, data):
if data["action"] in ["install-start"]:
# Node install has started
logger.info(f"Registering install start for host {data['hostname']}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering install start for host {data['hostname']}")
host.installer_init(config, cspec, data)
elif data["action"] in ["install-complete"]:
# Node install has finished
logger.info(f"Registering install complete for host {data['hostname']}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering install complete for host {data['hostname']}")
host.installer_complete(config, cspec, data)
elif data["action"] in ["system-boot_initial"]:
# Node has booted for the first time and can begin Ansible runs once all nodes up
logger.info(f"Registering first boot for host {data['hostname']}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering first boot for host {data['hostname']}")
target_state = "booted-initial"
host.set_boot_state(config, cspec, data, target_state)
@ -123,7 +118,6 @@ def host_checkin(config, data):
elif data["action"] in ["system-boot_configured"]:
# Node has been booted after Ansible run and can begin hook runs
logger.info(f"Registering post-Ansible boot for host {data['hostname']}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering post-Ansible boot for host {data['hostname']}")
target_state = "booted-configured"
host.set_boot_state(config, cspec, data, target_state)
@ -142,7 +136,6 @@ def host_checkin(config, data):
elif data["action"] in ["system-boot_completed"]:
# Node has been fully configured and can be shut down for the final time
logger.info(f"Registering post-hooks boot for host {data['hostname']}")
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Registering post-hooks boot for host {data['hostname']}")
target_state = "booted-completed"
host.set_boot_state(config, cspec, data, target_state)
@ -155,5 +148,4 @@ def host_checkin(config, data):
if len(ready_nodes) >= len(all_nodes):
cluster = db.update_cluster_state(config, cspec_cluster, "completed")
notifications.send_webhook(config, "completed", f"Cluster {cspec_cluster} deployment completed")
# Hosts will now power down ready for real activation in production

20
bootstrap-daemon/pvcbootstrapd/lib/notifications.py
View File

@ -33,20 +33,16 @@ def send_webhook(config, status, message):
Send an notification webhook
"""
if not config['notifications_enabled']:
if not config.get('notifications', None) or not config['notifications']['enabled']:
return
logger.debug(f"Sending notification to {config['notifications_uri']}")
logger.debug(f"Sending notification to {config['notifications']['uri']}")
# Get the body data
body = config['notifications_body']
formatted_body = dict()
for element, value in body.items():
formatted_body[element] = value.format(
icon=config['notifications_icons'][status],
message=message
)
data = json.dumps(formatted_body)
data = json.dumps(config['notifications']['body']).format(
icon=config['notifications']['icons'][status],
message=message
)
headers = {"content-type": "application/json"}
# Craft up a Requests endpoint set for this
@ -58,8 +54,8 @@ def send_webhook(config, status, message):
"delete": requests.delete,
"options": requests.options,
}
action = config['notifications_action']
action = config['notifications']["action"]
result = requests_actions[action](config['notifications_uri'], headers=headers, data=data)
result = requests_actions[action](config['notifications']["uri"], headers=headers, data=data)
logger.debug(f"Result: {result}")

6
bootstrap-daemon/pvcbootstrapd/lib/redfish.py
View File

@ -31,7 +31,6 @@ import math
from time import sleep
from celery.utils.log import get_task_logger
import pvcbootstrapd.lib.notifications as notifications
import pvcbootstrapd.lib.installer as installer
import pvcbootstrapd.lib.db as db
@ -689,8 +688,6 @@ def redfish_init(config, cspec, data):
cspec_cluster = cspec_node["node"]["cluster"]
cspec_hostname = cspec_node["node"]["hostname"]
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Beginning Redfish initialization of host {cspec_hostname}")
cluster = db.get_cluster(config, name=cspec_cluster)
if cluster is None:
cluster = db.add_cluster(config, cspec, cspec_cluster, "provisioning")
@ -855,7 +852,6 @@ def redfish_init(config, cspec, data):
node = db.update_node_state(config, cspec_cluster, cspec_hostname, "pxe-booting")
notifications.send_webhook(config, "success", f"Cluster {cspec_cluster}: Completed Redfish initialization of host {cspec_hostname}")
logger.info("Waiting for completion of node and cluster installation...")
# Wait for the system to install and be configured
while node.state != "booted-completed":
@ -866,7 +862,6 @@ def redfish_init(config, cspec, data):
node = db.get_node(config, cspec_cluster, name=cspec_hostname)
# Graceful shutdown of the machine
notifications.send_webhook(config, "begin", f"Cluster {cspec_cluster}: Powering off host {cspec_hostname}")
set_power_state(session, system_root, redfish_vendor, "GracefulShutdown")
system_power_state = "On"
while system_power_state != "Off":
@ -877,7 +872,6 @@ def redfish_init(config, cspec, data):
# Turn off the indicator to indicate bootstrap has completed
set_indicator_state(session, system_root, redfish_vendor, "off")
notifications.send_webhook(config, "completed", f"Cluster {cspec_cluster}: Powered off host {cspec_hostname}")
# We must delete the session
del session

5
bootstrap-daemon/pvcbootstrapd/lib/tftp.py
View File

@ -22,14 +22,11 @@
import os.path
import shutil
import pvcbootstrapd.lib.notifications as notifications
def build_tftp_repository(config):
# Generate an installer config
build_cmd = f"{config['ansible_path']}/pvc-installer/buildpxe.sh -o {config['tftp_root_path']} -u {config['deploy_username']}"
print(f"Building TFTP contents via pvc-installer command: {build_cmd}")
notifications.send_webhook(config, "begin", f"Building TFTP contents via pvc-installer command: {build_cmd}")
os.system(build_cmd)
@ -39,7 +36,6 @@ def init_tftp(config):
"""
if not os.path.exists(config["tftp_root_path"]):
print("First run: building TFTP root and contents - this will take some time!")
notifications.send_webhook(config, "begin", "First run: building TFTP root and contents")
os.makedirs(config["tftp_root_path"])
os.makedirs(config["tftp_host_path"])
shutil.copyfile(
@ -47,4 +43,3 @@ def init_tftp(config):
)
build_tftp_repository(config)
notifications.send_webhook(config, "success", "First run: successfully initialized TFTP root and contents")