Fix completion for all nodes

Avoids reprovisioning existing nodes until they are manually removed
from the database.

bootstrap-daemon/pvcbootstrapd/lib/hooks.py

@@ -69,6 +69,7 @@ def run_hook_osddb(config, targets, args):
             stdin, stdout, stderr = c.exec_command(pvc_cmd_string)
             logger.debug(stdout.readlines())
             logger.debug(stderr.readlines())
+        return stdout.channel.recv_exit_status()
 
 
 def run_hook_osd(config, targets, args):
@@ -98,6 +99,7 @@ def run_hook_osd(config, targets, args):
             stdin, stdout, stderr = c.exec_command(pvc_cmd_string)
             logger.debug(stdout.readlines())
             logger.debug(stderr.readlines())
+        return stdout.channel.recv_exit_status()
 
 
 def run_hook_pool(config, targets, args):
@@ -127,7 +129,7 @@ def run_hook_pool(config, targets, args):
             logger.debug(stderr.readlines())
 
         # This only runs once on whatever the first node is
-        break
+        return stdout.channel.recv_exit_status()
 
 
 def run_hook_network(config, targets, args):
@@ -191,7 +193,7 @@ def run_hook_network(config, targets, args):
             logger.debug(stderr.readlines())
 
         # This only runs once on whatever the first node is
-        break
+        return stdout.channel.recv_exit_status()
 
 
 def run_hook_copy(config, targets, args):
@@ -217,11 +219,14 @@ def run_hook_copy(config, targets, args):
                 tc.chmod(dfile, int(dmode, 8))
                 tc.close()
 
+    return 0
+
 
 def run_hook_script(config, targets, args):
     """
     Run a script on the targets
     """
+    return_status = 0
     for node in targets:
         node_name = node.name
         node_address = node.host_ipaddr
@@ -272,6 +277,10 @@ def run_hook_script(config, targets, args):
             stdin, stdout, stderr = c.exec_command(remote_command)
             logger.debug(stdout.readlines())
             logger.debug(stderr.readlines())
+            if stdout.channel.recv_exit_status() != 0:
+                return_status = stdout.channel.recv_exit_status()
+
+    return return_status
 
 
 def run_hook_webhook(config, targets, args):
@@ -345,7 +354,9 @@ def run_hooks(config, cspec, cluster, nodes):
         # Run the hook function
         try:
             notifications.send_webhook(config, "begin", f"Cluster {cluster.name}: Running hook task '{hook_name}'")
-            hook_functions[hook_type](config, target_nodes, hook_args)
+            retcode = hook_functions[hook_type](config, target_nodes, hook_args)
+            if retcode > 0:
+                raise Exception(f"Hook returned with code {retcode}")
             notifications.send_webhook(config, "success", f"Cluster {cluster.name}: Completed hook task '{hook_name}'")
         except Exception as e:
             logger.warning(f"Error running hook: {e}")

bootstrap-daemon/pvcbootstrapd/lib/host.py

@@ -84,3 +84,16 @@ def set_boot_state(config, cspec, data, state):
     db.update_node_state(config, cspec_cluster, cspec_hostname, state)
     node = db.get_node(config, cspec_cluster, name=cspec_hostname)
     logger.debug(node)
+
+
+def set_completed(config, cspec, cluster):
+    nodes = list()
+    for bmc_macaddr in cspec["bootstrap"]:
+        if cspec["bootstrap"][bmc_macaddr]["node"]["cluster"] == cluster:
+            nodes.append(cspec["bootstrap"][bmc_macaddr])
+    for node in nodes:
+        cspec_cluster = node["node"]["cluster"]
+        cspec_hostname = node["node"]["hostname"]
+        db.update_node_state(config, cspec_cluster, cspec_hostname, "completed")
+        node = db.get_node(config, cspec_cluster, name=cspec_hostname)
+        logger.debug(node)

bootstrap-daemon/pvcbootstrapd/lib/lib.py

@@ -50,24 +50,38 @@ def dnsmasq_checkin(config, data):
         )
         cspec = git.load_cspec_yaml(config)
         is_in_bootstrap_map = True if data["macaddr"] in cspec["bootstrap"] else False
-        if is_in_bootstrap_map:
-            notifications.send_webhook(config, "info", f"New host checkin from MAC {data['macaddr']} as host {cspec['bootstrap'][data['macaddr']]['node']['fqdn']} in cluster {cspec['bootstrap'][data['macaddr']]['node']['cluster']}")
-            if (
-                cspec["bootstrap"][data["macaddr"]]["bmc"].get("redfish", None)
-                is not None
-            ):
-                if cspec["bootstrap"][data["macaddr"]]["bmc"]["redfish"]:
-                    is_redfish = True
-                else:
-                    is_redfish = False
+        try:
+            if is_in_bootstrap_map:
+                cspec_cluster = cspec["bootstrap"][data["macaddr"]]["node"]["cluster"]
+                is_registered = True if data["macaddr"] in [x.bmc_macaddr for x in db.get_nodes_in_cluster(config, cspec_cluster)] else False
             else:
-                is_redfish = redfish.check_redfish(config, data)
+                is_registered = False
+        except Exception:
+            is_registered = False
 
-            logger.info(f"Is device '{data['macaddr']}' Redfish capable? {is_redfish}")
-            if is_redfish:
-                redfish.redfish_init(config, cspec, data)
-        else:
+        if not is_in_bootstrap_map:
             logger.warn(f"Device '{data['macaddr']}' not in bootstrap map; ignoring.")
+            return
+
+        if is_registered:
+            logger.info(f"Device '{data['macaddr']}' has already been bootstrapped; ignoring.")
+            return
+
+        notifications.send_webhook(config, "info", f"New host checkin from MAC {data['macaddr']} as host {cspec['bootstrap'][data['macaddr']]['node']['fqdn']} in cluster {cspec['bootstrap'][data['macaddr']]['node']['cluster']}")
+        if (
+            cspec["bootstrap"][data["macaddr"]]["bmc"].get("redfish", None)
+            is not None
+        ):
+            if cspec["bootstrap"][data["macaddr"]]["bmc"]["redfish"]:
+                is_redfish = True
+            else:
+                is_redfish = False
+        else:
+            is_redfish = redfish.check_redfish(config, data)
+
+        logger.info(f"Is device '{data['macaddr']}' Redfish capable? {is_redfish}")
+        if is_redfish:
+            redfish.redfish_init(config, cspec, data)
 
         return
 
@@ -140,9 +154,7 @@ def host_checkin(config, data):
 
             hooks.run_hooks(config, cspec, cluster, ready_nodes)
 
-            target_state = "completed"
-            for node in all_nodes:
-                host.set_boot_state(config, cspec, data, target_state)
+            host.set_completed(config, cspec, cluster)
 
             # Hosts will now power down ready for real activation in production
             sleep(60)