Standardize names and lock config

Avoids conflicting attempts when multiple hosts check in at once.

bootstrap-daemon/pvcbootstrapd.yaml.sample

@@ -66,7 +66,7 @@ pvc:
     path: "/var/home/joshua/pvc"
 
     # Path to the deploy key (if applicable) used to clone and pull the repository
-    keyfile: "/var/home/joshua/id_ed25519.joshua.key"
+    key_file: "/var/home/joshua/id_ed25519.joshua.key"
 
     # Git remote URI for the repository
     remote: "ssh://git@git.bonifacelabs.ca:2222/bonifacelabs/pvc.git"
@@ -77,6 +77,9 @@ pvc:
     # Clusters configuration file
     clusters_file: "clusters.yml"
 
+    # Lock file to use for Git interaction
+    lock_file: "/run/pvcbootstrapd.lock"
+
     # Filenames of the various group_vars components of a cluster
     # Generally with pvc-ansible this will contain 2 files: "base.yml", and "pvc.yml"; refer to the
     # pvc-ansible documentation and examples for details on these files.

bootstrap-daemon/pvcbootstrapd/Daemon.py

@@ -179,7 +179,7 @@ def read_config():
             )
 
     # Get the Ansible configuration
-    for key in ["path", "keyfile", "remote", "branch", "clusters_file"]:
+    for key in ["path", "key_file", "remote", "branch", "clusters_file", "lock_file"]:
         try:
             config[f"ansible_{key}"] = o_ansible[key]
         except Exception:

bootstrap-daemon/pvcbootstrapd/lib/ansible.py

@@ -66,7 +66,7 @@ def run_bootstrap(config, cspec, cluster, nodes):
                 limit=f"{cluster.name}",
                 playbook=f"{config['ansible_path']}/pvc.yml",
                 extravars={
-                    "ansible_ssh_private_key_file": config["ansible_keyfile"],
+                    "ansible_ssh_private_key_file": config["ansible_key_file"],
                     "bootstrap": "yes",
                 },
                 forks=len(nodes),

bootstrap-daemon/pvcbootstrapd/lib/git.py

@@ -22,6 +22,7 @@
 import os.path
 import git
 import yaml
+from filelock import FileLock
 
 import pvcbootstrapd.lib.notifications as notifications
 
@@ -36,7 +37,7 @@ def init_repository(config):
     Clone the Ansible git repository
     """
     try:
-        git_ssh_cmd = f"ssh -i {config['ansible_keyfile']} -o StrictHostKeyChecking=no"
+        git_ssh_cmd = f"ssh -i {config['ansible_key_file']} -o StrictHostKeyChecking=no"
         if not os.path.exists(config["ansible_path"]):
             print(
                 f"First run: cloning repository {config['ansible_remote']} branch {config['ansible_branch']} to {config['ansible_path']}"
@@ -60,25 +61,29 @@ def pull_repository(config):
     """
     Pull (with rebase) the Ansible git repository
     """
+    with FileLock(config['ansible_lock_file']):
         logger.info(f"Updating local configuration repository {config['ansible_path']}")
         try:
-        git_ssh_cmd = f"ssh -i {config['ansible_keyfile']} -o StrictHostKeyChecking=no"
+            git_ssh_cmd = f"ssh -i {config['ansible_key_file']} -o StrictHostKeyChecking=no"
             g = git.cmd.Git(f"{config['ansible_path']}")
+            logger.debug("Performing git pull")
             g.pull(rebase=True, env=dict(GIT_SSH_COMMAND=git_ssh_cmd))
+            logger.debug("Performing git submodule update")
             g.submodule("update", "--init", env=dict(GIT_SSH_COMMAND=git_ssh_cmd))
         except Exception as e:
             logger.warn(e)
             notifications.send_webhook(config, "failure", "Failed to update Git repository")
+    logger.info("Completed repository synchonization")
 
 
 def commit_repository(config):
     """
     Commit uncommitted changes to the Ansible git repository
     """
+    with FileLock(config['ansible_lock_file']):
         logger.info(
             f"Committing changes to local configuration repository {config['ansible_path']}"
         )
-
         try:
             g = git.cmd.Git(f"{config['ansible_path']}")
             g.add("--all")
@@ -102,12 +107,12 @@ def push_repository(config):
     """
     Push changes to the default remote
     """
+    with FileLock(config['ansible_lock_file']):
         logger.info(
             f"Pushing changes from local configuration repository {config['ansible_path']}"
         )
-
         try:
-        git_ssh_cmd = f"ssh -i {config['ansible_keyfile']} -o StrictHostKeyChecking=no"
+            git_ssh_cmd = f"ssh -i {config['ansible_key_file']} -o StrictHostKeyChecking=no"
             g = git.Repo(f"{config['ansible_path']}")
             origin = g.remote(name="origin")
             origin.push(env=dict(GIT_SSH_COMMAND=git_ssh_cmd))

bootstrap-daemon/pvcbootstrapd/lib/hooks.py

@@ -43,7 +43,7 @@ def run_paramiko(config, node_address):
     ssh_client.connect(
         hostname=node_address,
         username=config["deploy_username"],
-        key_filename=config["ansible_keyfile"],
+        key_filename=config["ansible_key_file"],
     )
     yield ssh_client
     ssh_client.close()

bootstrap-daemon/pvcbootstrapd/lib/tftp.py

@@ -43,7 +43,7 @@ def init_tftp(config):
         os.makedirs(config["tftp_root_path"])
         os.makedirs(config["tftp_host_path"])
         shutil.copyfile(
-            f"{config['ansible_keyfile']}.pub", f"{config['tftp_root_path']}/keys.txt"
+            f"{config['ansible_key_file']}.pub", f"{config['tftp_root_path']}/keys.txt"
         )
 
         build_tftp_repository(config)