Add locking to git commands

Avoids conflicting attempts when multiple hosts check in at once.

bootstrap-daemon/pvcbootstrapd/lib/git.py

@@ -22,6 +22,7 @@
 import os.path
 import git
 import yaml
+from filelock import FileLock
 
 import pvcbootstrapd.lib.notifications as notifications
 
@@ -60,61 +61,71 @@ def pull_repository(config):
     """
     Pull (with rebase) the Ansible git repository
     """
-    logger.info(f"Updating local configuration repository {config['ansible_path']}")
+    lockfile = "/tmp/pvcbootstrapd-git.lock"
-    try:
+    lock = FileLock(lockfile)
-        git_ssh_cmd = f"ssh -i {config['ansible_keyfile']} -o StrictHostKeyChecking=no"
+    with lock:
-        g = git.cmd.Git(f"{config['ansible_path']}")
+        logger.info(f"Updating local configuration repository {config['ansible_path']}")
-        g.pull(rebase=True, env=dict(GIT_SSH_COMMAND=git_ssh_cmd))
+        try:
-        g.submodule("update", "--init", env=dict(GIT_SSH_COMMAND=git_ssh_cmd))
+            git_ssh_cmd = f"ssh -i {config['ansible_keyfile']} -o StrictHostKeyChecking=no"
-    except Exception as e:
+            g = git.cmd.Git(f"{config['ansible_path']}")
-        logger.warn(e)
+            logger.debug("Performing git pull")
-        notifications.send_webhook(config, "failure", "Failed to update Git repository")
+            g.pull(rebase=True, env=dict(GIT_SSH_COMMAND=git_ssh_cmd))
+            logger.debug("Performing git submodule update")
+            g.submodule("update", "--init", env=dict(GIT_SSH_COMMAND=git_ssh_cmd))
+        except Exception as e:
+            logger.warn(e)
+            notifications.send_webhook(config, "failure", "Failed to update Git repository")
+    logger.info("Completed repository synchonization")
 
 
 def commit_repository(config):
     """
     Commit uncommitted changes to the Ansible git repository
     """
-    logger.info(
+    lockfile = "/tmp/pvcbootstrapd-git.lock"
-        f"Committing changes to local configuration repository {config['ansible_path']}"
+    lock = FileLock(lockfile)
-    )
+    with lock:
-
+        logger.info(
-    try:
+            f"Committing changes to local configuration repository {config['ansible_path']}"
-        g = git.cmd.Git(f"{config['ansible_path']}")
-        g.add("--all")
-        commit_env = {
-            "GIT_COMMITTER_NAME": "PVC Bootstrap",
-            "GIT_COMMITTER_EMAIL": "git@pvcbootstrapd",
-        }
-        g.commit(
-            "-m",
-            "Automated commit from PVC Bootstrap Ansible subsystem",
-            author="PVC Bootstrap <git@pvcbootstrapd>",
-            env=commit_env,
         )
-        notifications.send_webhook(config, "success", "Successfully committed to Git repository")
+        try:
-    except Exception as e:
+            g = git.cmd.Git(f"{config['ansible_path']}")
-        logger.warn(e)
+            g.add("--all")
-        notifications.send_webhook(config, "failure", "Failed to commit to Git repository")
+            commit_env = {
+                "GIT_COMMITTER_NAME": "PVC Bootstrap",
+                "GIT_COMMITTER_EMAIL": "git@pvcbootstrapd",
+            }
+            g.commit(
+                "-m",
+                "Automated commit from PVC Bootstrap Ansible subsystem",
+                author="PVC Bootstrap <git@pvcbootstrapd>",
+                env=commit_env,
+            )
+            notifications.send_webhook(config, "success", "Successfully committed to Git repository")
+        except Exception as e:
+            logger.warn(e)
+            notifications.send_webhook(config, "failure", "Failed to commit to Git repository")
 
 
 def push_repository(config):
     """
     Push changes to the default remote
     """
-    logger.info(
+    lockfile = "/tmp/pvcbootstrapd-git.lock"
-        f"Pushing changes from local configuration repository {config['ansible_path']}"
+    lock = FileLock(lockfile)
-    )
+    with lock:
-
+        logger.info(
-    try:
+            f"Pushing changes from local configuration repository {config['ansible_path']}"
-        git_ssh_cmd = f"ssh -i {config['ansible_keyfile']} -o StrictHostKeyChecking=no"
+        )
-        g = git.Repo(f"{config['ansible_path']}")
+        try:
-        origin = g.remote(name="origin")
+            git_ssh_cmd = f"ssh -i {config['ansible_keyfile']} -o StrictHostKeyChecking=no"
-        origin.push(env=dict(GIT_SSH_COMMAND=git_ssh_cmd))
+            g = git.Repo(f"{config['ansible_path']}")
-        notifications.send_webhook(config, "success", "Successfully pushed Git repository")
+            origin = g.remote(name="origin")
-    except Exception as e:
+            origin.push(env=dict(GIT_SSH_COMMAND=git_ssh_cmd))
-        logger.warn(e)
+            notifications.send_webhook(config, "success", "Successfully pushed Git repository")
-        notifications.send_webhook(config, "failure", "Failed to push Git repository")
+        except Exception as e:
+            logger.warn(e)
+            notifications.send_webhook(config, "failure", "Failed to push Git repository")
 
 
 def load_cspec_yaml(config):