#!/usr/bin/env python3 # git.py - PVC Cluster Auto-bootstrap Git repository libraries # Part of the Parallel Virtual Cluster (PVC) system # # Copyright (C) 2018-2021 Joshua M. Boniface # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, version 3. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # ############################################################################### import os.path import git import yaml import pvcbootstrapd.lib.notifications as notifications from celery.utils.log import get_task_logger logger = get_task_logger(__name__) def init_repository(config): """ Clone the Ansible git repository """ try: git_ssh_cmd = f"ssh -i {config['ansible_keyfile']} -o StrictHostKeyChecking=no" if not os.path.exists(config["ansible_path"]): print( f"First run: cloning repository {config['ansible_remote']} branch {config['ansible_branch']} to {config['ansible_path']}" ) notifications.send_webhook(config, "begin", f"First run: cloning repository {config['ansible_remote']} branch {config['ansible_branch']} to {config['ansible_path']}") git.Repo.clone_from( config["ansible_remote"], config["ansible_path"], branch=config["ansible_branch"], env=dict(GIT_SSH_COMMAND=git_ssh_cmd), ) g = git.cmd.Git(f"{config['ansible_path']}") g.checkout(config["ansible_branch"]) g.submodule("update", "--init", env=dict(GIT_SSH_COMMAND=git_ssh_cmd)) notifications.send_webhook(config, "success", "First run: successfully initialized Git repository") except Exception as e: print(f"Error: {e}") def pull_repository(config): """ Pull (with rebase) the Ansible git repository """ logger.info(f"Updating local configuration repository {config['ansible_path']}") try: git_ssh_cmd = f"ssh -i {config['ansible_keyfile']} -o StrictHostKeyChecking=no" g = git.cmd.Git(f"{config['ansible_path']}") g.pull(rebase=True, env=dict(GIT_SSH_COMMAND=git_ssh_cmd)) g.submodule("update", "--init", env=dict(GIT_SSH_COMMAND=git_ssh_cmd)) except Exception as e: logger.warn(e) def commit_repository(config): """ Commit uncommitted changes to the Ansible git repository """ logger.info( f"Committing changes to local configuration repository {config['ansible_path']}" ) try: g = git.cmd.Git(f"{config['ansible_path']}") g.add("--all") commit_env = { "GIT_COMMITTER_NAME": "PVC Bootstrap", "GIT_COMMITTER_EMAIL": "git@pvcbootstrapd", } g.commit( "-m", "Automated commit from PVC Bootstrap Ansible subsystem", author="PVC Bootstrap ", env=commit_env, ) except Exception as e: logger.warn(e) def push_repository(config): """ Push changes to the default remote """ logger.info( f"Pushing changes from local configuration repository {config['ansible_path']}" ) try: git_ssh_cmd = f"ssh -i {config['ansible_keyfile']} -o StrictHostKeyChecking=no" g = git.Repo(f"{config['ansible_path']}") origin = g.remote(name="origin") origin.push(env=dict(GIT_SSH_COMMAND=git_ssh_cmd)) except Exception as e: logger.warn(e) def load_cspec_yaml(config): """ Load the bootstrap group_vars for all known clusters """ # Pull down the repository pull_repository(config) # Load our clusters file and read the clusters from it clusters_file = f"{config['ansible_path']}/{config['ansible_clusters_file']}" logger.info(f"Loading cluster configuration from file '{clusters_file}'") with open(clusters_file, "r") as clustersfh: clusters = yaml.load(clustersfh, Loader=yaml.SafeLoader).get("clusters", list()) # Define a base cpec cspec = { "bootstrap": dict(), "hooks": dict(), "clusters": dict(), } # Read each cluster's cspec and update the base cspec logger.info("Loading per-cluster specifications") for cluster in clusters: cspec["clusters"][cluster] = dict() cspec["clusters"][cluster]["bootstrap_nodes"] = list() cspec_file = f"{config['ansible_path']}/group_vars/{cluster}/{config['ansible_cspec_files_bootstrap']}" if os.path.exists(cspec_file): with open(cspec_file, "r") as cpsecfh: try: cspec_yaml = yaml.load(cpsecfh, Loader=yaml.SafeLoader) except Exception as e: logger.warn( f"Failed to load {config['ansible_cspec_files_bootstrap']} for cluster {cluster}: {e}" ) continue cspec["clusters"][cluster]["cspec_yaml"] = cspec_yaml # Convert the MAC address keys to lowercase # DNSMasq operates with lowercase keys, but often these are written with uppercase. # Convert them to lowercase to prevent discrepancies later on. cspec_yaml["bootstrap"] = { k.lower(): v for k, v in cspec_yaml["bootstrap"].items() } # Load in the YAML for the cluster base_yaml = load_base_yaml(config, cluster) cspec["clusters"][cluster]["base_yaml"] = base_yaml pvc_yaml = load_pvc_yaml(config, cluster) cspec["clusters"][cluster]["pvc_yaml"] = pvc_yaml # Set per-node values from elsewhere for node in cspec_yaml["bootstrap"]: cspec["clusters"][cluster]["bootstrap_nodes"].append( cspec_yaml["bootstrap"][node]["node"]["hostname"] ) # Set the cluster value automatically cspec_yaml["bootstrap"][node]["node"]["cluster"] = cluster # Set the domain value automatically via base config cspec_yaml["bootstrap"][node]["node"]["domain"] = base_yaml[ "local_domain" ] # Set the node FQDN value automatically cspec_yaml["bootstrap"][node]["node"][ "fqdn" ] = f"{cspec_yaml['bootstrap'][node]['node']['hostname']}.{cspec_yaml['bootstrap'][node]['node']['domain']}" # Append bootstrap entries to the main dictionary cspec["bootstrap"] = {**cspec["bootstrap"], **cspec_yaml["bootstrap"]} # Append hooks to the main dictionary (per-cluster) if cspec_yaml.get("hooks"): cspec["hooks"][cluster] = cspec_yaml["hooks"] logger.info("Finished loading per-cluster specifications") return cspec def load_base_yaml(config, cluster): """ Load the base.yml group_vars for a cluster """ base_file = f"{config['ansible_path']}/group_vars/{cluster}/{config['ansible_cspec_files_base']}" with open(base_file, "r") as varsfile: base_yaml = yaml.load(varsfile, Loader=yaml.SafeLoader) return base_yaml def load_pvc_yaml(config, cluster): """ Load the pvc.yml group_vars for a cluster """ pvc_file = f"{config['ansible_path']}/group_vars/{cluster}/{config['ansible_cspec_files_pvc']}" with open(pvc_file, "r") as varsfile: pvc_yaml = yaml.load(varsfile, Loader=yaml.SafeLoader) return pvc_yaml