#!/usr/bin/env python3 # git.py - PVC Cluster Auto-bootstrap Git repository libraries # Part of the Parallel Virtual Cluster (PVC) system # # Copyright (C) 2018-2021 Joshua M. Boniface # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, version 3. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # ############################################################################### import os.path import git import yaml from celery.utils.log import get_task_logger logger = get_task_logger(__name__) def init_repository(config): """ Clone the Ansible git repository """ if not os.path.exists(config['ansible_path']): logger.info(f"Cloning configuration repository {config['ansible_remote']} branch {config['ansible_branch']} to {config['ansible_path']}") git_ssh_cmd = f"ssh -i {config['ansible_keyfile']}" with git.Git().custom_environment(GIT_SSH_COMMAND=git_ssh_cmd): git.Repo.clone_from(config['ansible_remote'], config['ansible_path'], branch=config['ansible_branch']) g = git.cmd.Git(f"{config['ansible_path']}") else: g = git.cmd.Git(f"{config['ansible_path']}") g.checkout(config['ansible_branch']) for submodule in g.submodules: submodule.update(init=True) def pull_repository(config): """ Pull (with rebase) the Ansible git repository """ logger.info(f"Updating local configuration repository {config['ansible_path']}") try: git_ssh_cmd = f"ssh -i {config['ansible_keyfile']}" with git.Git().custom_environment(GIT_SSH_COMMAND=git_ssh_cmd): g = git.cmd.Git(f"{config['ansible_path']}") g.pull(rebase=True) except Exception as e: logger.warn(e) def commit_repository(config): """ Commit uncommitted changes to the Ansible git repository """ logger.info(f"Committing changes to local configuration repository {config['ansible_path']}") try: g = git.cmd.Git(f"{config['ansible_path']}") g.add('--all') g.commit( '-m', 'Automated commit from PVC Bootstrap Ansible subsystem', author="PVC Bootstrap " ) except Exception as e: logger.warn(e) def push_repository(config): """ Push changes to the default remote """ logger.info(f"Pushing changes from local configuration repository {config['ansible_path']}") try: g = git.cmd.Git(f"{config['ansible_path']}") origin = g.remote(name='origin') origin.push() except Exception as e: logger.warn(e) def load_cspec_yaml(config): """ Load the bootstrap group_vars for all known clusters """ # Pull down the repository pull_repository(config) # Load our clusters file and read the clusters from it clusters_file = f"{config['ansible_path']}/{config['ansible_clusters_file']}" logger.info(f"Loading cluster configuration from file '{clusters_file}'") with open(clusters_file, 'r') as clustersfh: clusters = yaml.load(clustersfh, Loader=yaml.SafeLoader).get('clusters', list()) # Define a base cpec cspec = { 'bootstrap': dict(), 'hooks': dict(), } # Read each cluster's cspec and update the base cspec logger.info(f"Loading per-cluster specifications...") for cluster in clusters: cspec_file = f"{config['ansible_path']}/group_vars/{cluster}/{config['ansible_cspec_files_bootstrap']}" if os.path.exists(cspec_file): with open(cspec_file, 'r') as cpsecfh: try: cspec_yaml = yaml.load(cpsecfh, Loader=yaml.SafeLoader) except Exception as e: logger.warn(f"Failed to load {config['ansible_cspec_files_bootstrap']} for cluster {cluster}: {e}") continue # Convert the MAC address keys to lowercase # DNSMasq operates with lowercase keys, but often these are written with uppercase. # Convert them to lowercase to prevent discrepancies later on. cspec_yaml['bootstrap'] = {k.lower(): v for k, v in cspec_yaml['bootstrap'].items()} # Load in the base YAML for the cluster base_yaml = load_base_yaml(config, cluster) # Set per-node values from elsewhere for node in cspec_yaml['bootstrap']: # Set the cluster value automatically cspec_yaml['bootstrap'][node]['node']['cluster'] = cluster # Set the domain value automatically via base config cspec_yaml['bootstrap'][node]['node']['domain'] = base_yaml['local_domain'] # Set the node FQDN value automatically cspec_yaml['bootstrap'][node]['node']['fqdn'] = f"{cspec_yaml['bootstrap'][node]['node']['hostname']}.{cspec_yaml['bootstrap'][node]['node']['domain']}" # Append bootstrap entries to the main dictionary cspec['bootstrap'] = {**cspec['bootstrap'], **cspec_yaml['bootstrap']} # Append hooks to the main dictionary (per-cluster) if cspec_yaml.get('hooks'): cspec['hooks'][cluster] = cspec_yaml['hooks'] logger.info(f"Finished loading per-cluster specifications") logger.debug(f"cspec = {cspec}") return cspec def load_base_yaml(config, cluster): """ Load the base.yml group_vars for a cluster """ base_file = f"{config['ansible_path']}/group_vars/{cluster}/base.yml" with open(base_file, 'r') as varsfile: base_yaml = yaml.load(varsfile, Loader=yaml.SafeLoader) return base_yaml