pvc-bootstrap/bootstrap-daemon/pvcbootstrapd/lib/git.py

#!/usr/bin/env python3

# git.py - PVC Cluster Auto-bootstrap Git repository libraries
# Part of the Parallel Virtual Cluster (PVC) system
#
#    Copyright (C) 2018-2021 Joshua M. Boniface <joshua@boniface.me>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, version 3.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################

import os.path
import git
import yaml

from celery.utils.log import get_task_logger

 
logger = get_task_logger(__name__)


def init_repository(config):
    """
    Clone the Ansible git repository
    """
    if not os.path.exists(config['ansible_path']):
        logger.info(f"Cloning configuration repository {config['ansible_remote']} branch {config['ansible_branch']} to {config['ansible_path']}")
        git_ssh_cmd = f"ssh -i {config['ansible_keyfile']}"
        with git.Git().custom_environment(GIT_SSH_COMMAND=git_ssh_cmd):
            git.Repo.clone_from(config['ansible_remote'], config['ansible_path'], branch=config['ansible_branch'])
        g = git.cmd.Git(f"{config['ansible_path']}")
    else:
        g = git.cmd.Git(f"{config['ansible_path']}")
        g.checkout(config['ansible_branch'])

    for submodule in g.submodules:
        submodule.update(init=True)


def pull_repository(config):
    """
    Pull (with rebase) the Ansible git repository
    """
    logger.info(f"Updating local configuration repository {config['ansible_path']}")
    try:
        git_ssh_cmd = f"ssh -i {config['ansible_keyfile']}"
        with git.Git().custom_environment(GIT_SSH_COMMAND=git_ssh_cmd):
            g = git.cmd.Git(f"{config['ansible_path']}")
            g.pull(rebase=True)
    except Exception as e:
        logger.warn(e)


def commit_repository(config):
    """
    Commit uncommitted changes to the Ansible git repository
    """
    logger.info(f"Committing changes to local configuration repository {config['ansible_path']}")

    try:
        g = git.cmd.Git(f"{config['ansible_path']}")
        g.add('--all')
        g.commit(
            '-m',
            'Automated commit from PVC Bootstrap Ansible subsystem',
            author="PVC Bootstrap <git@parallelvirtualcluster.org>"
        )
    except Exception as e:
        logger.warn(e)


def push_repository(config):
    """
    Push changes to the default remote
    """
    logger.info(f"Pushing changes from local configuration repository {config['ansible_path']}")

    try:
        g = git.cmd.Git(f"{config['ansible_path']}")
        origin = g.remote(name='origin')
        origin.push()
    except Exception as e:
        logger.warn(e)


def load_cspec_yaml(config):
    """
    Load the bootstrap group_vars for all known clusters
    """
    # Pull down the repository
    pull_repository(config)

    # Load our clusters file and read the clusters from it
    clusters_file = f"{config['ansible_path']}/{config['ansible_clusters_file']}"
    logger.info(f"Loading cluster configuration from file '{clusters_file}'")
    with open(clusters_file, 'r') as clustersfh:
        clusters = yaml.load(clustersfh, Loader=yaml.SafeLoader).get('clusters', list())

    # Define a base cpec
    cspec = {
        'bootstrap': dict(),
        'hooks': dict(),
    }

    # Read each cluster's cspec and update the base cspec
    logger.info(f"Loading per-cluster specifications...")
    for cluster in clusters:
        cspec_file = f"{config['ansible_path']}/group_vars/{cluster}/{config['ansible_cspec_files_bootstrap']}"
        if os.path.exists(cspec_file):
            with open(cspec_file, 'r') as cpsecfh:
                try:
                    cspec_yaml = yaml.load(cpsecfh, Loader=yaml.SafeLoader)
                except Exception as e:
                    logger.warn(f"Failed to load {config['ansible_cspec_files_bootstrap']} for cluster {cluster}: {e}")
                    continue

            # Convert the MAC address keys to lowercase
            # DNSMasq operates with lowercase keys, but often these are written with uppercase.
            # Convert them to lowercase to prevent discrepancies later on.
            cspec_yaml['bootstrap'] = {k.lower(): v for k, v in cspec_yaml['bootstrap'].items()}

            # Load in the base YAML for the cluster
            base_yaml =  load_base_yaml(config, cluster)

            # Set per-node values from elsewhere
            for node in cspec_yaml['bootstrap']:
                # Set the cluster value automatically
                cspec_yaml['bootstrap'][node]['node']['cluster'] = cluster

                # Set the domain value automatically via base config
                cspec_yaml['bootstrap'][node]['node']['domain'] = base_yaml['local_domain']

                # Set the node FQDN value automatically
                cspec_yaml['bootstrap'][node]['node']['fqdn'] = f"{cspec_yaml['bootstrap'][node]['node']['hostname']}.{cspec_yaml['bootstrap'][node]['node']['domain']}"

            # Append bootstrap entries to the main dictionary
            cspec['bootstrap'] = {**cspec['bootstrap'], **cspec_yaml['bootstrap']}

            # Append hooks to the main dictionary (per-cluster)
            if cspec_yaml.get('hooks'):
                cspec['hooks'][cluster] = cspec_yaml['hooks']

    logger.info(f"Finished loading per-cluster specifications")
    logger.debug(f"cspec = {cspec}")
    return cspec


def load_base_yaml(config, cluster):
    """
    Load the base.yml group_vars for a cluster
    """
    base_file = f"{config['ansible_path']}/group_vars/{cluster}/base.yml"
    with open(base_file, 'r') as varsfile:
        base_yaml = yaml.load(varsfile, Loader=yaml.SafeLoader)

    return base_yaml