From 3f0b0b2d7b60e398ada282d4b5432ba8f45d6787 Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Wed, 30 Aug 2023 14:25:50 +0000 Subject: [PATCH] Add support for customized mirror and apt-cacher --- bootstrap-daemon/pvcbootstrapd.yaml.sample | 6 ++++++ bootstrap-daemon/pvcbootstrapd.yaml.template | 2 ++ bootstrap-daemon/pvcbootstrapd/Daemon.py | 10 ++++++++++ .../pvcbootstrapd/lib/installer.py | 2 +- bootstrap-daemon/pvcbootstrapd/lib/tftp.py | 2 +- install-pvcbootstrapd.sh | 18 +++++++++++++++++- 6 files changed, 37 insertions(+), 3 deletions(-) diff --git a/bootstrap-daemon/pvcbootstrapd.yaml.sample b/bootstrap-daemon/pvcbootstrapd.yaml.sample index b8e52ed..6ba9d91 100644 --- a/bootstrap-daemon/pvcbootstrapd.yaml.sample +++ b/bootstrap-daemon/pvcbootstrapd.yaml.sample @@ -58,6 +58,12 @@ pvc: # Per-host TFTP path (almost always "/host" under "root_path"; must be writable) host_path: "/srv/tftp/pvc-installer/host" + # Debian repository configuration + repo: + # Mirror path; defaults to using the apt-cacher-ng instance located on the current machine + # Replace "10.199.199.254" if you change "dhcp" -> "address" above + mirror: http://10.199.199.254:3142/ftp.debian.org/debian + # PVC Ansible repository configuration # Note: If "path" does not exist, "remote" will be cloned to it via Git using SSH private key "keyfile". # Note: The VCS will be refreshed regularly via the API in response to webhooks. diff --git a/bootstrap-daemon/pvcbootstrapd.yaml.template b/bootstrap-daemon/pvcbootstrapd.yaml.template index 075df41..f74c0f8 100644 --- a/bootstrap-daemon/pvcbootstrapd.yaml.template +++ b/bootstrap-daemon/pvcbootstrapd.yaml.template @@ -21,6 +21,8 @@ pvc: tftp: root_path: "ROOT_DIRECTORY/tftp" host_path: "ROOT_DIRECTORY/tftp/host" + repo: + mirror: http://BOOTSTRAP_ADDRESS:3142/UPSTREAM_MIRROR ansible: path: "ROOT_DIRECTORY/repo" keyfile: "ROOT_DIRECTORY/id_ed25519" diff --git a/bootstrap-daemon/pvcbootstrapd/Daemon.py b/bootstrap-daemon/pvcbootstrapd/Daemon.py index 16924a1..f0f053d 100755 --- a/bootstrap-daemon/pvcbootstrapd/Daemon.py +++ b/bootstrap-daemon/pvcbootstrapd/Daemon.py @@ -121,6 +121,7 @@ def read_config(): o_queue = o_base["queue"] o_dhcp = o_base["dhcp"] o_tftp = o_base["tftp"] + o_repo = o_base["repo"] o_ansible = o_base["ansible"] o_notifications = o_base["notifications"] except KeyError as k: @@ -178,6 +179,15 @@ def read_config(): f"Missing second-level key '{key}' under 'tftp'" ) + # Get the Repo configuration + for key in ["mirror"]: + try: + config[f"repo_{key}"] = o_repo[key] + except Exception: + raise MalformedConfigurationError( + f"Missing second-level key '{key}' under 'repo'" + ) + # Get the Ansible configuration for key in ["path", "key_file", "remote", "branch", "clusters_file", "lock_file"]: try: diff --git a/bootstrap-daemon/pvcbootstrapd/lib/installer.py b/bootstrap-daemon/pvcbootstrapd/lib/installer.py index 4b46cee..1e79963 100755 --- a/bootstrap-daemon/pvcbootstrapd/lib/installer.py +++ b/bootstrap-daemon/pvcbootstrapd/lib/installer.py @@ -67,7 +67,7 @@ def add_preseed(config, cspec_node, host_macaddr, system_drive_target): # We use the dhcp_address here to allow the listen_address to be 0.0.0.0 rendered = template.render( debrelease=cspec_node.get("config", {}).get("release"), - debmirror=cspec_node.get("config", {}).get("mirror"), + debmirror=config.get("repo_mirror"), addpkglist=add_packages, filesystem=cspec_node.get("config", {}).get("filesystem"), skip_blockcheck=False, diff --git a/bootstrap-daemon/pvcbootstrapd/lib/tftp.py b/bootstrap-daemon/pvcbootstrapd/lib/tftp.py index d297913..bf9f2be 100755 --- a/bootstrap-daemon/pvcbootstrapd/lib/tftp.py +++ b/bootstrap-daemon/pvcbootstrapd/lib/tftp.py @@ -28,7 +28,7 @@ import pvcbootstrapd.lib.notifications as notifications def build_tftp_repository(config): # Generate an installer config - build_cmd = [ f"{config['ansible_path']}/pvc-installer/buildpxe.sh", "-o", config['tftp_root_path'], "-u", config['deploy_username'] ] + build_cmd = [ f"{config['ansible_path']}/pvc-installer/buildpxe.sh", "-o", config['tftp_root_path'], "-u", config['deploy_username'], "-m", config["repo_mirror"] ] print(f"Building TFTP contents via pvc-installer command: {' '.join(build_cmd)}") notifications.send_webhook(config, "begin", f"Building TFTP contents via pvc-installer command: {' '.join(build_cmd)}") retcode = run(build_cmd) diff --git a/install-pvcbootstrapd.sh b/install-pvcbootstrapd.sh index 4e08251..0b0948d 100755 --- a/install-pvcbootstrapd.sh +++ b/install-pvcbootstrapd.sh @@ -95,12 +95,27 @@ if [[ -z ${deploy_username} ]]; then fi echo +echo "Please enter an upstream Debian mirror (hostname+directory without scheme) to use (e.g. ftp.debian.org/debian):" +echo -n "[ftp.debian.org/debian] > " +read upstream_mirror +if [[ -z ${upstream_mirror} ]]; then + upstream_mirror="ftp.debian.org/debian" +fi +echo + echo "Proceeding with setup!" echo echo "Installing APT dependencies..." sudo apt-get update -sudo apt-get install --yes vlan iptables dnsmasq redis python3 python3-pip python3-requests sqlite3 celery pxelinux syslinux-common live-build debootstrap uuid-runtime qemu-user-static +sudo apt-get install --yes vlan iptables dnsmasq redis python3 python3-pip python3-requests sqlite3 celery pxelinux syslinux-common live-build debootstrap uuid-runtime qemu-user-static apt-cacher-ng + +echo "Configuring apt-cacher-ng..." +sudo systemctl enable --now apt-cacher-ng +if ! grep -q ${upstream_mirror} /etc/apt-cacher-ng/backends_debian; then + echo "http://${upstream_mirror}" | sudo tee /etc/apt-cacher-ng/backends_debian &>/dev/null + sudo systemctl restart apt-cacher-ng +fi echo "Configuring dnsmasq..." sudo systemctl disable --now dnsmasq @@ -131,6 +146,7 @@ sed -i "s|BOOTSTRAP_DHCPSTART|${bootstrap_dhcpstart}|" ${root_directory}/pvcboot sed -i "s|BOOTSTRAP_DHCPEND|${bootstrap_dhcpend}|" ${root_directory}/pvcbootstrapd/pvcbootstrapd.yaml sed -i "s|GIT_REMOTE|${git_remote}|" ${root_directory}/pvcbootstrapd/pvcbootstrapd.yaml sed -i "s|GIT_BRANCH|${git_branch}|" ${root_directory}/pvcbootstrapd/pvcbootstrapd.yaml +sed -i "s|UPSTREAM_MIRROR|${upstream_mirror}|" ${root_directory}/pvcbootstrapd/pvcbootstrapd.yaml echo "Creating network configuration for interface ${bootstrap_interface} (is vLAN? ${is_bootstrap_interface_vlan})..." if [[ "${is_bootstrap_interface_vlan}" == "yes" ]]; then