Add support for customized mirror and apt-cacher

bootstrap-daemon/pvcbootstrapd.yaml.sample

@@ -58,6 +58,12 @@ pvc:
     # Per-host TFTP path (almost always "/host" under "root_path"; must be writable)
     host_path: "/srv/tftp/pvc-installer/host"
 
+  # Debian repository configuration
+  repo:
+    # Mirror path; defaults to using the apt-cacher-ng instance located on the current machine
+    # Replace "10.199.199.254" if you change "dhcp" -> "address" above
+    mirror: http://10.199.199.254:3142/ftp.debian.org/debian
+
   # PVC Ansible repository configuration
   # Note: If "path" does not exist, "remote" will be cloned to it via Git using SSH private key "keyfile".
   # Note: The VCS will be refreshed regularly via the API in response to webhooks.

bootstrap-daemon/pvcbootstrapd.yaml.template

@@ -21,6 +21,8 @@ pvc:
   tftp:
     root_path: "ROOT_DIRECTORY/tftp"
     host_path: "ROOT_DIRECTORY/tftp/host"
+  repo:
+    mirror: http://BOOTSTRAP_ADDRESS:3142/UPSTREAM_MIRROR
   ansible:
     path: "ROOT_DIRECTORY/repo"
     keyfile: "ROOT_DIRECTORY/id_ed25519"

bootstrap-daemon/pvcbootstrapd/Daemon.py

@@ -121,6 +121,7 @@ def read_config():
         o_queue = o_base["queue"]
         o_dhcp = o_base["dhcp"]
         o_tftp = o_base["tftp"]
+        o_repo = o_base["repo"]
         o_ansible = o_base["ansible"]
         o_notifications = o_base["notifications"]
     except KeyError as k:
@@ -178,6 +179,15 @@ def read_config():
                 f"Missing second-level key '{key}' under 'tftp'"
             )
 
+    # Get the Repo configuration
+    for key in ["mirror"]:
+        try:
+            config[f"repo_{key}"] = o_repo[key]
+        except Exception:
+            raise MalformedConfigurationError(
+                f"Missing second-level key '{key}' under 'repo'"
+            )
+
     # Get the Ansible configuration
     for key in ["path", "key_file", "remote", "branch", "clusters_file", "lock_file"]:
         try:

bootstrap-daemon/pvcbootstrapd/lib/installer.py

@@ -67,7 +67,7 @@ def add_preseed(config, cspec_node, host_macaddr, system_drive_target):
     # We use the dhcp_address here to allow the listen_address to be 0.0.0.0
     rendered = template.render(
         debrelease=cspec_node.get("config", {}).get("release"),
-        debmirror=cspec_node.get("config", {}).get("mirror"),
+        debmirror=config.get("repo_mirror"),
         addpkglist=add_packages,
         filesystem=cspec_node.get("config", {}).get("filesystem"),
         skip_blockcheck=False,

bootstrap-daemon/pvcbootstrapd/lib/tftp.py

@@ -28,7 +28,7 @@ import pvcbootstrapd.lib.notifications as notifications
 
 def build_tftp_repository(config):
     # Generate an installer config
-    build_cmd = [ f"{config['ansible_path']}/pvc-installer/buildpxe.sh", "-o", config['tftp_root_path'], "-u", config['deploy_username'] ]
+    build_cmd = [ f"{config['ansible_path']}/pvc-installer/buildpxe.sh", "-o", config['tftp_root_path'], "-u", config['deploy_username'], "-m", config["repo_mirror"] ]
     print(f"Building TFTP contents via pvc-installer command: {' '.join(build_cmd)}")
     notifications.send_webhook(config, "begin", f"Building TFTP contents via pvc-installer command: {' '.join(build_cmd)}")
     retcode = run(build_cmd)

install-pvcbootstrapd.sh

@@ -95,12 +95,27 @@ if [[ -z ${deploy_username} ]]; then
 fi
 echo
 
+echo "Please enter an upstream Debian mirror (hostname+directory without scheme) to use (e.g. ftp.debian.org/debian):"
+echo -n "[ftp.debian.org/debian] > "
+read upstream_mirror
+if [[ -z ${upstream_mirror} ]]; then
+    upstream_mirror="ftp.debian.org/debian"
+fi
+echo
+
 echo "Proceeding with setup!"
 echo
 
 echo "Installing APT dependencies..."
 sudo apt-get update
-sudo apt-get install --yes vlan iptables dnsmasq redis python3 python3-pip python3-requests sqlite3 celery pxelinux syslinux-common live-build debootstrap uuid-runtime qemu-user-static
+sudo apt-get install --yes vlan iptables dnsmasq redis python3 python3-pip python3-requests sqlite3 celery pxelinux syslinux-common live-build debootstrap uuid-runtime qemu-user-static apt-cacher-ng
+
+echo "Configuring apt-cacher-ng..."
+sudo systemctl enable --now apt-cacher-ng
+if ! grep -q ${upstream_mirror} /etc/apt-cacher-ng/backends_debian; then
+    echo "http://${upstream_mirror}" | sudo tee /etc/apt-cacher-ng/backends_debian &>/dev/null
+    sudo systemctl restart apt-cacher-ng
+fi
 
 echo "Configuring dnsmasq..."
 sudo systemctl disable --now dnsmasq
@@ -131,6 +146,7 @@ sed -i "s|BOOTSTRAP_DHCPSTART|${bootstrap_dhcpstart}|" ${root_directory}/pvcboot
 sed -i "s|BOOTSTRAP_DHCPEND|${bootstrap_dhcpend}|" ${root_directory}/pvcbootstrapd/pvcbootstrapd.yaml
 sed -i "s|GIT_REMOTE|${git_remote}|" ${root_directory}/pvcbootstrapd/pvcbootstrapd.yaml
 sed -i "s|GIT_BRANCH|${git_branch}|" ${root_directory}/pvcbootstrapd/pvcbootstrapd.yaml
+sed -i "s|UPSTREAM_MIRROR|${upstream_mirror}|" ${root_directory}/pvcbootstrapd/pvcbootstrapd.yaml
 
 echo "Creating network configuration for interface ${bootstrap_interface} (is vLAN? ${is_bootstrap_interface_vlan})..."
 if [[ "${is_bootstrap_interface_vlan}" == "yes" ]]; then