Handle target_disk detection strings

For preseed installs with pvcbootstrapd, implement "detect" strings,
which can be used instead of fixed block paths to determine the required
disk from a fixed set of information available to a human provisioning
the servers, or from Redfish.

The basic idea is thus:

  1. The user specifies some physical attributes of the disk, either
  manually in a detect string of the preseed configuration, or by a
  physical identifier that Redfish can identify.

  2. Redfish takes this and either passes it, or crafts a detect string
  itself based on its storage information, which is then passed to the
  installer preseed as the target_disk value.

  3. The installer uses the provided values along with the output of the
  "lsscsi" command to determine which block device to use for the system
  disk.

This supersedes and enhances the original "model-based" detection with
far greater reliability and the ability to specify specific indexes.

buildpxe.sh

@@ -124,8 +124,9 @@ build_pxe() {
     cp templates/boot.ipxe ${outputdir}/boot.ipxe
     echo "done."
 
-    echo -n "Copying preseed template... "
+    echo -n "Copying jinja2 templates... "
     cp templates/host-preseed.j2 ${outputdir}/host-preseed.j2
+    cp templates/host-ipxe.j2 ${outputdir}/host-ipxe.j2
     echo "done."
 
     sudo chown -R $(whoami) ${outputdir}

templates/host-ipxe.j2

@@ -1,5 +1,5 @@
 #!ipxe
 
-{% if imgargs_host is defined and imgargs_host %}
+{% if imgargs_host is defined and imgargs_host -%}
 set imgargs-host {{ imgargs_host }}
-{% endif %}
+{%- endif %}

templates/host-preseed.j2

@@ -11,53 +11,40 @@
 ### General definitions/overrides
 ###
 
-{% if debrelease is defined and debrelease %}
+{%- if debrelease is defined and debrelease %}
 # The Debian release to use (overrides the default)
-#debrelease="buster"
+debrelease="{{ debrelease }}"
-debrelease="{debrelease}"
 {% endif %}
 
-{% if debmirror is defined and debmirror %}
+{%- if debmirror is defined and debmirror %}
 # The Debian mirror to use (overrides the default)
-#debmirror="http://debian.mirror.rafal.ca/debian"
+debmirror="{{ debmirror }}"
-debmirror="{debmirror}"
+{%- endif %}
-{% endif %}
 
-{% if addpkglist is defined and addpkglist %}
+{%- if addpkglist is defined and addpkglist %}
 # Additional packages (comma-separated) to install in the base system
-#addpkglist="mypackage,otherpackage"
+addpkglist="{{ addpkglist }}"
-addpkglist="{addpkglist}"
+{%- endif %}
-{% endif %}
 
-{% if filesystem is defined and filesystem %}
+{%- if filesystem is defined and filesystem %}
 # Alternate filesystem for system volumes (/, /var/lib/ceph, /var/lib/zookeeper)
-#filesystem=ext4
+filesystem="{{ filesystem }}"
-filesystem="{filesystem}"
+{%- endif %}
-{% endif %}
 
-{% if skip_blockcheck is defined and skip_blockcheck %}
+{%- if skip_blockcheck is defined and skip_blockcheck %}
 # Skip block zeroing; only recommended for testing, slow, low-endurance, or known-zeroed block devices.
 skip_blockcheck="y"
-{% endif %}
+{%- endif %}
 
 ###
 ### Per-host definitions (required)
 ###
 
 # The hostname of the system (set per-run)
-#target_hostname="myhostname.domain.tld"
+target_hostname="{{ fqdn }}"
-target_hostname="{hostname}"
 
-# The target system disk (either a path or model to find; path overrides model if set)
+# The target system disk path
-# Install will fail if these devices aren't found.
+target_disk="{{ target_disk }}"
-{% if target_disk_path is defined and target_disk_path %}
-#target_disk_path="/dev/disk/by-path/pci-0000:03:00.0-scsi-0:1:0:0"  # Example: RAID device via explicit path
-target_disk_path="{target_disk_path}"
-{% end if %}
-{% if target_disk_model is defined and target_disk_model %}
-#target_disk_model="DELLBOSS VD"                                     # Example: Dell BOSS on R6515 via model name
-target_disk_model="{target_disk_model}"
-{% end if %}
 
 # SSH key method (usually tftp)
 target_keys_method="tftp"
@@ -80,5 +67,4 @@ target_deploy_user="deploy"
 #target_module_blacklist=( "hpwdt" )
 
 # Installer checkin URI (provided by pvcbootstrapd)
-#pvcbootstrapd_checkin_uri="http://10.199.199.254:9999/checkin/host"
+pvcbootstrapd_checkin_uri="{{ pvcbootstrapd_checkin_uri }}"
-pvcbootstrapd_checkin_uri="{pvcbootstrapd_checkin_uri}"

templates/install.sh

@@ -211,18 +211,63 @@ seed_config() {
     host_ipaddr=$( ip -br address show ${target_interface} | awk '{ print $3 }' | awk -F '/' '{ print $1 }' )
 
     # Handle the target disk
-    if [[ -n ${target_disk_path} ]]; then
+    case "${target_disk}" in
-        target_disk="$( realpath ${target_disk_path} )"
+        /dev/*)
-    else
+            # Get the real path of the block device (for /dev/disk/* symlink paths)
-        # Find the (first) disk with the given model
+            target_disk="$( realpath ${target_disk} )"
-        for disk in /dev/sd?; do
+        ;;
-            disk_model="$( fdisk -l ${disk} | grep 'Disk model:' | sed 's/Disk model: //g' )"
+        detect:*)
-            if [[ ${disk_model} == ${target_disk_model} ]]; then
+            # Read the detect string into separate variables
-                target_disk="${disk}"
+            # A detect string is formated thusly:
-                break
+            #   detect:<Controller-or-Vendor-Name>:<0-indexed-ID>:<Capacity-in-human-units>
-            fi
+            # For example:
-        done
+            #   detect:INTEL:1:800GB
-    fi
+            #   detect:DELLBOSS:0:240GB
+            #   detect:PERC H330 Mini:0:200GB
+            IFS=: read detect b_name b_id b_size <<<"${target_disk}"
+            # Get the lsscsi output (exclude NVMe)
+            lsscsi_data_all="$( lsscsi -s -N )"
+            # Get the available sizes, and match to within +/- 2%
+            lsscsi_sizes=( $( awk '{ print $NF }' <<<"${lsscsi_data_all}" | sort | uniq ) )
+            # For each size...
+            for size in ${lsscsi_sizes[@]}; do
+                # Get whether we match +2% and -2% sizes to handle human -> real deltas
+                # The break below is pretty safe. I can think of no two classes of disks
+                # where the difference is within 2% of each other. Even the common
+                # 120GB -> 128GB and 240GB -> 256GB size deltas are well outside of 2%,
+                # so this should be safe in all cases. 1% would be narrower but has more
+                # chance of mis-identifying due to rounding, while 3% gets into more
+                # contentious differences, so 2% seems like the best option.
+                # We use Python for this due to BASH's problematic handling of floating-
+                # point numbers.
+                is_match="$(
+                    python <<EOF
+from re import sub
+b_size = float(sub(r'\D','','${b_size}'))
+t_size = float(sub(r'\D','','${size}'))
+plustwopct = t_size * 1.02
+minustwopct = t_size * 0.98
+if b_size > minustwopct and b_size < plustwopct:
+    print("match")
+EOF
+                )"
+                # If we do, this size is our actual block size, not what was specified
+                if [[ -n ${is_match} ]]; then
+                    b_size=${size}
+                    break
+                fi
+            done
+            # Search for the b_name first
+            lsscsi_data_name="$( grep --color=none -Fiw "${b_name}" <<<"${lsscsi_data_all}" )"
+            # Search for the b_blocks second
+            lsscsi_data_name_size="$( grep --color=none -Fiw "${b_size}" <<<"${lsscsi_data_name}" )"
+            # Read the /dev/X results into an array
+            lsscsi_filtered=( $( awk '{ print $(NF-1) }' <<<"${lsscsi_data_name_size}" ) )
+            # Get the b_id-th entry
+            target_disk="${lsscsi_filtered[${b_id}]}"
+        ;;
+    esac
+
     if [[ ! -b ${target_disk} ]]; then
         echo "Invalid disk or disk not found!"
         exit 1

templates/installer.list.chroot

@@ -1 +1 @@
-live-task-standard live-tools live-boot live-boot-initramfs-tools linux-image-amd64 psmisc mdadm lvm2 parted gdisk dosfstools debootstrap grub-pc-bin grub-efi-amd64 sipcalc vim ca-certificates vlan tftp-hpa curl ipmitool
+live-task-standard live-tools live-boot live-boot-initramfs-tools linux-image-amd64 psmisc mdadm lvm2 parted gdisk dosfstools debootstrap grub-pc-bin grub-efi-amd64 sipcalc vim ca-certificates vlan tftp-hpa curl ipmitool lsscsi