feat: reduce memory consumption of Scout

.github/workflows/build-boot-artifacts.yml

@@ -415,14 +415,14 @@ jobs:
             if [[ "${{ inputs.arch }}" == "x86_64" ]]; then
               EXPECTED_FILES=(
                 "pxe/static/blobs/internal/x86_64/scout.efi"
-                "pxe/static/blobs/internal/x86_64/scout.cpio.zst"
+                "pxe/static/blobs/internal/x86_64/scout.squashfs"
                 "pxe/static/blobs/internal/x86_64/qcow-imager.efi"
                 # target/debs/* checked separately
               )
             else
               EXPECTED_FILES=(
                 "pxe/static/blobs/internal/aarch64/scout.efi"
-                "pxe/static/blobs/internal/aarch64/scout.cpio.zst"
+                "pxe/static/blobs/internal/aarch64/scout.squashfs"
                 "pxe/static/blobs/internal/aarch64/qcow-imager.efi"
                 # pxe/mkosi.profiles/scout-x86_64/mkosi.extra/build-output/* checked separately
                 # target/debs/* checked separately
@@ -472,13 +472,13 @@ jobs:
         run: |
           set -euo pipefail
 
-          SCOUT_IMAGE="pxe/static/blobs/internal/${{ inputs.arch }}/scout.cpio.zst"
+          SCOUT_IMAGE="pxe/static/blobs/internal/${{ inputs.arch }}/scout.squashfs"
           if [ ! -f "${SCOUT_IMAGE}" ]; then
             echo "ERROR: Missing scout image artifact: ${SCOUT_IMAGE}"
             exit 1
           fi
 
-          if zstd -dc "${SCOUT_IMAGE}" | cpio -it --quiet 2>/dev/null | grep -Eq '(^|\\./)opt/forge/bin/mnv_cli$'; then
+          if unsquashfs -l "${SCOUT_IMAGE}" 2>/dev/null | grep -Eq '^squashfs-root/opt/forge/bin/mnv_cli$'; then
             echo "OK: /opt/forge/bin/mnv_cli is present in ${SCOUT_IMAGE}"
           else
             echo "ERROR: /opt/forge/bin/mnv_cli missing from ${SCOUT_IMAGE}"
@@ -502,9 +502,9 @@ jobs:
             fi
           else
             if [[ "${{ inputs.arch }}" == "x86_64" ]]; then
-              ARTIFACTS='["pxe/static/blobs/internal/x86_64/scout.efi", "pxe/static/blobs/internal/x86_64/scout.cpio.zst", "pxe/static/blobs/internal/x86_64/qcow-imager.efi", "target/debs/*"]'
+              ARTIFACTS='["pxe/static/blobs/internal/x86_64/scout.efi", "pxe/static/blobs/internal/x86_64/scout.squashfs", "pxe/static/blobs/internal/x86_64/qcow-imager.efi", "target/debs/*"]'
             else
-              ARTIFACTS='["pxe/static/blobs/internal/aarch64/scout.efi", "pxe/static/blobs/internal/aarch64/scout.cpio.zst", "pxe/static/blobs/internal/aarch64/qcow-imager.efi", "pxe/mkosi.profiles/scout-x86_64/mkosi.extra/build-output/*", "target/debs/*"]'
+              ARTIFACTS='["pxe/static/blobs/internal/aarch64/scout.efi", "pxe/static/blobs/internal/aarch64/scout.squashfs", "pxe/static/blobs/internal/aarch64/qcow-imager.efi", "pxe/mkosi.profiles/scout-x86_64/mkosi.extra/build-output/*", "target/debs/*"]'
             fi
           fi
 

pxe/Makefile.toml

@@ -555,8 +555,8 @@ dependencies = ["mkdir-static"]
 category = "Ephemeral Image"
 command = "cp"
 args = [
-  "${MKOSI_BUILD_TMP}/scout.cpio.zst",
-  "static/blobs/internal/x86_64/scout.cpio.zst",
+  "${MKOSI_BUILD_TMP}/scout.squashfs",
+  "static/blobs/internal/x86_64/scout.squashfs",
 ]
 dependencies = ["mkdir-static"]
 
@@ -573,8 +573,8 @@ dependencies = ["mkdir-static"]
 category = "Ephemeral Image"
 command = "cp"
 args = [
-  "${MKOSI_BUILD_TMP}/scout.cpio.zst",
-  "static/blobs/internal/aarch64/scout.cpio.zst",
+  "${MKOSI_BUILD_TMP}/scout.squashfs",
+  "static/blobs/internal/aarch64/scout.squashfs",
 ]
 dependencies = ["mkdir-static"]
 

pxe/common_files/check-scout-updates.sh

@@ -34,7 +34,7 @@ fi
 pxe_uri=$(sed 's/ /\n/g' /proc/cmdline | grep '^pxe_uri=' | cut -d'=' -f2)
 static_pxe_base_url=${pxe_uri:-http://carbide-static-pxe.forge}
 arch=$(uname -m)
-scout_url="${static_pxe_base_url}/public/blobs/internal/${arch}/scout.cpio.zst"
+scout_url="${static_pxe_base_url}/public/blobs/internal/${arch}/scout.squashfs"
 www_last_modified_str=$(curl -sf --head ${scout_url} 2>/dev/null | sed 's/\r//g' | grep Last-Modified)
 if (( $? != 0 ))
 then

pxe/common_files/scout-loader-rclocal

@@ -18,13 +18,24 @@ set -o pipefail
 
 PCR=16
 OUTCONSOLE=""
-TMPFILE="/tmp/newrootfs"
 ROOTFS_INFO_FILE="/run/nextroot/rootfs_info.txt"
+ROOTFS_INFO_TMP="/tmp/rootfs_info.txt"
+ROOTFS_DIGEST_FILE="/tmp/rootfs.digest"
+ROOTFS_DIGEST_FIFO="/tmp/rootfs.digest.fifo"
+ROOTFS_IMAGE_FILE="/run/scout-rootfs.squashfs"
+SQUASHFS_LOWER="/run/scout-rootfs.lower"
+OVERLAY_ROOT="/run/scout-rootfs.overlay"
+OVERLAY_UPPER="${OVERLAY_ROOT}/upper"
+OVERLAY_WORK="${OVERLAY_ROOT}/work"
+CURL_ERR="/tmp/rootfs.curl.err"
+ZSTD_ERR="/tmp/rootfs.zstd.err"
+CPIO_ERR="/tmp/rootfs.cpio.err"
+MOUNT_ERR="/tmp/rootfs.mount.err"
 
 send_msg() {
 	for dest in ${OUTCONSOLE}
 	do
-		echo $1 >${dest} 2>/dev/null
+		echo "$1" >${dest} 2>/dev/null
 	done
 }
 
@@ -34,7 +45,7 @@ then
     newrootfsurl=$(cat /proc/cmdline | sed -e 's/.*newrootfs=//' -e 's/ .*//')
 else
     arch=$(uname -m)
-    newrootfsurl="http://carbide-static-pxe.forge/public/blobs/internal/${arch}/scout.cpio.zst"
+    newrootfsurl="http://carbide-static-pxe.forge/public/blobs/internal/${arch}/scout.squashfs"
 fi
 
 # Respect the console info from the kernel command line
@@ -51,56 +62,178 @@ do
 		fi
 	fi
 done
-send_msg "LOADER: Starting V1.0"
+send_msg "LOADER: Starting V1.2"
+
+detect_pcr_hash() {
+    PCR_HASH_ALG=""
+    PCR_HASH_CMD=""
+    if [ -e "/sys/class/tpm/tpm0/pcr-sha256/${PCR}" ]
+    then
+      PCR_HASH_ALG="sha256"
+      PCR_HASH_CMD="sha256sum"
+    elif [ -e "/sys/class/tpm/tpm0/pcr-sha384/${PCR}" ]
+    then
+      PCR_HASH_ALG="sha384"
+      PCR_HASH_CMD="sha384sum"
+    fi
+}
+
+measure_rootfs() {
+    if [ -n "${PCR_HASH_ALG}" ]
+    then
+      send_msg "LOADER: Measuring to PCR ${PCR_HASH_ALG}/${PCR}"
+      tpm2_pcrextend "${PCR}:${PCR_HASH_ALG}=$(<"${ROOTFS_DIGEST_FILE}")"
+    else
+      send_msg "LOADER: Unable to find PCR ${PCR}, is TPM installed and configured correctly?"
+    fi
+}
+
+prepare_cpio_rootfs() {
+    local rootfsurl="$1"
+
+    mkdir -p /nextroot /run/nextroot
+    if ! mountpoint -q /run/nextroot
+    then
+        mount -o bind,suid /nextroot /run/nextroot || return 1
+    fi
+    cd /run/nextroot || return 1
+
+    if [ -n "${PCR_HASH_CMD}" ]
+    then
+        rm -f "${ROOTFS_DIGEST_FILE}" "${ROOTFS_DIGEST_FIFO}"
+        mkfifo "${ROOTFS_DIGEST_FIFO}"
+        (
+            "${PCR_HASH_CMD}" < "${ROOTFS_DIGEST_FIFO}" | awk '{print $1}' > "${ROOTFS_DIGEST_FILE}"
+        ) &
+        HASH_PID=$!
+
+        curl -sSf -o - "${rootfsurl}" 2> "${CURL_ERR}" | tee "${ROOTFS_DIGEST_FIFO}" | zstd -d 2> "${ZSTD_ERR}" | cpio -i 2> "${CPIO_ERR}"
+        PIPE_RET=$?
+        wait "${HASH_PID}"
+        HASH_RET=$?
+        rm -f "${ROOTFS_DIGEST_FIFO}"
+
+        if (( ${PIPE_RET} != 0 ))
+        then
+            return "${PIPE_RET}"
+        fi
+        return "${HASH_RET}"
+    fi
+
+    curl -sSf -o - "${rootfsurl}" 2> "${CURL_ERR}" | zstd -d 2> "${ZSTD_ERR}" | cpio -i 2> "${CPIO_ERR}"
+}
+
+cleanup_squashfs_mounts() {
+    umount /run/nextroot 2>/dev/null || true
+    umount "${SQUASHFS_LOWER}" 2>/dev/null || true
+    umount "${OVERLAY_ROOT}" 2>/dev/null || true
+    rm -rf "${SQUASHFS_LOWER}" "${OVERLAY_ROOT}"
+    rm -f "${ROOTFS_IMAGE_FILE}"
+}
+
+ensure_loop_devices() {
+    [ -e /dev/loop-control ] || mknod /dev/loop-control c 10 237 2>/dev/null || true
+    for devnum in 0 1 2 3
+    do
+        [ -e "/dev/loop${devnum}" ] || mknod "/dev/loop${devnum}" b 7 "${devnum}" 2>/dev/null || true
+    done
+}
+
+prepare_squashfs_rootfs() {
+    local rootfsurl="$1"
+
+    cleanup_squashfs_mounts
+    rm -f "${ROOTFS_DIGEST_FILE}"
+
+    curl -sSf -o "${ROOTFS_IMAGE_FILE}" "${rootfsurl}" 2> "${CURL_ERR}"
+    if (( $? != 0 ))
+    then
+        return 1
+    fi
+
+    if [ -n "${PCR_HASH_CMD}" ]
+    then
+        "${PCR_HASH_CMD}" "${ROOTFS_IMAGE_FILE}" | awk '{print $1}' > "${ROOTFS_DIGEST_FILE}"
+    fi
+
+    modprobe loop 2>/dev/null || true
+    modprobe squashfs 2>/dev/null || true
+    modprobe overlay 2>/dev/null || true
+    ensure_loop_devices
+
+    mkdir -p "${SQUASHFS_LOWER}" "${OVERLAY_ROOT}" /run/nextroot
+    mount -t squashfs -o loop,ro "${ROOTFS_IMAGE_FILE}" "${SQUASHFS_LOWER}" 2> "${MOUNT_ERR}"
+    if (( $? != 0 ))
+    then
+        return 1
+    fi
+
+    mount -t tmpfs -o mode=0755 tmpfs "${OVERLAY_ROOT}" 2> "${MOUNT_ERR}"
+    if (( $? != 0 ))
+    then
+        return 1
+    fi
+
+    mkdir -p "${OVERLAY_UPPER}" "${OVERLAY_WORK}"
+    mount -t overlay overlay \
+        -o "lowerdir=${SQUASHFS_LOWER},upperdir=${OVERLAY_UPPER},workdir=${OVERLAY_WORK}" \
+        /run/nextroot 2> "${MOUNT_ERR}"
+}
+
+prepare_rootfs() {
+    local rootfsurl="$1"
+
+    : > "${CURL_ERR}"
+    : > "${ZSTD_ERR}"
+    : > "${CPIO_ERR}"
+    : > "${MOUNT_ERR}"
+
+    case "${rootfsurl}" in
+        *.cpio|*.cpio\?*|*.cpio.zst|*.cpio.zst\?*|*.cpio.zstd|*.cpio.zstd\?*)
+            prepare_cpio_rootfs "${rootfsurl}"
+            ;;
+        *)
+            prepare_squashfs_rootfs "${rootfsurl}"
+            ;;
+    esac
+}
 
 # We will use "none" as a filename to stay in this image for debugging
 if [ "${newrootfsurl}" != "none" ]
 then
-    # To allow things such as sudo to work once we've switched images, we need to
-    # make sure that the new root is mounted with suid enabled.
-    mkdir -p /nextroot /run/nextroot
-    mount -o bind,suid /nextroot /run/nextroot
-    cd /run/nextroot
+    detect_pcr_hash
 
     DONE=0
     send_msg "LOADER: Downloading ${newrootfsurl}"
     while (( ${DONE} != 1 ))
     do
       # We store the HTTP HEAD info for the script which will check for updates later.
-      curl -sSf --head ${newrootfsurl} 2> ./curl.err | sed 's/\r//g' > ${ROOTFS_INFO_FILE}
+      curl -sSf --head "${newrootfsurl}" 2> "${CURL_ERR}" | sed 's/\r//g' > "${ROOTFS_INFO_TMP}"
       if (( $? == 0 ))
       then
-        curl -sSf -o - ${newrootfsurl} 2> ./curl.err | tee ${TMPFILE} | zstd -d 2> ./zstd.err | cpio -i 2> ./cpio.err
+        prepare_rootfs "${newrootfsurl}"
         if (( $? == 0 ))
         then
-          if [ -e "/sys/class/tpm/tpm0/pcr-sha256/${PCR}" ]
-          then
-            send_msg "LOADER: Measuring to PCR sha256/${PCR}"
-            tpm2_pcrextend "${PCR}:sha256=$(sha256sum ${TMPFILE} | awk '{print $1}')"
-          elif [ -e "/sys/class/tpm/tpm0/pcr-sha384/${PCR}" ]
-          then
-            send_msg "LOADER: Measuring to PCR sha384/${PCR}"
-            tpm2_pcrextend "${PCR}:sha384=$(sha384sum ${TMPFILE} | awk '{print $1}')"
-          else
-            send_msg "LOADER: Unable to find PCR ${PCR}, is TPM installed and configured correctly?"
-          fi
+          cp "${ROOTFS_INFO_TMP}" "${ROOTFS_INFO_FILE}"
+          measure_rootfs
           DONE=1
         else
           send_msg "LOADER: Download failed:"
-          send_msg "LOADER: curl error: $(<./curl.err)"
-          send_msg "LOADER: zstd error: $(<./zstd.err)"
-          send_msg "LOADER: cpio error: $(<./cpio.err)"
-	  sleep 1
+          send_msg "LOADER: curl error: $(<"${CURL_ERR}")"
+          send_msg "LOADER: zstd error: $(<"${ZSTD_ERR}")"
+          send_msg "LOADER: cpio error: $(<"${CPIO_ERR}")"
+          send_msg "LOADER: mount error: $(<"${MOUNT_ERR}")"
+          sleep 1
         fi
       else
           send_msg "LOADER: Failed to gather root filesystem information:"
-          send_msg "LOADER: curl error: $(<./curl.err)"
-	  sleep 1
+          send_msg "LOADER: curl error: $(<"${CURL_ERR}")"
+          sleep 1
       fi
     done
 
     send_msg "LOADER: Cleaning up & switching"
-    rm -f ${TMPFILE}
+    rm -f "${ROOTFS_DIGEST_FILE}" "${ROOTFS_DIGEST_FIFO}" "${ROOTFS_INFO_TMP}"
     # We need this so that subsequent reboots don't call soft-reboot and put us in a loop
     echo "export SYSTEMCTL_SKIP_AUTO_SOFT_REBOOT=1" >>/run/nextroot/etc/environment
     systemctl soft-reboot

pxe/mkosi.profiles/scout-loader-aarch64/mkosi.conf

@@ -24,6 +24,7 @@ Packages=
          dbus
          file
          isc-dhcp-client
+         kmod
          linux-nvidia-64k-hwe-24.04
          net-tools
          netbase

pxe/mkosi.profiles/scout-loader-x86_64/mkosi.conf

@@ -22,6 +22,7 @@ Packages=
          curl
          dbus
          isc-dhcp-client
+         kmod
          linux-image-6.8.0-111-generic
          net-tools
          netbase

pxe/mkosi.profiles/scout-oss-aarch64/mkosi.conf

@@ -1,7 +1,7 @@
 [Output]
 Output=scout
 CompressOutput=zstd
-CompressLevel=19
+CompressLevel=6
 Format=cpio
 
 [Distribution]
@@ -17,6 +17,8 @@ ToolsTreeRelease=noble
 Environment=MKOSI_CHROOT_SUPPRESS_CHOWN=1
 
 [Content]
+WithDocs=no
+Bootable=no
 Packages=
      apt-utils
      bind9-dnsutils

pxe/mkosi.profiles/scout-oss-aarch64/mkosi.finalize.chroot

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+APT_SANDBOX_OPTS="-o APT::Sandbox::User=root"
+export DEBIAN_FRONTEND=noninteractive
+
+# Clean up unneeded packages and clean up apt
+apt-get ${APT_SANDBOX_OPTS} -y --purge autoremove
+apt-get ${APT_SANDBOX_OPTS} -y clean
+rm -rf /var/lib/apt/lists/*
+mkdir -p /var/lib/apt/lists/partial
+rm -rf /var/cache/apt/*.bin
+
+# Remove docs, locales etc
+rm -rf /usr/share/doc/*
+rm -rf /usr/share/locale/*
+rm -rf /build-output
+
+# Remove systemd journal
+systemctl stop systemd-journald.service systemd-journald-audit.socket systemd-journald.socket systemd-journald-dev-log.socket
+rm -rf /var/log/journal/*
+
+# Remove unnecessary kernel files
+rm -f /boot/System.map-* /boot/config-* /boot/vmlinu* /lib/modules/*/vmlinuz /boot/EFI/Linux/*
+
+# Make sure /tmp is empty
+rm -rf /tmp/*

pxe/mkosi.profiles/scout-oss-aarch64/mkosi.postinst.chroot

@@ -110,4 +110,3 @@ do
     exit 1
   fi
 done
-