Back system distro overlay with a per-instance scratch vhd

src/linux/init/main.cpp

@@ -121,9 +121,9 @@ std::optional<bool> g_EnableSocketLogging;
 
 int Chroot(const char* Target);
 
-void CreateSwap(unsigned int Lun);
+std::optional<std::string> FormatScratchDevice(unsigned int Lun);
 
-int CreateTempDirectory(const char* ParentPath, std::string& Path);
+void CreateSwap(unsigned int Lun);
 
 int DetachScsiDisk(unsigned int Lun);
 
@@ -170,7 +170,8 @@ void LaunchSystemDistro(
     const char* SharedMemoryRoot,
     const char* InstallPath,
     const char* UserProfile,
-    pid_t DistroInitPid);
+    pid_t DistroInitPid,
+    unsigned int ScratchLun);
 
 std::map<unsigned long, std::string> ListDiskPartitions(const std::string& DeviceName, std::optional<unsigned long> WaitForIndex = {});
 
@@ -295,87 +296,96 @@ Return Value:
     return Fd;
 }
 
-void CreateSwap(unsigned int Lun)
+std::optional<std::string> FormatScratchDevice(unsigned int Lun)
 
 /*++
 
 Routine Description:
 
-    This routine sets up a swap area on the specified SCSI device.
+    This routine formats the scratch vhd used to back the system distro overlay's
+    read/write layer.
+
+    N.B. This must run after the scratch device LUN is attached, but before the
+         overlay is created so the overlay can mount the device as its rw layer.
 
 Arguments:
 
-    Lun - Supplies the LUN number of the SCSI device.
+    Lun - Supplies the LUN of the scratch device, or UINT_MAX if none.
 
 Return Value:
 
-    None.
+    The scratch device path on success, std::nullopt on failure.
 
 --*/
 
+try
 {
+    if (Lun == UINT_MAX)
+    {
+        return std::nullopt;
+    }
+
+    const std::string DevicePath = GetLunDevicePath(Lun);
+    WaitForBlockDevice(DevicePath.c_str());
+
     //
-    // Create the swap file asynchronously using the mkswap and swapon utilities in the system distro.
+    // Format the scratch device with ext4.
     //
-    // N.B. This is done because creating the swap file can take some time and
-    //      the swap file does not need to be available immediately.
+    // N.B. This runs with the system distro as the root filesystem, so mkfs.ext4
+    //      and the scratch device node are directly reachable.
+    //
+    // N.B. The journal is omitted (the data is disposable) and the inode table is
+    //      lazily initialized to keep formatting off the boot critical path.
     //
 
-    UtilCreateChildProcess("CreateSwap", [Lun]() {
-        std::string DevicePath = GetLunDevicePath(Lun);
-
-        WaitForBlockDevice(DevicePath.c_str());
-
-        std::string CommandLine = std::format("/usr/sbin/mkswap '{}'", DevicePath);
-        THROW_LAST_ERROR_IF(UtilExecCommandLine(CommandLine.c_str(), nullptr) < 0);
+    const std::string CommandLine = std::format("/usr/sbin/mkfs.ext4 -q -m 0 -O ^has_journal -E lazy_itable_init=1 '{}'", DevicePath);
+    THROW_LAST_ERROR_IF(UtilExecCommandLine(CommandLine.c_str(), nullptr) < 0);
 
-        CommandLine = std::format("/usr/sbin/swapon '{}'", DevicePath);
-        UtilExecCommandLine(CommandLine.c_str(), nullptr);
-    });
+    return DevicePath;
+}
+catch (...)
+{
+    LOG_CAUGHT_EXCEPTION();
+    return std::nullopt;
 }
 
-int CreateTempDirectory(const char* ParentPath, std::string& Path)
+void CreateSwap(unsigned int Lun)
 
 /*++
 
 Routine Description:
 
-    This routine creates a unique directory under the specified parent path.
+    This routine sets up a swap area on the specified SCSI device.
 
 Arguments:
 
-    ParentPath - Supplies the path of the parent directory.
-
-    Path - Supplies a buffer to receive the path of the child directory that was
-        created.
+    Lun - Supplies the LUN number of the SCSI device.
 
 Return Value:
 
-    0 on success, -1 on failure.
+    None.
 
 --*/
 
 {
-    if (ParentPath)
-    {
-        Path = ParentPath;
-    }
-
     //
-    // Generate a random name for the directory.
+    // Create the swap file asynchronously using the mkswap and swapon utilities in the system distro.
     //
-    // N.B. mkdtemp requires a template string that ends in "XXXXXX".
+    // N.B. This is done because creating the swap file can take some time and
+    //      the swap file does not need to be available immediately.
     //
 
-    Path += "/wslXXXXXX";
+    UtilCreateChildProcess("CreateSwap", [Lun]() {
+        std::string DevicePath = GetLunDevicePath(Lun);
 
-    if (mkdtemp(Path.data()) == NULL)
-    {
-        LOG_ERROR("mkdtemp({}) failed {}", Path.c_str(), errno);
-        return -1;
-    }
+        WaitForBlockDevice(DevicePath.c_str());
 
-    return 0;
+        std::string CommandLine = std::format("/usr/sbin/mkswap '{}'", DevicePath);
+        THROW_LAST_ERROR_IF(UtilExecCommandLine(CommandLine.c_str(), nullptr) < 0);
+
+        CommandLine = std::format("/usr/sbin/swapon '{}'", DevicePath);
+        UtilExecCommandLine(CommandLine.c_str(), nullptr);
+    });
 }
 
 dev_t GetBlockDeviceNumber(const std::string& BlockDeviceName)
@@ -1477,7 +1487,7 @@ try
     size_t TargetPathLength = strlen(Target);
     auto AddTemporaryMount = [&](const char* Name, const char* Source, unsigned long MountFlags) {
         std::string Path;
-        THROW_LAST_ERROR_IF(CreateTempDirectory(Target, Path) < 0);
+        THROW_LAST_ERROR_IF(UtilCreateTempDirectory(Target, Path) < 0);
         THROW_LAST_ERROR_IF(mount(Source, Path.c_str(), nullptr, MountFlags, nullptr) < 0);
         AddEnvironmentVariable(Name, Path.substr(TargetPathLength).data());
     };
@@ -1665,7 +1675,8 @@ void LaunchSystemDistro(
     const char* SharedMemoryRoot,
     const char* InstallPath,
     const char* UserProfile,
-    pid_t DistroInitPid)
+    pid_t DistroInitPid,
+    unsigned int ScratchLun)
 
 /*++
 
@@ -1702,6 +1713,10 @@ Routine Description:
 
     DistroInitPid - Supplies the pid of the user distribution's init process.
 
+    ScratchLun - Supplies the SCSI LUN of the scratch device used to back the
+        overlay read/write layer, or UINT_MAX if none. When unavailable, the
+        overlay falls back to a tmpfs read/write layer.
+
 Return Value:
 
     None. This method does not return.
@@ -1711,10 +1726,26 @@ Return Value:
 try
 {
     //
-    // Create a writable layer on top of the read-only vhd.
+    // Format the scratch device (backed by a vhd) used to back the overlay
+    // read/write layer so it is disk-backed instead of consuming guest memory.
+    // On failure, the overlay transparently falls back to tmpfs.
     //
 
-    THROW_LAST_ERROR_IF(UtilMountOverlayFs(Target, SYSTEM_DISTRO_VHD_PATH) < 0);
+    const std::optional<std::string> ScratchDevice = FormatScratchDevice(ScratchLun);
+
+    //
+    // Create a writable layer on top of the read-only vhd. If the scratch-backed
+    // overlay fails to mount (e.g. the backing vhd is full or the rw layer setup
+    // fails), fall back to a tmpfs-backed overlay so the system distro can still
+    // launch.
+    //
+
+    if (UtilMountOverlayFs(Target, SYSTEM_DISTRO_VHD_PATH, 0, {}, ScratchDevice) < 0)
+    {
+        THROW_LAST_ERROR_IF(!ScratchDevice.has_value());
+        LOG_ERROR("scratch-backed overlay mount failed, falling back to tmpfs");
+        THROW_LAST_ERROR_IF(UtilMountOverlayFs(Target, SYSTEM_DISTRO_VHD_PATH, 0, {}, std::nullopt) < 0);
+    }
 
     //
     // Launch the init daemon, this method does not return.
@@ -1852,7 +1883,7 @@ try
     std::string MountPoint;
     if (Flags & LxMiniInitMessageFlagCreateOverlayFs)
     {
-        if (CreateTempDirectory(Target, MountPoint) < 0)
+        if (UtilCreateTempDirectory(Target, MountPoint) < 0)
         {
             return -1;
         }
@@ -1940,7 +1971,7 @@ int MountSystemDistro(LX_MINI_INIT_MOUNT_DEVICE_TYPE DeviceType, unsigned int De
 Routine Description:
 
     This routine mounts the system distro as read-only, creates a writable
-    tmpfs layer using overlayfs, and chroots to the mount point.
+    overlayfs layer, and chroots to the mount point.
 
 Arguments:
 
@@ -2301,7 +2332,8 @@ void ProcessLaunchInitMessage(
                         wsl::shared::string::FromSpan(Buffer, Message->SharedMemoryRootOffset),
                         wsl::shared::string::FromSpan(Buffer, Message->InstallPathOffset),
                         wsl::shared::string::FromSpan(Buffer, Message->UserProfileOffset),
-                        ChildPid);
+                        ChildPid,
+                        Message->ScratchLun);
                 }
             }