diff --git a/README.md b/README.md index 073fd0c..4f5dd1c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ App Sandbox is a virtual machine app for Windows and macOS that's focused on per Windows features: - Works on Windows 11 Home or Pro, without Hyper-V -- Windows 11 or Ubuntu 26.04 LTS VM Support +- Windows 11 or Ubuntu 24.04 / 26.04 LTS VM Support - Zero touch install - Copy and Paste - 2 Channel Audio diff --git a/src/app_win/ui.c b/src/app_win/ui.c index 6058f3a..5a3198f 100644 --- a/src/app_win/ui.c +++ b/src/app_win/ui.c @@ -131,7 +131,7 @@ static void build_host_info_json(JsonBuilder *jb) MEMORYSTATUSEX ms; DWORD host_cores, host_ram_mb; DWORD vm_cores = 0, vm_ram_mb = 0, vm_hdd_gb = 0; - wchar_t base_dir[MAX_PATH]; + wchar_t base_dir[MAX_PATH], vm_dir[MAX_PATH]; ULARGE_INTEGER free_bytes; DWORD free_gb = 0; int i, count = asb_vm_count(); @@ -148,9 +148,16 @@ static void build_host_info_json(JsonBuilder *jb) if (v) vm_hdd_gb += v->hdd_gb; } + /* Report free space on the volume that actually backs the VM data + directory (%ProgramData%\AppSandbox), not the %ProgramData% root. + Querying the leaf directory lets GetDiskFreeSpaceEx follow a junction / + mount point when the user has redirected VM storage to another drive. + Fall back to the root when the directory does not exist yet. */ if (!GetEnvironmentVariableW(L"ProgramData", base_dir, MAX_PATH)) wcscpy_s(base_dir, MAX_PATH, L"C:\\ProgramData"); - if (GetDiskFreeSpaceExW(base_dir, &free_bytes, NULL, NULL)) + swprintf_s(vm_dir, MAX_PATH, L"%s\\AppSandbox", base_dir); + if (GetDiskFreeSpaceExW(vm_dir, &free_bytes, NULL, NULL) || + GetDiskFreeSpaceExW(base_dir, &free_bytes, NULL, NULL)) free_gb = (DWORD)(free_bytes.QuadPart / (1024ULL * 1024 * 1024)); jb_int(jb, L"hostCores", (int)host_cores); @@ -207,6 +214,7 @@ static void build_vm_json(JsonBuilder *jb, int i) jb_bool(jb, L"buildingVhdx", v->building_vhdx); jb_bool(jb, L"vhdxStaging", v->vhdx_staging); jb_int(jb, L"vhdxProgress", v->vhdx_progress); + jb_string(jb, L"vhdxStep", v->vhdx_step); jb_bool(jb, L"installComplete", v->install_complete); jb_bool(jb, L"sshEnabled", v->ssh_enabled); jb_int(jb, L"sshPort", (int)v->ssh_port); diff --git a/src/backend_win/asb_core.c b/src/backend_win/asb_core.c index dc7e0f5..377fdec 100644 --- a/src/backend_win/asb_core.c +++ b/src/backend_win/asb_core.c @@ -733,6 +733,123 @@ static void remove_dir_recursive(const wchar_t *dir) RemoveDirectoryW(dir); } +/* ---- Utility: recursive directory copy ---- + * Returns the number of files copied, or -1 on the first failure. */ +static int copy_dir_recursive(const wchar_t *src, const wchar_t *dst) +{ + wchar_t pattern[MAX_PATH], s[MAX_PATH], d[MAX_PATH]; + WIN32_FIND_DATAW fd; + HANDLE h; + int n = 0; + + if (!CreateDirectoryW(dst, NULL) && GetLastError() != ERROR_ALREADY_EXISTS) + return -1; + swprintf_s(pattern, MAX_PATH, L"%s\\*", src); + h = FindFirstFileW(pattern, &fd); + if (h == INVALID_HANDLE_VALUE) return -1; + do { + if (fd.cFileName[0] == L'.' && (fd.cFileName[1] == L'\0' || + (fd.cFileName[1] == L'.' && fd.cFileName[2] == L'\0'))) + continue; + swprintf_s(s, MAX_PATH, L"%s\\%s", src, fd.cFileName); + swprintf_s(d, MAX_PATH, L"%s\\%s", dst, fd.cFileName); + if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + int r = copy_dir_recursive(s, d); + if (r < 0) { FindClose(h); return -1; } + n += r; + } else { + if (!CopyFileW(s, d, FALSE)) { FindClose(h); return -1; } + n++; + } + } while (FindNextFileW(h, &fd)); + FindClose(h); + return n; +} + +/* ---- Prefetch cache ---- + * + * %ProgramData%\AppSandbox\cache\prefetch\\ keeps the output of one + * iso-patch --prefetch-* run. Every Linux create used to re-download the + * repo tarball (~80 MB), the apt build-deps closure (~180 .debs) and the + * wsl-deps NuGet package from scratch, one file at a time - that is the + * whole "Building Disk 0%" wait, and the same bytes every time. A hit + * copies the cached tree into the staging dir instead. + * + * Entries carry a .cache-ok marker written after a complete download and + * expire by age (repo source daily, the rest weekly/monthly) so a moved + * branch or a refreshed -updates pocket is picked up without any manual + * step. APPSANDBOX_NO_PREFETCH_CACHE=1 bypasses the cache; deleting the + * cache dir does too. */ +static BOOL prefetch_cache_path(const wchar_t *key, wchar_t *out, size_t cap) +{ + wchar_t base[MAX_PATH]; + if (GetEnvironmentVariableW(L"APPSANDBOX_NO_PREFETCH_CACHE", NULL, 0) > 0) + return FALSE; + if (!GetEnvironmentVariableW(L"ProgramData", base, MAX_PATH)) + wcscpy_s(base, MAX_PATH, L"C:\\ProgramData"); + swprintf_s(out, cap, L"%s\\AppSandbox\\cache", base); + CreateDirectoryW(out, NULL); + swprintf_s(out, cap, L"%s\\AppSandbox\\cache\\prefetch", base); + CreateDirectoryW(out, NULL); + swprintf_s(out, cap, L"%s\\AppSandbox\\cache\\prefetch\\%s", base, key); + return TRUE; +} + +/* Age of the entry's .cache-ok marker in hours, or -1 if there is none. */ +static double prefetch_cache_age_hours(const wchar_t *dir) +{ + wchar_t marker[MAX_PATH]; + WIN32_FILE_ATTRIBUTE_DATA fad; + FILETIME now; + ULARGE_INTEGER a, b; + + swprintf_s(marker, MAX_PATH, L"%s\\.cache-ok", dir); + if (!GetFileAttributesExW(marker, GetFileExInfoStandard, &fad)) return -1.0; + GetSystemTimeAsFileTime(&now); + a.LowPart = fad.ftLastWriteTime.dwLowDateTime; a.HighPart = fad.ftLastWriteTime.dwHighDateTime; + b.LowPart = now.dwLowDateTime; b.HighPart = now.dwHighDateTime; + if (b.QuadPart <= a.QuadPart) return 0.0; + return (double)(b.QuadPart - a.QuadPart) / 36000000000.0; /* 100 ns -> h */ +} + +/* Satisfy a prefetch from the cache: copies the entry into dst. */ +static BOOL prefetch_cache_restore(const wchar_t *key, int max_age_hours, + const wchar_t *dst, const wchar_t *what) +{ + wchar_t dir[MAX_PATH], marker[MAX_PATH]; + double age; + int n; + + if (!prefetch_cache_path(key, dir, MAX_PATH)) return FALSE; + age = prefetch_cache_age_hours(dir); + if (age < 0.0 || age > (double)max_age_hours) return FALSE; + n = copy_dir_recursive(dir, dst); + if (n < 0) { + asb_log(L"%s: cached copy of %s unusable - downloading again", what, key); + remove_dir_recursive(dir); + return FALSE; + } + /* The marker is cache bookkeeping, not something to stage into the guest. */ + swprintf_s(marker, MAX_PATH, L"%s\\.cache-ok", dst); + DeleteFileW(marker); + asb_log(L"%s: reused cached download (%d file(s), %.1f h old)", what, n, age); + return TRUE; +} + +/* After a complete prefetch: snapshot src into the cache entry. */ +static void prefetch_cache_store(const wchar_t *key, const wchar_t *src) +{ + wchar_t dir[MAX_PATH], marker[MAX_PATH]; + HANDLE h; + + if (!prefetch_cache_path(key, dir, MAX_PATH)) return; + remove_dir_recursive(dir); + if (copy_dir_recursive(src, dir) < 0) { remove_dir_recursive(dir); return; } + swprintf_s(marker, MAX_PATH, L"%s\\.cache-ok", dir); + h = CreateFileW(marker, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (h != INVALID_HANDLE_VALUE) CloseHandle(h); +} + /* ---- HCS state callback (called from HCS worker thread) ---- */ static void asb_hcs_state_changed(VmInstance *instance, DWORD event) @@ -1807,12 +1924,46 @@ static int generate_vhdx_manifest_ubuntu(const wchar_t *manifest_path, return n; } +/* Read the kernel release ("6.17.0-14-generic") out of an x86 bzImage's + * setup header: "HdrS" at 0x202, and the u16 at 0x20E is the offset (less + * 0x200) of a NUL-terminated " () # ..." string. + * Returns FALSE for anything that is not a bzImage (e.g. an arm64 Image), + * in which case the caller falls back to scanning pool/. */ +static BOOL read_bzimage_release(const wchar_t *path, wchar_t *out, size_t cap) +{ + unsigned char *hdr = (unsigned char *)malloc(0x10000); + HANDLE h; + DWORD got = 0; + BOOL ok = FALSE; + size_t n = 0; + unsigned off; + + if (!hdr) return FALSE; + h = CreateFileW(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL); + if (h != INVALID_HANDLE_VALUE) { + ok = ReadFile(h, hdr, 0x10000, &got, NULL); + CloseHandle(h); + } + if (!ok || got < 0x210 || memcmp(hdr + 0x202, "HdrS", 4) != 0) { free(hdr); return FALSE; } + off = (unsigned)hdr[0x20E] | ((unsigned)hdr[0x20F] << 8); + if (off == 0) { free(hdr); return FALSE; } + off += 0x200; + while (off + n < got && hdr[off + n] > ' ' && n + 1 < cap) { + out[n] = (wchar_t)hdr[off + n]; + n++; + } + out[n] = 0; + free(hdr); + return n > 0; +} + /* ---- Detect the Ubuntu release codename + kernel version from an ISO. * * Mounts the ISO read-only via VirtDisk, reads: * - the only subdir under :\dists\ -> codename (e.g. "resolute") - * - :\pool\main\l\linux\linux-image-*-generic_*.deb - * -> kernel version (e.g. "7.0.0-14-generic") parsed from filename + * - :\casper\vmlinuz -> kernel version (e.g. + * "7.0.0-14-generic") from the bzImage header; falls back to the + * linux-headers-*-generic_*.deb filename under :\pool\main\l\linux\ * * Returns 0 on success and fills the two output buffers; non-zero on * any error (caller logs + skips the build-deps prefetch). @@ -1930,7 +2081,24 @@ static int detect_iso_kernel(const wchar_t *iso_path, } } - /* 2. Kernel version from linux-headers-_*_amd64.deb + /* 2a. Preferred: the kernel that will actually boot. iso-patch stages + * whatever /boot/vmlinuz-* it finds (minimal.squashfs on 26.04, the + * live overlay on 24.04) and on Ubuntu ISOs that is the same build + * as casper/vmlinuz. Reading the release out of the bzImage header + * keeps the prefetched linux-headers- in step with the guest's + * uname -r. (On 24.04 pool/main/l/linux/ holds the GA 6.8 headers + * while the ISO boots the HWE 6.17 kernel, so the pool scan below + * would pick the wrong one.) */ + { + wchar_t vmlinuz[MAX_PATH]; + swprintf_s(vmlinuz, MAX_PATH, L"%c:\\casper\\vmlinuz", iso_drive); + if (read_bzimage_release(vmlinuz, kver_out, kver_cap)) + asb_log(L"detect_iso_kernel: casper/vmlinuz is %s", kver_out); + else + kver_out[0] = 0; + } + + /* 2b. Fallback: kernel version from linux-headers-_*_amd64.deb * in pool/main/l/linux/. We use linux-headers- not linux-image- * because the image .deb lives under pool/main/l/linux-signed/ * on signed-kernel Ubuntu releases, but the headers (which we @@ -1942,7 +2110,7 @@ static int detect_iso_kernel(const wchar_t *iso_path, * * FindFirstFileW doesn't reliably handle multi-`*` patterns on * ISO 9660; glob with single wildcard and filter for "-generic_". */ - { + if (!kver_out[0]) { wchar_t spec[MAX_PATH]; swprintf_s(spec, MAX_PATH, L"%c:\\pool\\main\\l\\linux\\linux-headers-*.deb", iso_drive); @@ -1993,7 +2161,57 @@ static int detect_iso_kernel(const wchar_t *iso_path, * * args: extra argv tail (no quotes — caller is responsible for safe paths). * Returns 0 on success; -1 on any failure. Logs progress to asb_log. */ -static int spawn_iso_patch_prefetch(const wchar_t *args) +/* Mirror for the apt build-deps prefetch. APPSANDBOX_APT_MIRROR wins; + otherwise Ubuntu's per-country alias (.archive.ubuntu.com, which + CNAMEs to a mirror in that country) derived from the Windows region - + the same choice the Ubuntu installer makes. iso-patch falls back to the + main archive if the alias does not resolve or lacks the release, so a + bad guess costs one failed request, not the build. Returns FALSE when + there is nothing better than the default. */ +static BOOL choose_apt_mirror(wchar_t *out, size_t cap, wchar_t *why, size_t why_cap) +{ + wchar_t cc[8]; + if (GetEnvironmentVariableW(L"APPSANDBOX_APT_MIRROR", out, (DWORD)cap) > 0) { + wcscpy_s(why, why_cap, L"APPSANDBOX_APT_MIRROR"); + return TRUE; + } + GEOID geo = GetUserGeoID(GEOCLASS_NATION); + if (geo == GEOID_NOT_AVAILABLE || + GetGeoInfoW(geo, GEO_ISO2, cc, ARRAYSIZE(cc), 0) < 2) + return FALSE; + for (wchar_t *p = cc; *p; p++) *p = (wchar_t)towlower(*p); + /* The main archive already lives in the UK. */ + if (wcscmp(cc, L"gb") == 0) return FALSE; + swprintf_s(out, cap, L"http://%s.archive.ubuntu.com/ubuntu", cc); + swprintf_s(why, why_cap, L"Windows region %s", cc); + return TRUE; +} + +/* Push a build-progress percentage for a VM that is still being created. + Same bookkeeping as the PROGRESS: handler in run_iso_patch_ubuntu. */ +static void report_build_progress(UINT64 vm_unique_id, int pct, const wchar_t *step) +{ + VmInstance *pvm; + EnterCriticalSection(&g_cs); + pvm = asb_find_vm_by_id(vm_unique_id); + if (pvm) { + pvm->vhdx_progress = pct; + pvm->vhdx_staging = FALSE; + /* vhdx_step is what the status cell shows instead of the generic + "Building Disk" while a phase has a better name. */ + if (step) wcsncpy_s(pvm->vhdx_step, ARRAYSIZE(pvm->vhdx_step), step, _TRUNCATE); + } + LeaveCriticalSection(&g_cs); + if (g_progress_cb && pvm) + g_progress_cb(vm_handle(pvm), pct, FALSE, g_progress_ud); +} + +/* args: iso-patch argv tail. pct_from..pct_to: the slice of the build + progress bar this prefetch owns; it advances per downloaded .deb (the + apt closure is the only prefetch with many files), so the UI moves + during the minutes that used to sit at "Building Disk (0%)". */ +static int spawn_iso_patch_prefetch(const wchar_t *args, UINT64 vm_unique_id, + int pct_from, int pct_to, const wchar_t *step_label) { wchar_t exe_dir[MAX_PATH]; GetModuleFileNameW(g_dll_module, exe_dir, MAX_PATH); @@ -2003,16 +2221,78 @@ static int spawn_iso_patch_prefetch(const wchar_t *args) swprintf_s(cmdline, 2048, L"\"%s\\iso-patch.exe\" %s", exe_dir, args); + /* Capture stdout/stderr so a failed prefetch says WHY in the app log + (previously the child's output went nowhere and the only trace was + "WARN: prefetch-... failed"). Per-file "GET" chatter is dropped; + summary STATUS: lines and every ERROR: line are forwarded. */ + HANDLE hRead = INVALID_HANDLE_VALUE, hWrite = INVALID_HANDLE_VALUE; + SECURITY_ATTRIBUTES sa = { sizeof(sa), NULL, TRUE }; + BOOL capture = CreatePipe(&hRead, &hWrite, &sa, 0); + if (capture) SetHandleInformation(hRead, HANDLE_FLAG_INHERIT, 0); + STARTUPINFOW si = { sizeof(si) }; PROCESS_INFORMATION pi = { 0 }; si.dwFlags = STARTF_USESHOWWINDOW; si.wShowWindow = SW_HIDE; - if (!CreateProcessW(NULL, cmdline, NULL, NULL, FALSE, + if (capture) { + si.dwFlags |= STARTF_USESTDHANDLES; + si.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + si.hStdOutput = hWrite; + si.hStdError = hWrite; + } + if (!CreateProcessW(NULL, cmdline, NULL, NULL, capture, CREATE_NO_WINDOW, NULL, NULL, &si, &pi)) { asb_log(L"prefetch: CreateProcess failed (%lu) for: %s", GetLastError(), cmdline); + if (capture) { CloseHandle(hRead); CloseHandle(hWrite); } return -1; } + if (vm_unique_id) report_build_progress(vm_unique_id, pct_from, step_label); + if (capture) { + CloseHandle(hWrite); + char buf[4096]; + int pos = 0; + DWORD n = 0; + int total = 0, got = 0; + while (ReadFile(hRead, buf + pos, (DWORD)(sizeof(buf) - pos - 1), &n, NULL) && n > 0) { + int end = pos + (int)n, start = 0; + buf[end] = '\0'; + for (int i = 0; i < end; i++) { + if (buf[i] != '\n' && buf[i] != '\r') continue; + buf[i] = '\0'; + if (i > start) { + const char *line = buf + start; + if (strncmp(line, "ERROR:", 6) == 0) { + asb_log(L"iso-patch: ERROR: %S", line + 6); + } else if (strncmp(line, "STATUS:prefetch: GET ", 21) == 0) { + /* one line per file; only the .debs are numerous */ + size_t len = strlen(line); + if (len > 4 && strcmp(line + len - 4, ".deb") == 0 && total > 0) { + got++; + if (vm_unique_id) { + wchar_t step[128]; + swprintf_s(step, ARRAYSIZE(step), L"%s %d/%d", + step_label ? step_label : L"Downloading", got, total); + report_build_progress(vm_unique_id, + pct_from + (int)((pct_to - pct_from) * (long long)got / total), + step); + } + } + } else if (strncmp(line, "STATUS:", 7) == 0 && + strncmp(line + 7, "xz_decompress", 13) != 0) { + if (strncmp(line + 7, "prefetch: closure = ", 20) == 0) + total = atoi(line + 27); + asb_log(L"iso-patch: %S", line + 7); + } + } + start = i + 1; + } + if (start < end) { memmove(buf, buf + start, end - start); pos = end - start; } + else pos = 0; + if (pos >= (int)sizeof(buf) - 1) pos = 0; /* overlong line: drop it */ + } + CloseHandle(hRead); + } WaitForSingleObject(pi.hProcess, INFINITE); DWORD ec = 1; GetExitCodeProcess(pi.hProcess, &ec); @@ -2112,18 +2392,42 @@ static HRESULT run_iso_patch_ubuntu(const wchar_t *iso_path, EnterCriticalSection(&g_cs); pvm = asb_find_vm_by_id(vm_unique_id); if (pvm) { + /* The prefetch phase already used 1..9%; keep + the bar monotonic through iso-patch's own + early 2/4/6% steps. */ + if (pct < 10 && pct < pvm->vhdx_progress) + pct = pvm->vhdx_progress; pvm->vhdx_progress = pct; pvm->vhdx_staging = is_staging; + /* PROGRESS:: - surface the step name + ("Building rootfs", "Installing grub modules", + ...) instead of the generic "Building Disk". */ + { + const char *step = strchr(line + 9, ':'); + if (step && step[1]) + MultiByteToWideChar(CP_UTF8, 0, step + 1, -1, + pvm->vhdx_step, (int)ARRAYSIZE(pvm->vhdx_step)); + else + pvm->vhdx_step[0] = L'\0'; + } } LeaveCriticalSection(&g_cs); if (g_progress_cb && pvm) g_progress_cb(vm_handle(pvm), pct, is_staging, g_progress_ud); } else if (strncmp(line, "ERROR:", 6) == 0) { + asb_log(L"iso-patch: ERROR: %S", line + 6); if (error_msg && error_msg[0] == L'\0') MultiByteToWideChar(CP_ACP, 0, line + 6, -1, error_msg, (int)error_msg_cap); result = E_FAIL; } else if (strncmp(line, "DONE:", 5) == 0) { result = S_OK; + } else if (strncmp(line, "STATUS:", 7) == 0) { + /* Unlike the Windows --to-vhdx path there is no + in-guest installer log to fall back on, so the + iso-patch status lines (source squashfs, detected + kernel, ingest totals, ...) are the only host-side + record of how the Linux disk was built. */ + asb_log(L"iso-patch: %S", line + 7); } } start = ci + 1; @@ -2219,11 +2523,16 @@ static DWORD WINAPI linux_create_thread(LPVOID param) asb_log(L"Prefetch 1/3: cloning repo source from GitHub..."); /* Branch must match the branch this binary was built from, so the Linux guest builds its agent/driver source from the SAME revision. */ - swprintf_s(args_buf, 2048, - L"--prefetch-repo --branch \"main\" --out-dir \"%s\"", - extras); - if (spawn_iso_patch_prefetch(args_buf) != 0) - asb_log(L"WARN: prefetch-repo failed (agent + DKMS build will fail)"); + if (!prefetch_cache_restore(L"repo-main", 24, extras, L"Prefetch 1/3")) { + swprintf_s(args_buf, 2048, + L"--prefetch-repo --branch \"main\" --out-dir \"%s\"", + extras); + if (spawn_iso_patch_prefetch(args_buf, args->vm_unique_id, 1, 3, + L"Downloading sources") != 0) + asb_log(L"WARN: prefetch-repo failed (agent + DKMS build will fail)"); + else + prefetch_cache_store(L"repo-main", extras); /* extras holds only repo output here */ + } /* Prefetch 2: apt build-deps closure from archive.ubuntu.com. Needs (codename, kernel) detected from the ISO. */ @@ -2232,14 +2541,25 @@ static DWORD WINAPI linux_create_thread(LPVOID param) if (detect_iso_kernel(args->config.image_path, codename, ARRAYSIZE(codename), kver, ARRAYSIZE(kver)) == 0) { - wchar_t apt_out[MAX_PATH]; + wchar_t apt_out[MAX_PATH], cache_key[160]; swprintf_s(apt_out, MAX_PATH, L"%s\\local-apt-extras", extras); - swprintf_s(args_buf, 2048, - L"--prefetch-build-deps --codename \"%s\" --kernel \"%s\" " - L"--out-dir \"%s\"", - codename, kver, apt_out); - if (spawn_iso_patch_prefetch(args_buf) != 0) - asb_log(L"WARN: prefetch-build-deps failed"); + swprintf_s(cache_key, ARRAYSIZE(cache_key), L"build-deps-%s-%s", codename, kver); + if (!prefetch_cache_restore(cache_key, 7 * 24, apt_out, L"Prefetch 2/3")) { + wchar_t mirror[512], why[64], mirror_arg[600] = L""; + if (choose_apt_mirror(mirror, ARRAYSIZE(mirror), why, ARRAYSIZE(why))) { + asb_log(L"Prefetch 2/3: mirror %s (%s)", mirror, why); + swprintf_s(mirror_arg, ARRAYSIZE(mirror_arg), L" --mirror \"%s\"", mirror); + } + swprintf_s(args_buf, 2048, + L"--prefetch-build-deps --codename \"%s\" --kernel \"%s\" " + L"--out-dir \"%s\"%s", + codename, kver, apt_out, mirror_arg); + if (spawn_iso_patch_prefetch(args_buf, args->vm_unique_id, 3, 8, + L"Downloading packages") != 0) + asb_log(L"WARN: prefetch-build-deps failed"); + else + prefetch_cache_store(cache_key, apt_out); + } } else { asb_log(L"WARN: could not detect ISO kernel — skipping build-deps"); } @@ -2248,10 +2568,15 @@ static DWORD WINAPI linux_create_thread(LPVOID param) asb_log(L"Prefetch 3/3: wsl-deps .so libs..."); wchar_t wsl_out[MAX_PATH]; swprintf_s(wsl_out, MAX_PATH, L"%s\\wsl-deps", extras); - swprintf_s(args_buf, 2048, - L"--prefetch-wsl-deps --out-dir \"%s\"", wsl_out); - if (spawn_iso_patch_prefetch(args_buf) != 0) - asb_log(L"WARN: prefetch-wsl-deps failed"); + if (!prefetch_cache_restore(L"wsl-deps", 30 * 24, wsl_out, L"Prefetch 3/3")) { + swprintf_s(args_buf, 2048, + L"--prefetch-wsl-deps --out-dir \"%s\"", wsl_out); + if (spawn_iso_patch_prefetch(args_buf, args->vm_unique_id, 8, 9, + L"Downloading GPU libraries") != 0) + asb_log(L"WARN: prefetch-wsl-deps failed"); + else + prefetch_cache_store(L"wsl-deps", wsl_out); + } } GetModuleFileNameW(g_dll_module, exe_dir, MAX_PATH); diff --git a/tools/iso-patch/prefetch_build_deps.c b/tools/iso-patch/prefetch_build_deps.c index a5dd66e..b7f7e34 100644 --- a/tools/iso-patch/prefetch_build_deps.c +++ b/tools/iso-patch/prefetch_build_deps.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -230,14 +231,31 @@ static int parse_url(const wchar_t *url, return 0; } +/* log_err() or log_msg() depending on `quiet`: a failed attempt that is + * going to be retried, or an optional resource, should not surface as an + * ERROR: line in the app log. */ +static void dl_log(int quiet, const wchar_t *fmt, ...) +{ + wchar_t buf[2304]; + va_list ap; + va_start(ap, fmt); + _vsnwprintf_s(buf, ARRAYSIZE(buf), _TRUNCATE, fmt, ap); + va_end(ap); + if (quiet) log_msg(L"%s", buf); + else log_err(L"%s", buf); +} + /* HTTP GET -> file. Returns 0 on success. Uses a persistent - * connection per-call (simple; for one-shot fetches this is fine). */ -static int http_download(const wchar_t *url, const wchar_t *out_path) + * connection per-call (simple; for one-shot fetches this is fine). + * quiet_http: report failures as STATUS: instead of ERROR: - for + * optional resources (e.g. a -updates pocket that may not exist) and + * for attempts that will be retried. */ +static int http_download_ex(const wchar_t *url, const wchar_t *out_path, int quiet_http) { wchar_t host[256], path[2048]; INTERNET_PORT port = 80; if (parse_url(url, host, ARRAYSIZE(host), &port, path, ARRAYSIZE(path)) != 0) { - log_err(L"prefetch: bad URL: %s", url); + dl_log(quiet_http, L"prefetch: bad URL: %s", url); return -1; } @@ -245,26 +263,26 @@ static int http_download(const wchar_t *url, const wchar_t *out_path) WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY, WINHTTP_NO_PROXY_NAME, WINHTTP_NO_PROXY_BYPASS, 0); - if (!hSession) { log_err(L"prefetch: WinHttpOpen failed: %lu", GetLastError()); return -1; } + if (!hSession) { dl_log(quiet_http, L"prefetch: WinHttpOpen failed: %lu", GetLastError()); return -1; } int rc = -1; HINTERNET hConn = WinHttpConnect(hSession, host, port, 0); - if (!hConn) { log_err(L"prefetch: WinHttpConnect %s:%u failed: %lu", host, port, GetLastError()); goto cleanup_sess; } + if (!hConn) { dl_log(quiet_http, L"prefetch: WinHttpConnect %s:%u failed: %lu", host, port, GetLastError()); goto cleanup_sess; } DWORD reqFlags = (port == 443) ? WINHTTP_FLAG_SECURE : 0; HINTERNET hReq = WinHttpOpenRequest(hConn, L"GET", path, NULL, WINHTTP_NO_REFERER, WINHTTP_DEFAULT_ACCEPT_TYPES, reqFlags); - if (!hReq) { log_err(L"prefetch: WinHttpOpenRequest failed: %lu", GetLastError()); goto cleanup_conn; } + if (!hReq) { dl_log(quiet_http, L"prefetch: WinHttpOpenRequest failed: %lu", GetLastError()); goto cleanup_conn; } if (!WinHttpSendRequest(hReq, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0)) { - log_err(L"prefetch: WinHttpSendRequest %s failed: %lu", url, GetLastError()); + dl_log(quiet_http, L"prefetch: WinHttpSendRequest %s failed: %lu", url, GetLastError()); goto cleanup_req; } if (!WinHttpReceiveResponse(hReq, NULL)) { - log_err(L"prefetch: WinHttpReceiveResponse failed: %lu", GetLastError()); + dl_log(quiet_http, L"prefetch: WinHttpReceiveResponse failed: %lu", GetLastError()); goto cleanup_req; } @@ -274,14 +292,15 @@ static int http_download(const wchar_t *url, const wchar_t *out_path) WINHTTP_HEADER_NAME_BY_INDEX, &status, &statusLen, WINHTTP_NO_HEADER_INDEX); if (status != 200) { - log_err(L"prefetch: HTTP %lu for %s", status, url); + if (quiet_http) log_msg(L"prefetch: HTTP %lu for %s", status, url); + else dl_log(quiet_http, L"prefetch: HTTP %lu for %s", status, url); goto cleanup_req; } HANDLE hFile = CreateFileW(out_path, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); if (hFile == INVALID_HANDLE_VALUE) { - log_err(L"prefetch: CreateFileW(%s) failed: %lu", out_path, GetLastError()); + dl_log(quiet_http, L"prefetch: CreateFileW(%s) failed: %lu", out_path, GetLastError()); goto cleanup_req; } @@ -290,20 +309,20 @@ static int http_download(const wchar_t *url, const wchar_t *out_path) for (;;) { DWORD avail = 0; if (!WinHttpQueryDataAvailable(hReq, &avail)) { - log_err(L"prefetch: WinHttpQueryDataAvailable failed: %lu", GetLastError()); + dl_log(quiet_http, L"prefetch: WinHttpQueryDataAvailable failed: %lu", GetLastError()); CloseHandle(hFile); goto cleanup_req; } if (avail == 0) break; DWORD n = avail > sizeof(buf) ? sizeof(buf) : avail; DWORD read = 0; if (!WinHttpReadData(hReq, buf, n, &read)) { - log_err(L"prefetch: WinHttpReadData failed: %lu", GetLastError()); + dl_log(quiet_http, L"prefetch: WinHttpReadData failed: %lu", GetLastError()); CloseHandle(hFile); goto cleanup_req; } if (read == 0) break; DWORD wrote = 0; if (!WriteFile(hFile, buf, read, &wrote, NULL) || wrote != read) { - log_err(L"prefetch: WriteFile failed: %lu", GetLastError()); + dl_log(quiet_http, L"prefetch: WriteFile failed: %lu", GetLastError()); CloseHandle(hFile); goto cleanup_req; } total += read; @@ -317,6 +336,52 @@ cleanup_sess: WinHttpCloseHandle(hSession); return rc; } +/* Retry transient failures (DNS hiccups, dropped connections, mirror + * 5xx): 3 attempts with a short back-off. Only the last attempt is + * allowed to log ERROR:. One flaky GET out of ~180 used to abort the + * whole prefetch and leave the guest without any build tools. */ +#define DL_ATTEMPTS 3 +static int http_download_retry(const wchar_t *url, const wchar_t *out_path, int quiet_http) +{ + for (int attempt = 1; attempt <= DL_ATTEMPTS; attempt++) { + int last = (attempt == DL_ATTEMPTS); + if (http_download_ex(url, out_path, quiet_http || !last) == 0) return 0; + DeleteFileW(out_path); + if (!last) { + log_msg(L"prefetch: retry %d/%d for %s", attempt + 1, DL_ATTEMPTS, url); + Sleep(2000UL * (DWORD)attempt); + } + } + return -1; +} + +static int http_download(const wchar_t *url, const wchar_t *out_path) +{ + return http_download_retry(url, out_path, 0); +} + +/* Read a whole file into a malloc'd, NUL-terminated buffer. */ +static int slurp_file(const wchar_t *path, char **buf_out, size_t *len_out) +{ + HANDLE h = CreateFileW(path, GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (h == INVALID_HANDLE_VALUE) return -1; + LARGE_INTEGER sz; + if (!GetFileSizeEx(h, &sz) || sz.QuadPart > 0x7fffffff) { CloseHandle(h); return -1; } + size_t len = (size_t)sz.QuadPart; + char *buf = (char *)malloc(len + 1); + if (!buf) { CloseHandle(h); return -1; } + DWORD br = 0; + if (len > 0 && (!ReadFile(h, buf, (DWORD)len, &br, NULL) || br != len)) { + free(buf); CloseHandle(h); return -1; + } + CloseHandle(h); + buf[len] = 0; + *buf_out = buf; + *len_out = len; + return 0; +} + /* ==================================================================== * SHA256 via BCrypt — verify the .debs against Packages metadata. * ==================================================================== */ @@ -679,17 +744,94 @@ static int write_closure_json(pkg_table_t *t, return 0; } +/* ==================================================================== + * Parallel .deb download + * ==================================================================== */ + +#define DL_WORKERS 6 + +typedef struct { + const wchar_t *mirror; + const wchar_t *staging; + pkg_record_t **list; /* closure records with a Filename: */ + size_t count; + volatile LONG next; /* next index to claim */ + volatile LONG downloaded; + volatile LONG failed; /* first failure stops the others */ +} dl_job_t; + +/* Download + verify one record. Returns 0 on success. */ +static int dl_one(const dl_job_t *job, pkg_record_t *r) +{ + char fn_utf8[1024], sha_utf8[128]; + if (r->filename_len >= sizeof(fn_utf8)) { + log_err(L"prefetch: Filename too long (%zu) for %.*hs", + r->filename_len, (int)r->name_len, r->name); + return -1; + } + memcpy(fn_utf8, r->filename, r->filename_len); fn_utf8[r->filename_len] = 0; + if (r->sha256_hex && r->sha256_len < sizeof(sha_utf8)) { + memcpy(sha_utf8, r->sha256_hex, r->sha256_len); sha_utf8[r->sha256_len] = 0; + } else { sha_utf8[0] = 0; } + + wchar_t fn_wide[1024]; + MultiByteToWideChar(CP_UTF8, 0, fn_utf8, -1, fn_wide, ARRAYSIZE(fn_wide)); + const wchar_t *basename = wcsrchr(fn_wide, L'/'); + basename = basename ? basename + 1 : fn_wide; + + wchar_t url2[2048], dst[MAX_PATH]; + swprintf_s(url2, 2048, L"%s/%s", job->mirror, fn_wide); + swprintf_s(dst, MAX_PATH, L"%s\\%s", job->staging, basename); + + if (http_download(url2, dst) != 0) { + log_err(L"prefetch: download %ls failed", basename); + return -1; + } + if (sha_utf8[0]) { + char actual[65]; + if (sha256_file(dst, actual) != 0) { + log_err(L"prefetch: SHA256 hash compute failed for %ls", basename); + return -1; + } + if (_stricmp(actual, sha_utf8) != 0) { + log_err(L"prefetch: SHA256 mismatch for %ls (got %hs, want %hs)", + basename, actual, sha_utf8); + return -1; + } + } + /* Logged after the fact so the "GET " line count equals files + done - the app derives its progress bar from it. */ + log_msg(L"prefetch: GET %s", basename); + return 0; +} + +static DWORD WINAPI dl_worker(LPVOID arg) +{ + dl_job_t *job = (dl_job_t *)arg; + for (;;) { + if (job->failed) break; + LONG idx = InterlockedIncrement(&job->next) - 1; + if ((size_t)idx >= job->count) break; + if (dl_one(job, job->list[idx]) != 0) { + InterlockedExchange(&job->failed, 1); + break; + } + InterlockedIncrement(&job->downloaded); + } + return 0; +} + /* ==================================================================== * Main entry point * ==================================================================== */ -int do_prefetch_build_deps(const wchar_t *codename, - const wchar_t *kernel_ver, - const wchar_t *out_dir, - const wchar_t *mirror_arg) +static int prefetch_build_deps_inner(const wchar_t *codename, + const wchar_t *kernel_ver, + const wchar_t *out_dir, + const wchar_t *mirror_arg) { - const wchar_t *mirror = mirror_arg ? mirror_arg - : L"http://archive.ubuntu.com/ubuntu"; + static const wchar_t *const default_mirror = L"http://archive.ubuntu.com/ubuntu"; + const wchar_t *mirror = mirror_arg ? mirror_arg : default_mirror; log_msg(L"prefetch: codename=%s kver=%s mirror=%s out=%s", codename, kernel_ver, mirror, out_dir); @@ -708,9 +850,22 @@ int do_prefetch_build_deps(const wchar_t *codename, swprintf_s(url, 1024, L"%s/dists/%s/main/binary-" IP_DEB_ARCH L"/Packages.xz", mirror, codename); log_msg(L"prefetch: GET %s", url); - if (http_download(url, pkgs_xz) != 0) { - log_err(L"prefetch: download Packages.xz failed"); - return -1; + if (http_download_retry(url, pkgs_xz, mirror != default_mirror) != 0) { + /* A country mirror (id.archive.ubuntu.com, ...) that is down or + lacks the release is not fatal: fall back to the main archive + for everything. */ + if (mirror != default_mirror) { + log_msg(L"prefetch: WARN mirror %s unusable - falling back to %s", + mirror, default_mirror); + mirror = default_mirror; + swprintf_s(url, 1024, L"%s/dists/%s/main/binary-" IP_DEB_ARCH L"/Packages.xz", + mirror, codename); + log_msg(L"prefetch: GET %s", url); + } + if (mirror != default_mirror || http_download(url, pkgs_xz) != 0) { + log_err(L"prefetch: download Packages.xz failed"); + return -1; + } } /* ---- 2. In-process xz decompression via vendored xz-embedded ---- */ @@ -719,22 +874,55 @@ int do_prefetch_build_deps(const wchar_t *codename, return -1; } - /* ---- 3. Slurp Packages into memory + parse ---- */ + /* ---- 2b. -updates as well. + Point-release ISOs (24.04.4, ...) are built from the -updates + pocket, so the base system already carries e.g. libasound2t64 + 1.2.11-1ubuntu0.1 while the release pocket's libasound2-dev pins + "= 1.2.11-1build2": apt in the guest then refuses the whole build + tool set. Merging the -updates index (preferred over the release + one, see step 3) keeps the closure in step with the ISO. A fresh + LTS without a -updates pocket yet simply skips this. ---- */ + wchar_t pkgs_upd_xz[MAX_PATH], pkgs_upd[MAX_PATH]; + int have_updates = 0; + swprintf_s(pkgs_upd_xz, MAX_PATH, L"%s\\Packages-updates.xz", staging); + swprintf_s(pkgs_upd, MAX_PATH, L"%s\\Packages-updates", staging); + swprintf_s(url, 1024, L"%s/dists/%s-updates/main/binary-" IP_DEB_ARCH L"/Packages.xz", + mirror, codename); + log_msg(L"prefetch: GET %s", url); + if (http_download_retry(url, pkgs_upd_xz, 1) == 0 && + xz_decompress_file_to_file(pkgs_upd_xz, pkgs_upd) == 0) { + have_updates = 1; + } else { + log_msg(L"prefetch: WARN no usable %s-updates index - using the release pocket only", + codename); + } + + /* ---- 3. Slurp Packages (+ Packages-updates) into memory + parse ---- */ pkg_table_t T = { 0 }; { - HANDLE h = CreateFileW(pkgs, GENERIC_READ, FILE_SHARE_READ, NULL, - OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (h == INVALID_HANDLE_VALUE) { log_err(L"prefetch: open Packages failed"); return -1; } - LARGE_INTEGER sz; GetFileSizeEx(h, &sz); - T.buf_len = (size_t)sz.QuadPart; - T.buf = (char *)malloc(T.buf_len + 1); - if (!T.buf) { CloseHandle(h); return -1; } - DWORD br = 0; - if (!ReadFile(h, T.buf, (DWORD)T.buf_len, &br, NULL) || br != T.buf_len) { - log_err(L"prefetch: read Packages failed"); CloseHandle(h); return -1; + char *base_buf = NULL, *upd_buf = NULL; + size_t base_len = 0, upd_len = 0; + if (slurp_file(pkgs, &base_buf, &base_len) != 0) { + log_err(L"prefetch: read Packages failed"); + return -1; + } + if (have_updates && slurp_file(pkgs_upd, &upd_buf, &upd_len) != 0) { + log_msg(L"prefetch: WARN read Packages-updates failed - ignoring it"); + upd_buf = NULL; upd_len = 0; } + /* Release pocket first, -updates after it. parse_packages() + prepends every stanza to its hash chain, so for a name present + in both pockets lookup_pkg() returns the -updates record. */ + T.buf_len = base_len + 2 + upd_len; + T.buf = (char *)malloc(T.buf_len + 1); + if (!T.buf) { free(base_buf); free(upd_buf); return -1; } + memcpy(T.buf, base_buf, base_len); + T.buf[base_len] = '\n'; + T.buf[base_len + 1] = '\n'; + if (upd_len) memcpy(T.buf + base_len + 2, upd_buf, upd_len); T.buf[T.buf_len] = 0; - CloseHandle(h); + free(base_buf); + free(upd_buf); } if (parse_packages(&T) != 0) { log_err(L"prefetch: parse failed"); return -1; } log_msg(L"prefetch: parsed %zu package records", T.n_records); @@ -743,7 +931,18 @@ int do_prefetch_build_deps(const wchar_t *codename, const char *seeds[] = { "libasound2-dev", "libxcb1-dev", "libxcb-xfixes0-dev", "libdrm-dev", "pkg-config", - "openssh-server" /* for ssh_enabled VMs; firstboot installs conditionally */ + "openssh-server", /* for ssh_enabled VMs; firstboot installs conditionally */ + /* The ISO pool carries build-essential/gcc/dkms but not always their + whole closure (24.04.4: no libstdc++-13-dev / libgcc-13-dev / + cpp-13), and on point releases the pool is a -updates snapshot, + so the closure has to come from the same pocket to satisfy the + strict "= version" Depends. Seeding them here (rather than + relying on the pool) makes firstboot STEP 7.4 self-contained. */ + "build-essential", "dkms", + /* Layered ISOs (24.04): the kernel is staged from the ISO's live layer + and firstboot STEP 7.6 tries to register it with dpkg. linux-image + needs initramfs-tools + linux-base, which the 24.04 ISO pool lacks. */ + "initramfs-tools", "linux-base" }; int closure_count = 0; for (size_t i = 0; i < sizeof(seeds) / sizeof(seeds[0]); i++) { @@ -775,54 +974,42 @@ int do_prefetch_build_deps(const wchar_t *codename, } log_msg(L"prefetch: closure = %d packages", total_added); - /* ---- 6. Download each .deb in closure, SHA256 verify ---- */ - int downloaded = 0; - for (size_t i = 0; i < T.n_records; i++) { - pkg_record_t *r = &T.records[i]; - if (!r->in_closure) continue; - if (!r->filename) { log_msg(L"prefetch: WARN %.*hs has no Filename", (int)r->name_len, r->name); continue; } - - /* Compose URL + local path. */ - char fn_utf8[1024], sha_utf8[128]; - if (r->filename_len >= sizeof(fn_utf8)) { - log_err(L"prefetch: Filename too long (%zu) for %.*hs", - r->filename_len, (int)r->name_len, r->name); - return -1; + /* ---- 6. Download each .deb in closure, SHA256 verify. + ~180 files; done on DL_WORKERS connections at once. The archive + (and most mirrors) cap per-connection throughput well below what + the link can do, so this is where the wall-clock goes. ---- */ + { + dl_job_t job = { 0 }; + job.mirror = mirror; + job.staging = staging; + job.list = (pkg_record_t **)calloc(T.n_records, sizeof(pkg_record_t *)); + if (!job.list) return -1; + for (size_t i = 0; i < T.n_records; i++) { + pkg_record_t *r = &T.records[i]; + if (!r->in_closure) continue; + if (!r->filename) { log_msg(L"prefetch: WARN %.*hs has no Filename", (int)r->name_len, r->name); continue; } + job.list[job.count++] = r; } - memcpy(fn_utf8, r->filename, r->filename_len); fn_utf8[r->filename_len] = 0; - if (r->sha256_hex && r->sha256_len < sizeof(sha_utf8)) { - memcpy(sha_utf8, r->sha256_hex, r->sha256_len); sha_utf8[r->sha256_len] = 0; - } else { sha_utf8[0] = 0; } - - wchar_t fn_wide[1024]; - MultiByteToWideChar(CP_UTF8, 0, fn_utf8, -1, fn_wide, ARRAYSIZE(fn_wide)); - const wchar_t *basename = wcsrchr(fn_wide, L'/'); - basename = basename ? basename + 1 : fn_wide; - - wchar_t url2[2048], dst[MAX_PATH]; - swprintf_s(url2, 2048, L"%s/%s", mirror, fn_wide); - swprintf_s(dst, MAX_PATH, L"%s\\%s", staging, basename); - - log_msg(L"prefetch: GET %s", basename); - if (http_download(url2, dst) != 0) { - log_err(L"prefetch: download %ls failed", basename); - return -1; + int nthreads = (int)job.count < DL_WORKERS ? (int)job.count : DL_WORKERS; + HANDLE th[DL_WORKERS]; + int started = 0; + for (int t = 0; t < nthreads; t++) { + th[t] = CreateThread(NULL, 0, dl_worker, &job, 0, NULL); + if (th[t]) started++; } - if (sha_utf8[0]) { - char actual[65]; - if (sha256_file(dst, actual) != 0) { - log_err(L"prefetch: SHA256 hash compute failed for %ls", basename); - return -1; - } - if (_stricmp(actual, sha_utf8) != 0) { - log_err(L"prefetch: SHA256 mismatch for %ls (got %hs, want %hs)", - basename, actual, sha_utf8); - return -1; - } + if (started == 0) { + /* Thread creation failed outright: do it inline instead. */ + dl_worker(&job); } - downloaded++; + for (int t = 0; t < started; t++) { + WaitForSingleObject(th[t], INFINITE); + CloseHandle(th[t]); + } + free(job.list); + if (job.failed) return -1; + log_msg(L"prefetch: downloaded %ld .debs (%d connection(s))", + (long)job.downloaded, started ? started : 1); } - log_msg(L"prefetch: downloaded %d .debs", downloaded); /* ---- 7. Write synthetic Packages + .closure.json ---- */ { @@ -842,11 +1029,34 @@ int do_prefetch_build_deps(const wchar_t *codename, } /* Clean up Packages.xz — we don't ship it (we wrote our own - synthetic Packages with closure entries only). */ + synthetic Packages with closure entries only). Same for the + -updates index. */ DeleteFileW(pkgs_xz); + DeleteFileW(pkgs_upd_xz); + DeleteFileW(pkgs_upd); free(T.buf); free(T.records); log_msg(L"prefetch: OK -> %s", out_dir); return 0; } + +int do_prefetch_build_deps(const wchar_t *codename, + const wchar_t *kernel_ver, + const wchar_t *out_dir, + const wchar_t *mirror_arg) +{ + int rc = prefetch_build_deps_inner(codename, kernel_ver, out_dir, mirror_arg); + if (rc != 0) { + /* A partial staging dir is worse than none: the raw Packages index + downloaded in step 1 lists the release-pocket versions with + archive Filename: paths, and apt in the guest happily picks + those "= version" candidates over the ISO pool - then fails on + every build tool. Leave an empty dir so firstboot reports + "no local-apt-extras (host prefetch failed)" instead. */ + log_msg(L"prefetch: failed - discarding partial output in %s", out_dir); + u_rmdir_recursive(out_dir); + u_mkdir_p(out_dir); + } + return rc; +} diff --git a/tools/iso-patch/ubuntu_vhdx.c b/tools/iso-patch/ubuntu_vhdx.c index 696650f..a90b1f8 100644 --- a/tools/iso-patch/ubuntu_vhdx.c +++ b/tools/iso-patch/ubuntu_vhdx.c @@ -581,6 +581,13 @@ typedef struct { uint32_t total_entries; char kernel_version[64]; + uint64_t kernel_size; /* size of the /boot/vmlinuz-* seen */ + + /* Optional include filter: NULL-terminated list of path prefixes. When + set, the producer only ingests entries whose path starts with one of + them. Used by the layered-ISO fallback to pull just the kernel + + modules out of the live overlay. NULL = ingest everything. */ + const char *const *include_prefixes; work_queue_t wq; HANDLE workers[DECOMPRESS_WORKERS_MAX]; @@ -719,6 +726,14 @@ static int producer_cb(const sqfs_entry_t *e, void *user) if (e->path[0] == 0) return 0; + if (p->include_prefixes) { + int keep = 0; + for (const char *const *pp = p->include_prefixes; *pp; pp++) { + if (strncmp(e->path, *pp, strlen(*pp)) == 0) { keep = 1; break; } + } + if (!keep) return 0; + } + if (p->kernel_version[0] == 0) { const char *m = strstr(e->path, "/boot/vmlinuz-"); if (m) { @@ -727,6 +742,7 @@ static int producer_cb(const sqfs_entry_t *e, void *user) while (v[n] && v[n] != '/' && n < sizeof(p->kernel_version) - 1) n++; memcpy(p->kernel_version, v, n); p->kernel_version[n] = 0; + p->kernel_size = e->size; log_msg(L"detected kernel from squashfs: %hs (path=%hs)", p->kernel_version, e->path); } @@ -820,13 +836,15 @@ static DWORD WINAPI consumer_thread(LPVOID arg) the post-ingest phases (grub modules install, manifest staging, ESP setup) which together take ~10-15 s vs the ~2-minute ingest. */ - int pct = 10; + /* total_entries == 0 means "no meaningful total" (filtered + ingest of an overlay layer): keep the previous percentage + instead of snapping the bar back to 10. */ if (p->total_entries > 0) { uint64_t frac = ((uint64_t)p->n_processed * 65ULL) / p->total_entries; - pct = 10 + (int)frac; + int pct = 10 + (int)frac; if (pct > 75) pct = 75; + log_progress(pct, L"Building rootfs"); } - log_progress(pct, L"Building rootfs"); last_progress = now; } @@ -1313,6 +1331,33 @@ static void plant_firstboot_service(ext4_writer_t *ew) " apt-cache policy $APT_OPTS 2>&1 | head -30\n" "fi\n" "\n" + "# --- STEP 7.6: register the ISO-staged kernel with dpkg (layered ISOs) ---\n" + "# On ISOs whose minimal.squashfs has no kernel (24.04), iso-patch staged\n" + "# /boot/vmlinuz-$KVER + /usr/lib/modules/$KVER from the live layer and\n" + "# casper/initrd as /boot/initrd.img-$KVER. That boots (boot=local), but\n" + "# dpkg knows nothing about it. If the local mirrors can satisfy the\n" + "# kernel packages, install them so dpkg owns the files and a regular\n" + "# initrd gets generated. Best-effort: on any failure the ISO initrd +\n" + "# the boot=local grub drop-in keep working.\n" + "if [ -f /etc/appsandbox-kernel-from-iso ]; then\n" + " echo \"==== STEP 7.6: register ISO-staged kernel $TGT_KVER with dpkg ====\"\n" + " KPKGS=\"linux-image-$TGT_KVER linux-modules-$TGT_KVER linux-modules-extra-$TGT_KVER\"\n" + " cp -f \"/boot/initrd.img-$TGT_KVER\" \"/boot/initrd.img-$TGT_KVER.iso\" 2>/dev/null || true\n" + " if DEBIAN_FRONTEND=noninteractive apt-get install -y $APT_OPTS --dry-run $KPKGS >/dev/null 2>&1; then\n" + " if DEBIAN_FRONTEND=noninteractive apt-get install -y $APT_OPTS $KPKGS 2>&1 | tail -10; then\n" + " echo \"OK: kernel packages registered with dpkg\"\n" + " else\n" + " echo \"WARN: kernel package install failed (rc=$?) - keeping ISO initrd\"\n" + " fi\n" + " else\n" + " echo \"SKIP STEP 7.6: kernel packages not resolvable from local apt - keeping ISO initrd\"\n" + " fi\n" + " if [ ! -s \"/boot/initrd.img-$TGT_KVER\" ] && [ -s \"/boot/initrd.img-$TGT_KVER.iso\" ]; then\n" + " cp -f \"/boot/initrd.img-$TGT_KVER.iso\" \"/boot/initrd.img-$TGT_KVER\"\n" + " echo \"WARN: restored ISO initrd\"\n" + " fi\n" + "fi\n" + "\n" "# --- STEP 7.5: optional openssh-server (gated on host marker) ---\n" "# Host drops /etc/appsandbox-ssh-enabled in the manifest when the\n" "# user requested SSH. openssh-server came in through prefetch-build-deps.\n" @@ -1452,11 +1497,28 @@ static void plant_firstboot_service(ext4_writer_t *ew) " zstd -d \"$EXTRAS/wsl-mesa.tar.zst\" -c | tar -C / -x \\\n" " && echo \"OK: wsl-mesa extracted to /opt/wsl-mesa\" \\\n" " || echo \"FAIL: wsl-mesa extract\"\n" - " echo /opt/wsl-mesa/lib/" IP_MULTIARCH_A " > /etc/ld.so.conf.d/wsl-mesa.conf\n" - " install -d /etc/vulkan/icd.d\n" - " if [ -f /opt/wsl-mesa/share/vulkan/icd.d/" IP_DZN_ICD_A " ]; then\n" - " ln -sf /opt/wsl-mesa/share/vulkan/icd.d/" IP_DZN_ICD_A " \\\n" - " /etc/vulkan/icd.d/" IP_DZN_ICD_A "\n" + " # The prebuilt is tied to the release it was built on (26.04:\n" + " # Mesa 25.3 against LLVM 21). Putting it on ld.so.conf shadows the\n" + " # distro Mesa for every process, so if its libraries do not\n" + " # resolve here mutter cannot even create a GBM device and the\n" + " # desktop never comes up (black display). Check before activating;\n" + " # on a mismatch drop it and stay on stock Mesa, whose own d3d12\n" + " # driver still gives OpenGL apps the host GPU (see 50-appsandbox-gpu).\n" + " MISSING=$(ldd /opt/wsl-mesa/lib/" IP_MULTIARCH_A "/libgallium-*.so \\\n" + " /opt/wsl-mesa/lib/" IP_MULTIARCH_A "/libgbm.so.1 2>/dev/null \\\n" + " | grep 'not found' | awk '{print $1}' | sort -u | tr '\\n' ' ')\n" + " if [ -n \"$MISSING\" ]; then\n" + " echo \"WARN: wsl-mesa prebuilt needs libraries this release lacks: $MISSING\"\n" + " echo \"WARN: removing /opt/wsl-mesa - desktop + apps use the distro Mesa\"\n" + " rm -rf /opt/wsl-mesa\n" + " rm -f /etc/ld.so.conf.d/wsl-mesa.conf\n" + " else\n" + " echo /opt/wsl-mesa/lib/" IP_MULTIARCH_A " > /etc/ld.so.conf.d/wsl-mesa.conf\n" + " install -d /etc/vulkan/icd.d\n" + " if [ -f /opt/wsl-mesa/share/vulkan/icd.d/" IP_DZN_ICD_A " ]; then\n" + " ln -sf /opt/wsl-mesa/share/vulkan/icd.d/" IP_DZN_ICD_A " \\\n" + " /etc/vulkan/icd.d/" IP_DZN_ICD_A "\n" + " fi\n" " fi\n" " fi\n" "else\n" @@ -1645,6 +1707,26 @@ static void plant_firstboot_service(ext4_writer_t *ew) "echo \" timezone: $(cat /etc/timezone 2>/dev/null)\"\n" "set -x\n" "\n" + "# --- STEP 98: ext4 journal for / ---\n" + "# iso-patch's ext4 writer lays the root fs down without a journal. That\n" + "# is fine for the build, but an unclean stop of the running VM (Force\n" + "# Stop, host crash) then needs a full fsck, and a journal-less fs is\n" + "# far more likely to end up 'not clean with errors' and stall the next\n" + "# boot at a maintenance prompt. tune2fs can add the journal to the\n" + "# mounted root (it lands as a regular file and is adopted by the kernel\n" + "# on the next mount). fsck.repair=yes on the cmdline covers the rest.\n" + "echo \"==== STEP 98: ext4 journal on / ====\"\n" + "ROOTDEV=$(findmnt -n -o SOURCE / 2>/dev/null)\n" + "if [ -b \"$ROOTDEV\" ] && ! tune2fs -l \"$ROOTDEV\" 2>/dev/null | grep -q has_journal; then\n" + " if tune2fs -O has_journal \"$ROOTDEV\" 2>&1 | tail -3; then\n" + " echo \"OK: journal added to $ROOTDEV\"\n" + " else\n" + " echo \"WARN: tune2fs -O has_journal $ROOTDEV failed (rc=$?)\"\n" + " fi\n" + "else\n" + " echo \"SKIP STEP 98: $ROOTDEV already has a journal (or not a block device)\"\n" + "fi\n" + "\n" "# --- STEP 99: Mark done + reboot ---\n" "echo \"==== STEP 99: mark done + reboot ====\"\n" "mkdir -p /var/lib || true\n" @@ -1767,6 +1849,269 @@ static int stage_manifest_into_rootfs(const wchar_t *manifest_path, return count; } +/* ====================================================================== + * Squashfs -> ext4 ingest driver. + * + * Runs the producer / decompress-worker / consumer pipeline over one + * squashfs. include_prefixes == NULL ingests everything (the rootfs + * layer); a NULL-terminated prefix list ingests only matching paths + * (used to lift the kernel + modules out of an overlay layer). + * + * Returns 0 when the walk completed with no per-entry errors, else -1. + * On return all threads are joined and the sync objects destroyed; the + * caller still owns sq and ew. kernel_ver is only written when a + * /boot/vmlinuz- was seen. + * ====================================================================== */ +static int ingest_squashfs(sqfs_ctx_t *sq, ext4_writer_t *ew, + const char *const *include_prefixes, + char *kernel_ver, size_t kernel_ver_cap, + uint64_t *kernel_size_out) +{ + pipeline_t pl = { 0 }; + pl.sq = sq; + pl.ew = ew; + pl.include_prefixes = include_prefixes; + /* Drives the progress %. A filtered walk has no meaningful total + (most entries are skipped), so leave it 0 = "don't report". */ + pl.total_entries = include_prefixes ? 0 : sqfs_sb(sq)->inode_count; + InitializeCriticalSection(&pl.cs); + InitializeConditionVariable(&pl.cv_consumer); + InitializeConditionVariable(&pl.cv_producer); + InitializeCriticalSection(&pl.wq.cs); + InitializeConditionVariable(&pl.wq.cv_worker); + InitializeConditionVariable(&pl.wq.cv_walker); + pl.n_workers = decide_worker_count(); + log_msg(L"ingest: %d decompress workers", pl.n_workers); + + for (int wi = 0; wi < pl.n_workers; wi++) { + DWORD tid; + pl.workers[wi] = CreateThread(NULL, 0, decompress_worker, &pl, 0, &tid); + if (!pl.workers[wi]) { + log_err(L"CreateThread(worker) failed"); + /* Stop + join the workers already created so they don't + dereference the stack-local 'pl' after we return. */ + EnterCriticalSection(&pl.wq.cs); + pl.wq.stop = 1; + WakeAllConditionVariable(&pl.wq.cv_worker); + LeaveCriticalSection(&pl.wq.cs); + if (wi > 0) { + WaitForMultipleObjects(wi, pl.workers, TRUE, INFINITE); + for (int wj = 0; wj < wi; wj++) CloseHandle(pl.workers[wj]); + } + DeleteCriticalSection(&pl.cs); + DeleteCriticalSection(&pl.wq.cs); + return -1; + } + } + DWORD ctid; + HANDLE consumer = CreateThread(NULL, 0, consumer_thread, &pl, 0, &ctid); + if (!consumer) { + log_err(L"CreateThread(consumer) failed"); + /* Stop + join all workers (no slots pushed yet; they are + blocked in work_queue_pop) before the stack-local 'pl' dies. */ + EnterCriticalSection(&pl.wq.cs); + pl.wq.stop = 1; + WakeAllConditionVariable(&pl.wq.cv_worker); + LeaveCriticalSection(&pl.wq.cs); + WaitForMultipleObjects(pl.n_workers, pl.workers, TRUE, INFINITE); + for (int wi = 0; wi < pl.n_workers; wi++) CloseHandle(pl.workers[wi]); + DeleteCriticalSection(&pl.cs); + DeleteCriticalSection(&pl.wq.cs); + return -1; + } + + int walk_rc = sqfs_walk(sq, producer_cb, &pl); + EnterCriticalSection(&pl.wq.cs); + pl.wq.stop = 1; + WakeAllConditionVariable(&pl.wq.cv_worker); + LeaveCriticalSection(&pl.wq.cs); + WaitForMultipleObjects(pl.n_workers, pl.workers, TRUE, INFINITE); + for (int wi = 0; wi < pl.n_workers; wi++) CloseHandle(pl.workers[wi]); + + EnterCriticalSection(&pl.cs); + pl.stop = 1; + WakeConditionVariable(&pl.cv_consumer); + LeaveCriticalSection(&pl.cs); + WaitForSingleObject(consumer, INFINITE); + CloseHandle(consumer); + + DeleteCriticalSection(&pl.cs); + DeleteCriticalSection(&pl.wq.cs); + log_msg(L"ingest: walk=%d files=%zu dirs=%zu syms=%zu special=%zu errors=%zu bytes=%.1f MiB", + walk_rc, pl.n_files, pl.n_dirs, pl.n_syms, pl.n_special, pl.n_errors, + (double)pl.bytes_total / (1024.0 * 1024.0)); + /* The squashfs reader is the integrity detector: a non-zero walk_rc + means a structural traversal failure and n_errors counts per-file + data/decompress failures. Either one means the rootfs is only + partially ingested, so refuse to finalise it as success (leaving + exit_code = 1 deletes the incomplete VHDX in cleanup). */ + if (walk_rc != 0 || pl.n_errors != 0) { + log_err(L"squashfs ingest incomplete (walk=%d errors=%zu): " + L"aborting, source image may be truncated or corrupt", + walk_rc, pl.n_errors); + return -1; + } + if (kernel_ver && pl.kernel_version[0]) { + strncpy(kernel_ver, pl.kernel_version, kernel_ver_cap - 1); + kernel_ver[kernel_ver_cap - 1] = 0; + } + if (kernel_size_out) *kernel_size_out = pl.kernel_size; + return 0; +} + +/* Read a whole host file into a malloc'd buffer. Returns NULL on failure. */ +static void *u_read_whole_file(const wchar_t *path, uint64_t *size_out) +{ + HANDLE h = CreateFileW(path, GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL); + if (h == INVALID_HANDLE_VALUE) return NULL; + LARGE_INTEGER sz; + if (!GetFileSizeEx(h, &sz) || sz.QuadPart <= 0 || sz.QuadPart > 0x7fffffff) { + CloseHandle(h); + return NULL; + } + void *buf = malloc((size_t)sz.QuadPart); + DWORD br = 0; + if (!buf || !ReadFile(h, buf, (DWORD)sz.QuadPart, &br, NULL) || br != sz.QuadPart) { + free(buf); + CloseHandle(h); + return NULL; + } + CloseHandle(h); + if (size_out) *size_out = (uint64_t)sz.QuadPart; + return buf; +} + +/* ====================================================================== + * Layered-ISO kernel fallback. + * + * Ubuntu Desktop ISOs from 23.04 on are "fsimage-layered": the installer + * stacks casper/minimal.squashfs (base) + minimal.standard.squashfs + + * minimal.standard.live.squashfs. On 26.04 the base layer already holds + * /boot/vmlinuz-* + /boot/initrd.img-* + /usr/lib/modules/*, so the plain + * ingest is bootable. On 24.04 the base layer has no kernel at all: it is + * only in the live overlay, and the initrd only exists as casper/initrd. + * + * This stages exactly those pieces on top of the already-ingested base: + * /boot/vmlinuz-, /boot/System.map-, /boot/config- + * /usr/lib/modules//... from the live overlay + * /boot/initrd.img- = casper/initrd (the live initrd) + * + * The live initrd defaults to BOOT=casper but honours boot= on the kernel + * command line, so the bootstrap grub.cfg gets "boot=local" and a + * /etc/default/grub.d drop-in keeps it there after update-grub. A marker + * file lets the first-boot script register the kernel with dpkg when the + * local apt mirrors can satisfy it (best-effort; the ISO initrd keeps + * working either way). + * + * Returns 0 and fills kernel_ver on success, -1 if the ISO has no usable + * live layer / initrd. + * ====================================================================== */ +static int stage_kernel_from_live_layer(wchar_t iso_drive, ext4_writer_t *ew, + char *kernel_ver, size_t kernel_ver_cap) +{ + static const wchar_t *const layers[] = { + L"minimal.standard.live.squashfs", /* 24.04 desktop */ + L"minimal.live.squashfs", + L"filesystem.squashfs", /* legacy single-layer ISOs */ + }; + static const char *const prefixes[] = { + "/boot/vmlinuz-", + "/boot/System.map-", + "/boot/config-", + "/usr/lib/modules/", + NULL + }; + wchar_t sqfs_path[MAX_PATH] = { 0 }; + const wchar_t *layer = NULL; + + for (size_t i = 0; i < sizeof(layers) / sizeof(layers[0]); i++) { + swprintf(sqfs_path, MAX_PATH, L"%c:\\casper\\%s", iso_drive, layers[i]); + if (GetFileAttributesW(sqfs_path) != INVALID_FILE_ATTRIBUTES) { + layer = layers[i]; + break; + } + } + if (!layer) { + log_msg(L"layered ISO: no live overlay layer found under casper/"); + return -1; + } + log_msg(L"layered ISO: staging kernel + modules from casper/%s", layer); + + /* The base layer of a layered ISO usually has no /usr/lib/modules at + all; the overlay's entries need their parent to exist. Idempotent. */ + ext4_mkdir_p(ew, "/usr/lib/modules"); + + sqfs_ctx_t *sq = sqfs_open(sqfs_path); + if (!sq) { + log_msg(L"layered ISO: sqfs_open(%s) failed", sqfs_path); + return -1; + } + uint64_t kernel_size = 0; + int rc = ingest_squashfs(sq, ew, prefixes, kernel_ver, kernel_ver_cap, &kernel_size); + sqfs_close(sq); + if (rc != 0) return -1; + if (kernel_ver[0] == 0) { + log_msg(L"layered ISO: casper/%s has no /boot/vmlinuz-* either", layer); + return -1; + } + + /* Initrd: the live one under casper/. Sanity-check that casper/vmlinuz + is the same build as the kernel we just staged (same size) - on + official Ubuntu ISOs they are identical files. */ + { + wchar_t p[MAX_PATH]; + swprintf(p, MAX_PATH, L"%c:\\casper\\vmlinuz", iso_drive); + WIN32_FILE_ATTRIBUTE_DATA fad; + if (GetFileAttributesExW(p, GetFileExInfoStandard, &fad)) { + uint64_t sz = ((uint64_t)fad.nFileSizeHigh << 32) | fad.nFileSizeLow; + if (sz != kernel_size) + log_msg(L"WARN: casper/vmlinuz (%llu bytes) differs from /boot/vmlinuz-%hs (%llu bytes); " + L"casper/initrd may not match the staged kernel", + (unsigned long long)sz, kernel_ver, (unsigned long long)kernel_size); + } + } + { + wchar_t p[MAX_PATH]; + uint64_t initrd_size = 0; + swprintf(p, MAX_PATH, L"%c:\\casper\\initrd", iso_drive); + void *initrd = u_read_whole_file(p, &initrd_size); + if (!initrd) { + log_msg(L"layered ISO: cannot read %s", p); + return -1; + } + char dst[128]; + snprintf(dst, sizeof(dst), "/boot/initrd.img-%s", kernel_ver); + int arc = ext4_writer_add_file(ew, dst, 0644, 0, 0, (uint32_t)time(NULL), + initrd, initrd_size); + free(initrd); + if (arc != 0) { + log_msg(L"layered ISO: staging %hs failed", dst); + return -1; + } + log_msg(L"layered ISO: casper/initrd (%.1f MiB) staged as %hs", + (double)initrd_size / (1024.0 * 1024.0), dst); + } + + /* Keep boot=local on the cmdline after the guest regenerates grub.cfg, + and leave a marker for the first-boot script (STEP 7.6). */ + { + static const char dropin[] = + "# Written by AppSandbox iso-patch: the kernel and initrd were staged\n" + "# from the ISO's live layer. The live initrd defaults to BOOT=casper\n" + "# unless told otherwise, so keep boot=local on the kernel cmdline.\n" + "GRUB_CMDLINE_LINUX=\"$GRUB_CMDLINE_LINUX boot=local\"\n"; + ext4_mkdir_p(ew, "/etc/default/grub.d"); + ext4_writer_add_file(ew, "/etc/default/grub.d/90-appsandbox-iso-kernel.cfg", + 0644, 0, 0, (uint32_t)time(NULL), dropin, sizeof(dropin) - 1); + char marker[80]; + int n = snprintf(marker, sizeof(marker), "%s\n", kernel_ver); + ext4_writer_add_file(ew, "/etc/appsandbox-kernel-from-iso", + 0644, 0, 0, (uint32_t)time(NULL), marker, (uint64_t)n); + } + return 0; +} + /* ====================================================================== * Top-level orchestrator. * ====================================================================== */ @@ -1990,6 +2335,7 @@ int do_ubuntu_to_vhdx(const wchar_t *iso_path_arg, log_msg(L"root UUID: %hs", uuid_text); char kernel_ver[64] = { 0 }; + int kernel_from_iso = 0; { sqfs_ctx_t *sq = sqfs_open(sqfs_path); if (!sq) { @@ -1997,92 +2343,36 @@ int do_ubuntu_to_vhdx(const wchar_t *iso_path_arg, ext4_writer_close(ew); goto cleanup; } - pipeline_t pl = { 0 }; - pl.sq = sq; - pl.ew = ew; - pl.total_entries = sqfs_sb(sq)->inode_count; /* drives progress % */ - InitializeCriticalSection(&pl.cs); - InitializeConditionVariable(&pl.cv_consumer); - InitializeConditionVariable(&pl.cv_producer); - InitializeCriticalSection(&pl.wq.cs); - InitializeConditionVariable(&pl.wq.cv_worker); - InitializeConditionVariable(&pl.wq.cv_walker); - pl.n_workers = decide_worker_count(); - log_msg(L"ingest: %d decompress workers", pl.n_workers); - - for (int wi = 0; wi < pl.n_workers; wi++) { - DWORD tid; - pl.workers[wi] = CreateThread(NULL, 0, decompress_worker, &pl, 0, &tid); - if (!pl.workers[wi]) { - log_err(L"CreateThread(worker) failed"); - /* Stop + join the workers already created so they don't - dereference the stack-local 'pl' after we return. */ - EnterCriticalSection(&pl.wq.cs); - pl.wq.stop = 1; - WakeAllConditionVariable(&pl.wq.cv_worker); - LeaveCriticalSection(&pl.wq.cs); - if (wi > 0) { - WaitForMultipleObjects(wi, pl.workers, TRUE, INFINITE); - for (int wj = 0; wj < wi; wj++) CloseHandle(pl.workers[wj]); - } - DeleteCriticalSection(&pl.cs); - DeleteCriticalSection(&pl.wq.cs); - sqfs_close(sq); ext4_writer_close(ew); goto cleanup; - } - } - DWORD ctid; - HANDLE consumer = CreateThread(NULL, 0, consumer_thread, &pl, 0, &ctid); - if (!consumer) { - log_err(L"CreateThread(consumer) failed"); - /* Stop + join all workers (no slots pushed yet; they are - blocked in work_queue_pop) before the stack-local 'pl' dies. */ - EnterCriticalSection(&pl.wq.cs); - pl.wq.stop = 1; - WakeAllConditionVariable(&pl.wq.cv_worker); - LeaveCriticalSection(&pl.wq.cs); - WaitForMultipleObjects(pl.n_workers, pl.workers, TRUE, INFINITE); - for (int wi = 0; wi < pl.n_workers; wi++) CloseHandle(pl.workers[wi]); - DeleteCriticalSection(&pl.cs); - DeleteCriticalSection(&pl.wq.cs); - sqfs_close(sq); ext4_writer_close(ew); goto cleanup; + int rc = ingest_squashfs(sq, ew, NULL, kernel_ver, sizeof(kernel_ver), NULL); + sqfs_close(sq); + if (rc != 0) { + ext4_writer_close(ew); + goto cleanup; } + } - int walk_rc = sqfs_walk(sq, producer_cb, &pl); - EnterCriticalSection(&pl.wq.cs); - pl.wq.stop = 1; - WakeAllConditionVariable(&pl.wq.cv_worker); - LeaveCriticalSection(&pl.wq.cs); - WaitForMultipleObjects(pl.n_workers, pl.workers, TRUE, INFINITE); - for (int wi = 0; wi < pl.n_workers; wi++) CloseHandle(pl.workers[wi]); - - EnterCriticalSection(&pl.cs); - pl.stop = 1; - WakeConditionVariable(&pl.cv_consumer); - LeaveCriticalSection(&pl.cs); - WaitForSingleObject(consumer, INFINITE); - CloseHandle(consumer); - - DeleteCriticalSection(&pl.cs); - DeleteCriticalSection(&pl.wq.cs); - log_msg(L"ingest: walk=%d files=%zu dirs=%zu syms=%zu special=%zu errors=%zu bytes=%.1f MiB", - walk_rc, pl.n_files, pl.n_dirs, pl.n_syms, pl.n_special, pl.n_errors, - (double)pl.bytes_total / (1024.0 * 1024.0)); - /* The squashfs reader is the integrity detector: a non-zero walk_rc - means a structural traversal failure and n_errors counts per-file - data/decompress failures. Either one means the rootfs is only - partially ingested, so refuse to finalise it as success (leaving - exit_code = 1 deletes the incomplete VHDX in cleanup). */ - if (walk_rc != 0 || pl.n_errors != 0) { - log_err(L"squashfs ingest incomplete (walk=%d errors=%zu): " - L"aborting, source image may be truncated or corrupt", - walk_rc, pl.n_errors); - sqfs_close(sq); + /* ---- Step 5b: Layered-ISO fallback. + minimal.squashfs carries the kernel on the 26.04 desktop ISO. On + 24.04 (and other "fsimage-layered" ISOs) it does not: the kernel and + its modules live only in the minimal.standard.live overlay and the + initrd only under casper/. Without this the bootstrap grub.cfg would + point at "/boot/vmlinuz-" and the VM would sit at the GRUB prompt + forever (jamesstringer90/appsandbox#66). Pull the kernel pieces out + of the ISO instead; if even that fails, refuse to build a disk that + cannot boot. ---- */ + if (kernel_ver[0] == 0) { + log_msg(L"no /boot/vmlinuz-* in %s (layered ISO) - staging the kernel from the live layer", sqfs_path); + if (stage_kernel_from_live_layer(iso_drive, ew, kernel_ver, sizeof(kernel_ver)) != 0) { + log_err(L"no kernel (/boot/vmlinuz-*) found in %s and none could be staged " + L"from the ISO's live layer - the disk would not boot. " + L"Use an Ubuntu Desktop 26.04 LTS or 24.04 LTS ISO.", sqfs_path); ext4_writer_close(ew); goto cleanup; } - strncpy(kernel_ver, pl.kernel_version, sizeof(kernel_ver) - 1); - sqfs_close(sq); + kernel_from_iso = 1; } + log_msg(L"kernel: %hs%s", kernel_ver, + kernel_from_iso ? L" (staged from the ISO live layer)" : L""); /* ---- Step 6: Write /etc/fstab + mount points. ---- */ { @@ -2123,16 +2413,40 @@ int do_ubuntu_to_vhdx(const wchar_t *iso_path_arg, " insmod part_gpt\n" " insmod ext2\n" " set root='hd0,gpt2'\n" - " linux /boot/vmlinuz-%s root=UUID=%s ro" + " linux /boot/vmlinuz-%s root=UUID=%s ro%s fsck.repair=yes" " " IP_EARLYCON_A "console=tty0 console=" IP_SERIAL_A ",115200\n" " initrd /boot/initrd.img-%s\n" "}\n", - kernel_ver, uuid_text, kernel_ver); + kernel_ver, uuid_text, + /* The live initrd staged by the layered-ISO fallback defaults to + BOOT=casper; boot=local makes it mount root=UUID like a normal + initramfs-tools initrd. */ + kernel_from_iso ? " boot=local" : "", + kernel_ver); ext4_writer_add_file(ew, "/boot/grub/grub.cfg", 0644, 0, 0, (uint32_t)time(NULL), boot_cfg, strlen(boot_cfg)); } + /* ---- Step 8b: keep the boot resilient after unclean stops. + update-grub in the guest regenerates grub.cfg from /etc/default/grub + + grub.d, so the bootstrap cmdline alone would lose fsck.repair=yes + on the second boot. ---- */ + { + static const char dropin[] = + "# Written by AppSandbox iso-patch. The root ext4 is created by iso-patch's\n" + "# own ext4 writer, i.e. without a journal (first boot adds one), so an\n" + "# unclean stop (Force Stop, host crash) can leave it needing a full fsck.\n" + "# Let the boot-time fsck repair it automatically instead of dropping to a\n" + "# maintenance prompt nobody can see, and do not sit at the GRUB menu\n" + "# indefinitely after such a boot.\n" + "GRUB_CMDLINE_LINUX=\"$GRUB_CMDLINE_LINUX fsck.repair=yes\"\n" + "GRUB_RECORDFAIL_TIMEOUT=5\n"; + ext4_mkdir_p(ew, "/etc/default/grub.d"); + ext4_writer_add_file(ew, "/etc/default/grub.d/91-appsandbox-resilient-boot.cfg", + 0644, 0, 0, (uint32_t)time(NULL), dropin, sizeof(dropin) - 1); + } + /* ---- Step 9: First-boot service. ---- */ plant_firstboot_service(ew); diff --git a/tools/linux/wsl-mesa/50-appsandbox-gpu b/tools/linux/wsl-mesa/50-appsandbox-gpu index cb12090..d10c571 100644 --- a/tools/linux/wsl-mesa/50-appsandbox-gpu +++ b/tools/linux/wsl-mesa/50-appsandbox-gpu @@ -20,8 +20,20 @@ WSL_MESA=/opt/wsl-mesa WSL_MESA_LIB="$WSL_MESA/lib/x86_64-linux-gnu" WSL_LIB=/usr/lib/wsl/lib +# Distro Mesa's own d3d12 gallium driver (Ubuntu builds it for WSL). Used +# when no wsl-mesa prebuilt exists for this release (only 26.04 ships one; +# firstboot removes an ABI-incompatible one). Needs libdxcore/libd3d12, +# which firstboot puts on the loader path from /opt/appsandbox/wsl-deps. +STOCK_D3D12=/usr/lib/x86_64-linux-gnu/dri/d3d12_dri.so -if [ -e /dev/dxg ] && [ -d "$WSL_MESA_LIB" ]; then +if [ -e /dev/dxg ] && [ ! -d "$WSL_MESA_LIB" ] && [ -e "$STOCK_D3D12" ]; then + echo "LD_LIBRARY_PATH=$WSL_LIB" + echo "GALLIUM_DRIVER=d3d12" + echo "MESA_LOADER_DRIVER_OVERRIDE=d3d12" + echo "__GLX_VENDOR_LIBRARY_NAME=mesa" + # No VK_DRIVER_FILES: stock mesa-vulkan-drivers has no dzn ICD on + # 24.04, so Vulkan stays on lavapipe. OpenGL apps still hit the GPU. +elif [ -e /dev/dxg ] && [ -d "$WSL_MESA_LIB" ]; then # LD_LIBRARY_PATH covers two needs: # - $WSL_MESA_LIB picks Mesa's d3d12 gallium + dzn Vulkan over stock. # - $WSL_LIB lets apps that dlopen the WSL NVIDIA userspace libs by diff --git a/tools/linux/wsl-mesa/appsandbox-gpu b/tools/linux/wsl-mesa/appsandbox-gpu index 55551db..7abd1f1 100644 --- a/tools/linux/wsl-mesa/appsandbox-gpu +++ b/tools/linux/wsl-mesa/appsandbox-gpu @@ -23,8 +23,17 @@ WSL_MESA=/opt/wsl-mesa WSL_MESA_LIB="$WSL_MESA/lib/x86_64-linux-gnu" WSL_MESA_VK_ICD="$WSL_MESA/share/vulkan/icd.d/dzn_icd.x86_64.json" WSL_LIB=/usr/lib/wsl/lib +# Distro Mesa's d3d12 driver: the fallback when this release has no +# wsl-mesa prebuilt (see 50-appsandbox-gpu). OpenGL only; Vulkan stays +# on lavapipe because stock mesa-vulkan-drivers ships no dzn ICD. +STOCK_D3D12=/usr/lib/x86_64-linux-gnu/dri/d3d12_dri.so -if [ -e /dev/dxg ] && [ -d "$WSL_MESA_LIB" ]; then +if [ -e /dev/dxg ] && [ ! -d "$WSL_MESA_LIB" ] && [ -e "$STOCK_D3D12" ]; then + export LD_LIBRARY_PATH="$WSL_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + export __GLX_VENDOR_LIBRARY_NAME=mesa + export GALLIUM_DRIVER=d3d12 + export MESA_LOADER_DRIVER_OVERRIDE=d3d12 +elif [ -e /dev/dxg ] && [ -d "$WSL_MESA_LIB" ]; then # LD_LIBRARY_PATH covers two needs: # - $WSL_MESA_LIB picks Mesa's d3d12 gallium + dzn Vulkan over stock. # - $WSL_LIB lets apps that dlopen the WSL NVIDIA userspace libs by diff --git a/web/app.js b/web/app.js index ece3a1d..0de9fab 100644 --- a/web/app.js +++ b/web/app.js @@ -649,7 +649,12 @@ function updateStatusCell(td, vm) { if (vm.buildingVhdx) { needsSpinner = true; - label = vm.vhdxStaging ? 'Staging files... ' : 'Building Disk (' + (vm.vhdxProgress || 0) + '%) '; + /* vhdxStep names the current phase ("Downloading packages 42/182", + "Building rootfs", ...); the prefetch downloads used to sit at + "Building Disk (0%)" for minutes and look hung. */ + if (vm.vhdxStaging) label = 'Staging files... '; + else if (vm.vhdxStep) label = vm.vhdxStep + ' (' + (vm.vhdxProgress || 0) + '%) '; + else label = 'Building Disk (' + (vm.vhdxProgress || 0) + '%) '; className = 'status-building'; } else if (vm.running && vm.shuttingDown) { className = 'status-shutting-down';