diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1115db4..c3bbcaa 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -29,9 +29,10 @@ concurrency: # The Linux job runs the suite natively, the way the dev container runs it # locally: Go at the dev container's version (.devcontainer/Dockerfile) plus # the runner's stock Docker, with the same TESTCONTAINERS_RYUK_DISABLED knob. -# The Windows jobs keep the compose runner image -# (tests/e2e/windows/Dockerfile-runner), which carries its own Go toolchain, -# so a Windows host needs only Docker. See docs/e2e.md. +# The Windows jobs run it natively too, with the same Go, rather than through +# the compose runner image (tests/e2e/windows/Dockerfile-runner) that lets a +# local Windows host need only Docker: on windows-11-arm, extracting Go into +# that image alone took two and a half minutes. See docs/e2e.md. jobs: e2e-linux: @@ -68,30 +69,32 @@ jobs: matrix: # TODO See https://github.com/actions/runner-images/issues/13729 runs-on: [windows-latest, windows-11-arm] - include: - - runs-on: windows-latest - compose: tests/e2e/windows/compose.amd64.yaml - - runs-on: windows-11-arm - compose: tests/e2e/windows/compose.arm64.yaml runs-on: ${{ matrix.runs-on }} timeout-minutes: 60 + env: + # Three containers at a time, as on Linux, once Docker is off C:. + E2E_MAX_JOBS: "3" steps: - uses: actions/checkout@v7 with: persist-credentials: false - # The windows-11-arm runner ships neither Docker for Windows containers - # nor a Go new enough to build moby, so both are provisioned here. - - if: matrix.runs-on == 'windows-11-arm' - uses: actions/setup-go@v6 + # C: is a network-attached disk on these runners, where extracting the + # servercore image took five minutes; Docker, Go's caches and the temp + # directory go to a local disk instead. + - run: pwsh -NoProfile -File tests/e2e/windows/use-local-disk.ps1 + - uses: actions/setup-go@v6 with: go-version: 1.27.1 cache: false + # The windows-11-arm runner ships no Docker for Windows containers, so + # the engine is built here with that Go; the build, minutes long, is + # cached on the script that pins it. - if: matrix.runs-on == 'windows-11-arm' - run: pwsh -NoProfile -File tests/e2e/windows/install-docker-arm64.ps1 + uses: actions/cache@v6 + with: + path: C:\vfox-docker\moby\bundles + key: moby-windows-arm64-${{ hashFiles('tests/e2e/windows/install-docker-arm64.ps1') }} - if: matrix.runs-on == 'windows-11-arm' - name: Install Compose CLI plugin - run: | - New-Item -ItemType Directory -Force -Path "$env:USERPROFILE\.docker\cli-plugins" | Out-Null - curl.exe -fsSL "https://github.com/docker/compose/releases/download/v5.5.1/docker-compose-windows-aarch64.exe" -o "$env:USERPROFILE\.docker\cli-plugins\docker-compose.exe" - docker compose version - - run: docker compose -f ${{ matrix.compose }} run --rm --build e2e + run: pwsh -NoProfile -File tests/e2e/windows/install-docker-arm64.ps1 + - run: go test -v -timeout 55m -run TestE2E ./windows + working-directory: tests/e2e diff --git a/docs/e2e.md b/docs/e2e.md index 342c107..a07d687 100644 --- a/docs/e2e.md +++ b/docs/e2e.md @@ -3,10 +3,10 @@ The offline Lua hook tests (`tests/*_test.lua`) run in the **Test Plugin** workflow. The end-to-end suite runs in containers against real vfox on Linux and Windows; each `vfox` x `distribution` combination gets its own throwaway -container. On Linux, CI runs the suite natively on the stock runners (Docker -plus Go from `actions/setup-go`), the way the repository dev container -reproduces it locally. On Windows it runs through a Docker Compose runner -image, since dev containers cannot host Windows container images (see below). +container. CI runs the suite natively on the stock runners (Docker plus Go +from `actions/setup-go`). Locally, the repository dev container reproduces +that on Linux; on Windows a Docker Compose runner image does, since dev +containers cannot host Windows container images (see below). ## Dev container @@ -60,11 +60,16 @@ JDK, so `java` can only come from what vfox installs: 1. vfox is installed from the requested channel — the latest release (`latest`) or built from source (`main`) — and the local `java` plugin is - registered from this checkout. + registered from this checkout. On Windows, the runner builds `main` once, + while the `latest` combinations run, and the `main` containers start from + the e2e image plus that `vfox.exe`. 2. A release no distribution has published (`.0.99`) fails to install, with `PreInstall` naming the platform it searched (`linux/amd64`, `windows/arm64`, ...), so the Foojay lookup is checked to target the - runner. + runner. So does a build of the pinned release that was never published + (`+99`, e.g. `21.0.8+99-tem`), proving the lookup keeps the + build number: Foojay reads an unencoded `+` as a space and drops the + build, and the release's newest build would install instead. 3. The pinned release installs under the name `PreInstall` gives it: the requested version plus the build number (`21.0.8-tem` installs as `21.0.8+9-tem`). `vfox search java ` (the `Available` hook) @@ -150,15 +155,26 @@ suite needs an ARM64 host (a Copilot+ PC or the `windows-11-arm` CI runner). There is no dev container for this suite — the Dev Containers spec rejects Windows container images (see [Dev container](#dev-container)) — so it keeps its Docker Compose runner. The runner image (`Dockerfile-runner`: stock -servercore plus the pinned Go MSI) drives the host daemon through the mounted -named pipe; the host requirement is Docker in Windows-container mode plus the -Compose v5 plugin (installed once per machine below). Go never touches the -host, so it cannot conflict with a vfox-managed toolchain. +servercore plus the pinned Go, which also builds vfox `main`) drives the host +daemon through the mounted named pipe; the host requirement is Docker in +Windows-container mode plus the Compose v5 plugin (installed once per machine +below). Go never touches the host, so it cannot conflict with a vfox-managed +toolchain. The `windows-11-arm` runner has no Windows Hypervisor Platform feature, so Hyper-V isolation is unavailable; process isolation needs only the Containers feature. +CI skips the runner image: extracting its Go took two and a half minutes on +`windows-11-arm`, where `actions/setup-go` already provides the same Go. It +runs `go test ./windows` on the runner with `E2E_MAX_JOBS=3`, after +`use-local-disk.ps1` moves Docker's data root, Go's caches and the temp +directory off the runners' network-attached C: onto a local disk: +`windows-latest`'s D:, or the unformatted NVMe disk of `windows-11-arm`, +which it formats. Image layers and the containers' files are written file by +file, so on C: the servercore pull alone took five minutes. The script +refuses to run outside GitHub Actions. + The `main` combos run with `VFOX_HOME` pointing inside a path containing a space (`C:\Users\John Doe\.vfox`), mirroring [vfox-flutter#33](https://github.com/version-fox/vfox-flutter/issues/33). @@ -237,7 +253,9 @@ docker compose -f tests/e2e/windows/compose.arm64.yaml run --rm --build e2e On arm64, Go is also what builds the engine, so `install-docker-arm64.ps1` runs before the suite; the compose runner image then carries its own pinned -Go for the tests, and the host Go is not used again. Scope the run down with +Go for the tests, and the host Go is not used again. The script reuses an +engine of its pinned version that it built before; CI restores one from a +cache keyed on the script. Scope the run down with `$env:VFOX_VERSION = 'latest'; $env:DISTRIBUTION = 'tem'` in front of the compose command, as on the other platforms. diff --git a/lib/foojay.lua b/lib/foojay.lua index c0c37c1..c605d7d 100644 --- a/lib/foojay.lua +++ b/lib/foojay.lua @@ -34,6 +34,16 @@ local function detect_lib_c_type() return "glibc" end +--- Percent-encodes a query string value. Foojay reads a bare + as a space and +--- drops the build after it, so 21.0.5+63130 would match every 21.0.5 build. +--- @param value string +--- @return string +local function encode(value) + return (value:gsub("[^%w%-._~]", function(c) + return ("%%%02X"):format(c:byte()) + end)) +end + foojay.fetchtJdkList = function(distribution, version) local os = RUNTIME.osType local arch = RUNTIME.archType @@ -66,7 +76,7 @@ foojay.fetchtJdkList = function(distribution, version) -- latest=available would look at the major alone. local latest = version:match("^%d+$") and "&latest=available" or "" - local reqUrl = URL:format(version, distribution, arch, archive_types, os, lib_c_type) .. latest + local reqUrl = URL:format(encode(version), distribution, arch, archive_types, os, lib_c_type) .. latest local resp, err = http.get({ url = reqUrl, diff --git a/tests/e2e/common/common.go b/tests/e2e/common/common.go index e576a62..a82dd5a 100644 --- a/tests/e2e/common/common.go +++ b/tests/e2e/common/common.go @@ -250,6 +250,15 @@ func MissingSpec(distribution, version string) string { return Spec(distribution, Major(version)+".0.99") } +// MissingBuildSpec renders a build of the release that no distribution has +// published (+99). Foojay answers it with an empty package list +// only when the + reaches it encoded: read as a space, it drops the build +// and lists every build of the release, so PreInstall installed the newest +// (vfox-java#73). +func MissingBuildSpec(distribution, version string) string { + return Spec(distribution, version+"+99") +} + // Major returns the leading version component: "21.0.8" -> "21". func Major(version string) string { return strings.SplitN(version, ".", 2)[0] diff --git a/tests/e2e/common/common_test.go b/tests/e2e/common/common_test.go index b7a7296..f79af10 100644 --- a/tests/e2e/common/common_test.go +++ b/tests/e2e/common/common_test.go @@ -184,11 +184,11 @@ func TestRetryAttemptsDefault(t *testing.T) { } func TestSpecs(t *testing.T) { - for _, c := range []struct{ dist, version, spec, missing, legacy string }{ - {"open", "25.0.2", "25.0.2", "25.0.99", "25.0.2-open"}, - {"tem", "21.0.8", "21.0.8-tem", "21.0.99-tem", "21.0.8-tem"}, - {"librca-fx", "25.0.4", "25.0.4-librca-fx", "25.0.99-librca-fx", "25.0.4.fx-librca"}, - {"jb", "21.0.5", "21.0.5-jb", "21.0.99-jb", "21.0.5-jbr"}, + for _, c := range []struct{ dist, version, spec, missing, missingBuild, legacy string }{ + {"open", "25.0.2", "25.0.2", "25.0.99", "25.0.2+99", "25.0.2-open"}, + {"tem", "21.0.8", "21.0.8-tem", "21.0.99-tem", "21.0.8+99-tem", "21.0.8-tem"}, + {"librca-fx", "25.0.4", "25.0.4-librca-fx", "25.0.99-librca-fx", "25.0.4+99-librca-fx", "25.0.4.fx-librca"}, + {"jb", "21.0.5", "21.0.5-jb", "21.0.99-jb", "21.0.5+99-jb", "21.0.5-jbr"}, } { if got := Spec(c.dist, c.version); got != c.spec { t.Errorf("Spec(%s, %s) = %q, want %q", c.dist, c.version, got, c.spec) @@ -196,6 +196,9 @@ func TestSpecs(t *testing.T) { if got := MissingSpec(c.dist, c.version); got != c.missing { t.Errorf("MissingSpec(%s, %s) = %q, want %q", c.dist, c.version, got, c.missing) } + if got := MissingBuildSpec(c.dist, c.version); got != c.missingBuild { + t.Errorf("MissingBuildSpec(%s, %s) = %q, want %q", c.dist, c.version, got, c.missingBuild) + } if got := LegacyIdentifier(c.dist, c.version); got != c.legacy { t.Errorf("LegacyIdentifier(%s, %s) = %q, want %q", c.dist, c.version, got, c.legacy) } diff --git a/tests/e2e/linux/setup.go b/tests/e2e/linux/setup.go index 832b39e..dcbee3e 100644 --- a/tests/e2e/linux/setup.go +++ b/tests/e2e/linux/setup.go @@ -51,26 +51,27 @@ func setupVfox(ctx context.Context, t *testing.T, ctr tc.Container, vfoxVersion t.Logf("PASS %s with the local java plugin", strings.TrimSpace(execOK(ctx, t, ctr, "vfox --version"))) } -// checkMissingVersionRejected proves an unpublished release fails loudly -// instead of installing something else: PreInstall finds no Foojay package -// and names the platform it searched, which must be the runner's -// (RUNTIME.osType/archType). +// checkMissingVersionRejected proves an unpublished release, and an +// unpublished build of the pinned one, fail loudly instead of installing +// something else: PreInstall finds no Foojay package and names the platform +// it searched, which must be the runner's (RUNTIME.osType/archType). func checkMissingVersionRejected(ctx context.Context, t *testing.T, ctr tc.Container, distribution, version, arch string) { t.Helper() - spec := common.MissingSpec(distribution, version) - want := "No JDK found for " + spec - out := common.Retry(ctx, t, "vfox install java@"+spec+" (must fail)", func() (int, string) { - code, out := execRun(ctx, t, ctr, "vfox install java@"+spec) - if code == 0 { - t.Fatalf("installing the unpublished %s unexpectedly succeeded\n--- output ---\n%s", spec, out) - } - // Any other failure (e.g. Foojay unreachable) is retried like the - // other network-touching calls. - if strings.Contains(out, want) { - return 0, out - } - return code, out - }) - require.Contains(t, out, " on linux/"+arch, "the lookup must target the runner platform") - t.Logf("PASS java@%s is rejected on linux/%s", spec, arch) + for _, spec := range []string{common.MissingSpec(distribution, version), common.MissingBuildSpec(distribution, version)} { + want := "No JDK found for " + spec + out := common.Retry(ctx, t, "vfox install java@"+spec+" (must fail)", func() (int, string) { + code, out := execRun(ctx, t, ctr, "vfox install java@"+spec) + if code == 0 { + t.Fatalf("installing the unpublished %s unexpectedly succeeded\n--- output ---\n%s", spec, out) + } + // Any other failure (e.g. Foojay unreachable) is retried like the + // other network-touching calls. + if strings.Contains(out, want) { + return 0, out + } + return code, out + }) + require.Contains(t, out, " on linux/"+arch, "the lookup must target the runner platform") + t.Logf("PASS java@%s is rejected on linux/%s", spec, arch) + } } diff --git a/tests/e2e/windows/Dockerfile-runner b/tests/e2e/windows/Dockerfile-runner index 8bacead..aeb3fb1 100644 --- a/tests/e2e/windows/Dockerfile-runner +++ b/tests/e2e/windows/Dockerfile-runner @@ -17,6 +17,7 @@ # pinned Go toolchain, so no host Go is needed on any arch. (There is no # stock golang Windows image for arm64, hence this file instead of pinning # one.) Built via compose --build (see compose.amd64.yaml / compose.arm64.yaml). +# Its Go also builds vfox from main for the suite's main combos. ARG BASE_IMAGE=mcr.microsoft.com/windows/servercore:ltsc2025-KB5122871-amd64 FROM ${BASE_IMAGE} @@ -26,12 +27,15 @@ ARG GO_VERSION=1.27.1 # NB: no double quotes or backticks below. The classic builder wraps the # joined lines in powershell -Command "..." through cmd.exe, where both # break parsing. Single quotes, ${var} braces and array arguments only. +# A zip, not the MSI, which took over five minutes to install on arm64; the +# zip only extracts, and leaves the machine PATH to us. RUN $goArch = if ($env:PROCESSOR_ARCHITECTURE -eq 'ARM64') { 'arm64' } else { 'amd64' }; ` - $goMsi = $env:TEMP + '\go.msi'; ` - curl.exe -fsSL --retry 3 --retry-delay 5 --retry-all-errors -o $goMsi https://go.dev/dl/go${env:GO_VERSION}.windows-${goArch}.msi; ` - $goInstall = Start-Process msiexec.exe -Wait -PassThru -ArgumentList '/i', $goMsi, '/quiet', '/norestart'; ` - if ($goInstall.ExitCode -notin 0, 3010, 1937) { throw 'FAIL the Go MSI install exited with code ' + $goInstall.ExitCode }; ` - Remove-Item $goMsi; ` + $goZip = $env:TEMP + '\go.zip'; ` + curl.exe -fsSL --retry 3 --retry-delay 5 --retry-all-errors -o $goZip https://go.dev/dl/go${env:GO_VERSION}.windows-${goArch}.zip; ` + tar.exe -xf $goZip -C 'C:\Program Files'; ` + if ($LASTEXITCODE -ne 0) { throw 'FAIL the Go zip extraction exited with code ' + $LASTEXITCODE }; ` + Remove-Item $goZip; ` + [Environment]::SetEnvironmentVariable('PATH', [Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';C:\Program Files\Go\bin', 'Machine'); ` New-Item -ItemType Directory -Force -Path 'C:\m', 'C:\b' | Out-Null; ` & 'C:\Program Files\Go\bin\go.exe' version diff --git a/tests/e2e/windows/bootstrap-tools.ps1 b/tests/e2e/windows/bootstrap-tools.ps1 index 7d04931..ae64dac 100644 --- a/tests/e2e/windows/bootstrap-tools.ps1 +++ b/tests/e2e/windows/bootstrap-tools.ps1 @@ -32,22 +32,19 @@ function Invoke-Native { if ($code -notin $Ok) { throw "FAIL $What exited with code $code" } } -$IsArm64 = $env:PROCESSOR_ARCHITECTURE -eq 'ARM64' -$pwshArch = if ($IsArm64) { 'arm64' } else { 'x64' } -$gitAsset = if ($IsArm64) { 'Git-2.55.0.5-arm64.exe' } else { 'Git-2.55.0.5-64-bit.exe' } - -$pwshInstaller = "$env:TEMP\pwsh.msi" -$pwshUrl = "https://github.com/PowerShell/PowerShell/releases/download/v7.6.6/PowerShell-7.6.6-win-$pwshArch.msi" -$gitInstaller = "$env:TEMP\git-installer.exe" -$gitUrl = "https://github.com/git-for-windows/git/releases/download/v2.55.0.windows.5/$gitAsset" - -Invoke-Native { & curl.exe -fsSL --retry 3 --retry-delay 5 --retry-all-errors -o $pwshInstaller $pwshUrl } 'the PowerShell download' -Invoke-Native { & curl.exe -fsSL --retry 3 --retry-delay 5 --retry-all-errors -o $gitInstaller $gitUrl } 'the Git download' -$pwshSetup = Start-Process msiexec.exe -Wait -PassThru -ArgumentList '/i', "`"$pwshInstaller`"", '/quiet', '/norestart' -if ($pwshSetup.ExitCode -notin 0, 3010) { throw "FAIL the PowerShell MSI install exited with code $($pwshSetup.ExitCode)" } -$gitSetup = Start-Process $gitInstaller -Wait -PassThru -ArgumentList '/VERYSILENT', '/NORESTART', '/SP-', '/SUPPRESSMSGBOXES' -if ($gitSetup.ExitCode -ne 0) { throw "FAIL the Git installer exited with code $($gitSetup.ExitCode)" } -if (-not (Test-Path 'C:\Program Files\PowerShell\7\pwsh.exe')) { throw 'FAIL pwsh.exe is missing after the PowerShell MSI install' } -if (-not (Test-Path 'C:\Program Files\Git\cmd\git.exe')) { throw 'FAIL git.exe is missing after the Git installer' } -Invoke-Native { & 'C:\Program Files\Git\cmd\git.exe' config --system core.longpaths true } 'enable system-wide git long paths' +$pwshArch = if ($env:PROCESSOR_ARCHITECTURE -eq 'ARM64') { 'arm64' } else { 'x64' } + +# A zip, not the MSI: installers took minutes here on arm64, while the zip +# only extracts, and leaves the machine PATH to us. No Git: vfox from main +# is built on the runner (setup.go), and nothing in the containers clones. +$pwshZip = "$env:TEMP\pwsh.zip" +$pwshUrl = "https://github.com/PowerShell/PowerShell/releases/download/v7.6.6/PowerShell-7.6.6-win-$pwshArch.zip" +$pwshHome = 'C:\Program Files\PowerShell\7' + +Invoke-Native { & curl.exe -fsSL --retry 3 --retry-delay 5 --retry-all-errors -o $pwshZip $pwshUrl } 'the PowerShell download' +New-Item -ItemType Directory -Force -Path $pwshHome | Out-Null +Invoke-Native { & tar.exe -xf $pwshZip -C $pwshHome } 'the PowerShell extraction' +Remove-Item $pwshZip +if (-not (Test-Path "$pwshHome\pwsh.exe")) { throw 'FAIL pwsh.exe is missing after the PowerShell extraction' } +[Environment]::SetEnvironmentVariable('PATH', [Environment]::GetEnvironmentVariable('PATH', 'Machine') + ";$pwshHome", 'Machine') New-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -Value 1 -PropertyType DWORD -Force | Out-Null diff --git a/tests/e2e/windows/compose.amd64.yaml b/tests/e2e/windows/compose.amd64.yaml index 4439dcc..11ecfb3 100644 --- a/tests/e2e/windows/compose.amd64.yaml +++ b/tests/e2e/windows/compose.amd64.yaml @@ -20,8 +20,8 @@ # The host requirement is Docker plus the Compose v5 plugin (installed once # per machine, see docs/e2e.md); Go never touches the host, so it cannot # conflict with a vfox-managed toolchain. The runner image -# (Dockerfile-runner: stock servercore plus the pinned Go MSI) is built for -# the host arch, because there is no stock golang Windows image for arm64. +# (Dockerfile-runner: stock servercore plus the pinned Go) is built for the +# host arch, because there is no stock golang Windows image for arm64. # It drives the host daemon through the mounted named pipe. Ryuk is disabled # in code (it has no Windows image); every container is terminated explicitly # via t.Cleanup instead. diff --git a/tests/e2e/windows/compose.arm64.yaml b/tests/e2e/windows/compose.arm64.yaml index 1a0901a..add9525 100644 --- a/tests/e2e/windows/compose.arm64.yaml +++ b/tests/e2e/windows/compose.arm64.yaml @@ -20,8 +20,8 @@ # The host requirement is Docker plus the Compose v5 plugin (installed once # per machine, see docs/e2e.md); Go never touches the host, so it cannot # conflict with a vfox-managed toolchain. The runner image -# (Dockerfile-runner: stock servercore plus the pinned Go MSI) is built for -# the host arch, because there is no stock golang Windows image for arm64. +# (Dockerfile-runner: stock servercore plus the pinned Go) is built for the +# host arch, because there is no stock golang Windows image for arm64. # It drives the host daemon through the mounted named pipe. Ryuk is disabled # in code (it has no Windows image); every container is terminated explicitly # via t.Cleanup instead. diff --git a/tests/e2e/windows/e2e_test.go b/tests/e2e/windows/e2e_test.go index 4b27ace..8fdee18 100644 --- a/tests/e2e/windows/e2e_test.go +++ b/tests/e2e/windows/e2e_test.go @@ -23,9 +23,13 @@ package windows import ( + "archive/tar" + "bytes" "context" + "fmt" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -66,9 +70,6 @@ func TestE2E(t *testing.T) { } repoRoot := filepath.Join(here, "..", "..", "..") - ctx := context.Background() - buildImage(t, ctx, repoRoot, image, platform, baseImage) - combos, err := common.Matrix() if err != nil { t.Fatal(err) @@ -77,11 +78,24 @@ func TestE2E(t *testing.T) { t.Fatal("empty matrix: nothing to run") } + ctx := context.Background() + // The main combos' vfox compiles while the image builds. + var vfoxMain *vfoxMainImage + if slices.ContainsFunc(combos, func(c common.Combo) bool { return c.Vfox == "main" }) { + vfoxMain = startVfoxMain(ctx, t, image, arch) + } + buildImage(t, ctx, repoRoot, image, platform, baseImage) + sem := make(chan struct{}, maxJobs()) for _, c := range combos { c := c t.Run(c.Slug(), func(t *testing.T) { t.Parallel() + image := image + if c.Vfox == "main" { + // Before taking a slot, so the latest combos run meanwhile. + image = vfoxMain.get(ctx, t) + } sem <- struct{}{} defer func() { <-sem }() runOne(t, ctx, image, platform, arch, c) @@ -129,6 +143,51 @@ func buildImage(t *testing.T, ctx context.Context, repoRoot, image, platform, ba } } +// buildMainImage layers vfox.exe onto base as -main, for the main +// combos. Its build context is that file and the Dockerfile alone, +// assembled in memory. +func buildMainImage(ctx context.Context, base, exe string) (string, error) { + vfox, err := os.ReadFile(exe) + if err != nil { + return "", err + } + var buildContext bytes.Buffer + tw := tar.NewWriter(&buildContext) + for _, f := range []struct { + name string + data []byte + }{ + {"Dockerfile", []byte("FROM " + base + "\nCOPY vfox.exe " + mainVfoxExe + "\n")}, + {"vfox.exe", vfox}, + } { + if err := tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0o644, Size: int64(len(f.data))}); err != nil { + return "", err + } + if _, err := tw.Write(f.data); err != nil { + return "", err + } + } + if err := tw.Close(); err != nil { + return "", err + } + + provider, err := tc.NewDockerProvider() + if err != nil { + return "", fmt.Errorf("new docker provider: %w", err) + } + defer provider.Close() + // No PullParent: base exists only on this daemon. + return provider.BuildImage(ctx, &tc.ContainerRequest{ + FromDockerfile: tc.FromDockerfile{ + ContextArchive: bytes.NewReader(buildContext.Bytes()), + Repo: "vfox-java-e2e", + Tag: base[strings.LastIndex(base, ":")+1:] + "-main", + KeepImage: true, + BuildLogWriter: os.Stdout, + }, + }) +} + // runOne starts one long-lived container per combo and drives the whole // E2E flow (setup -> install -> verify) through pwsh exec calls, asserting // in Go. A combo passes when every phase exits 0 and the outputs carry the diff --git a/tests/e2e/windows/install-docker-arm64.ps1 b/tests/e2e/windows/install-docker-arm64.ps1 index 6523c35..83257bf 100644 --- a/tests/e2e/windows/install-docker-arm64.ps1 +++ b/tests/e2e/windows/install-docker-arm64.ps1 @@ -50,6 +50,17 @@ function Wait-Engine { return $false } +# Whether $Bin holds a whole engine build of $Version: one from an earlier +# run, or the one CI restores from its cache (keyed on this file). +function Test-Build { + param([string] $Bin, [string] $Version) + $dockerd = Join-Path $Bin 'dockerd.exe' + if (-not ((Test-Path $dockerd) -and (Test-Path (Join-Path $Bin 'docker.exe')))) { return $false } + $PSNativeCommandUseErrorActionPreference = $false + $banner = (& $dockerd --version 2>&1) -join '' + return $banner -like "Docker version $Version,*" +} + if ($env:PROCESSOR_ARCHITECTURE -ne 'ARM64') { throw "FAIL this script builds a windows/arm64 engine and needs an arm64 host" } if (Test-Engine) { @@ -59,31 +70,41 @@ else { $mobyVersion = '29.8.1' $root = 'C:\vfox-docker' New-Item -ItemType Directory -Force -Path $root | Out-Null - Write-Output 'Building docker from source for windows/arm64 ...' - - if (-not (Get-Command go -ErrorAction SilentlyContinue)) { throw 'FAIL go is not on the PATH' } - $env:GOPATH = Join-Path $root 'gopath' - $moby = Join-Path $root 'moby' - Invoke-Native { git clone --depth 1 --branch "docker-v$mobyVersion" https://github.com/moby/moby.git $moby } 'the moby clone' - Invoke-Native { go install github.com/tc-hib/go-winres@v0.3.1 } 'the go-winres install' - $env:DOCKERCLI_VERSION = $mobyVersion - $env:VERSION = $mobyVersion - Push-Location $moby - try { - & .\hack\make.ps1 -Client -Daemon + $bin = Join-Path $moby 'bundles' + + if (Test-Build $bin $mobyVersion) { + Write-Output "Using the docker $mobyVersion build in $bin" } - finally { - Pop-Location + else { + Write-Output 'Building docker from source for windows/arm64 ...' + + if (-not (Get-Command go -ErrorAction SilentlyContinue)) { throw 'FAIL go is not on the PATH' } + $env:GOPATH = Join-Path $root 'gopath' + + # A partial or older build gives way to a fresh clone. + if (Test-Path $moby) { Remove-Item -Recurse -Force $moby } + Invoke-Native { git clone --depth 1 --branch "docker-v$mobyVersion" https://github.com/moby/moby.git $moby } 'the moby clone' + Invoke-Native { go install github.com/tc-hib/go-winres@v0.3.1 } 'the go-winres install' + $env:DOCKERCLI_VERSION = $mobyVersion + $env:VERSION = $mobyVersion + Push-Location $moby + try { + & .\hack\make.ps1 -Client -Daemon + } + finally { + Pop-Location + } } - $bin = Join-Path $moby 'bundles' if (-not (Test-Path (Join-Path $bin 'dockerd.exe'))) { throw 'FAIL the moby build produced no dockerd.exe' } if (-not (Test-Path (Join-Path $bin 'docker.exe'))) { throw 'FAIL the moby build produced no docker.exe' } $env:PATH = "$bin;$env:PATH" if ($env:GITHUB_PATH) { Add-Content -Path $env:GITHUB_PATH -Value $bin } - $dataRoot = Join-Path $root 'data' + # CI points E2E_DOCKER_DATA_ROOT at the runner's local disk + # (use-local-disk.ps1). + $dataRoot = if ($env:E2E_DOCKER_DATA_ROOT) { $env:E2E_DOCKER_DATA_ROOT } else { Join-Path $root 'data' } $log = Join-Path $root 'dockerd.log' $dockerdArgs = "--data-root `"$dataRoot`" --debug --exec-opt isolation=process" Start-Process -FilePath (Join-Path $bin 'dockerd.exe') -ArgumentList $dockerdArgs -RedirectStandardOutput $log -RedirectStandardError (Join-Path $root 'dockerd.err.log') diff --git a/tests/e2e/windows/setup.go b/tests/e2e/windows/setup.go index 7650f1a..ee19f93 100644 --- a/tests/e2e/windows/setup.go +++ b/tests/e2e/windows/setup.go @@ -17,7 +17,12 @@ package windows import ( "context" "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" "strings" + "sync" "testing" "github.com/stretchr/testify/require" @@ -26,17 +31,20 @@ import ( "github.com/version-fox/vfox-java/tests/e2e/common" ) -// Java defaults live in the common package; only the Windows-specific -// toolchain pin stays here. +// Java defaults live in the common package; only Windows-specific values +// stay here. const ( - // goWindowsVersion pins the Go toolchain MSI used to build vfox@main - // inside the container. - goWindowsVersion = "1.27.1" - // spacedVfoxHome is the default vfox home of a Windows account whose // name contains a space, the layout reported in // https://github.com/version-fox/vfox-flutter/issues/33 . spacedVfoxHome = `C:\Users\John Doe\.vfox` + + // vfoxMainSource is vfox's main branch, built once per run for every + // main combo (see vfoxMainImage). + vfoxMainSource = "https://github.com/version-fox/vfox/archive/refs/heads/main.tar.gz" + + // mainVfoxExe is where the main combos' image carries that build. + mainVfoxExe = "C:/vfox-main/vfox.exe" ) // pwshQuote renders a single-quoted PowerShell string literal. @@ -131,49 +139,99 @@ func installVfoxRelease(ctx context.Context, t *testing.T, ctr tc.Container, box t.Logf("PASS installed vfox %s (%s)", tag, vfoxArch) } -// installVfoxMain installs the pinned Go MSI, clones vfox, and builds -// vfox.exe. msiexec exit codes 0/3010/1937 all mean success (ok / -// reboot-required / in-use-files). -func installVfoxMain(ctx context.Context, t *testing.T, ctr tc.Container, box slotBox, procArch string) { +// vfoxMainImage is the e2e image plus vfox.exe built from main, which every +// main combo starts from. The runner compiles vfox once, with its own Go, +// while the image builds; building it in each main combo's container meant +// installing the Go MSI there, then cloning and building, over two minutes a +// combo on arm64. +type vfoxMainImage struct { + base, arch string + + built chan struct{} // closed once exe and err are set + exe string + once sync.Once + image string + err error +} + +// startVfoxMain starts compiling vfox from main in the background, for the +// image that layers it onto base. +func startVfoxMain(ctx context.Context, t *testing.T, base, arch string) *vfoxMainImage { t.Helper() - goArch := "amd64" - if procArch == "ARM64" { - goArch = "arm64" + m := &vfoxMainImage{base: base, arch: arch, built: make(chan struct{})} + dir := t.TempDir() + ctx, cancel := context.WithCancel(ctx) + go func() { + defer close(m.built) + m.exe, m.err = buildVfoxMain(ctx, dir, arch) + }() + // Before dir is removed: a run that fails early stops the build. + t.Cleanup(func() { + cancel() + <-m.built + }) + return m +} + +// get returns the image, layering the compiled vfox.exe onto base on first +// use; the other main combos wait for it. A failed build fails every main +// combo. +func (m *vfoxMainImage) get(ctx context.Context, t *testing.T) string { + t.Helper() + m.once.Do(func() { + <-m.built + if m.err == nil { + m.image, m.err = buildMainImage(ctx, m.base, m.exe) + } + if m.err == nil { + t.Logf("PASS built vfox from main with %s (%s)", runtime.Version(), m.arch) + } + }) + if m.err != nil { + t.Fatalf("build vfox from main: %v", m.err) } - goMsi := box.tmp + `\go.msi` - execOK(ctx, t, ctr, fmt.Sprintf( - "curl.exe %s -fsSL -o %s %s", - curlRetry, pwshQuote(goMsi), - pwshQuote(fmt.Sprintf("https://go.dev/dl/go%s.windows-%s.msi", goWindowsVersion, goArch))), - box.vars...) - execOK(ctx, t, ctr, fmt.Sprintf(`$goInstall = $null -for ($attempt = 1; $attempt -le 5; $attempt++) { - $goInstall = Start-Process msiexec.exe -Wait -PassThru -ArgumentList '/i', %s, '/quiet', '/norestart' - if ($goInstall.ExitCode -in 0, 3010, 1937) { break } - Start-Sleep -Seconds 15 + return m.image } -if ($goInstall.ExitCode -notin 0, 3010, 1937) { throw ('FAIL the Go MSI install exited with code ' + $goInstall.ExitCode) } -Remove-Item %s`, pwshQuote(goMsi), pwshQuote(goMsi)), box.vars...) - src := box.workDir + `\vfox-src` - execOK(ctx, t, ctr, - fmt.Sprintf("git clone --depth 1 https://github.com/version-fox/vfox %s", pwshQuote(src)), - box.vars...) - execOK(ctx, t, ctr, fmt.Sprintf( - `& 'C:\Program Files\Go\bin\go.exe' -C %s build -trimpath -o %s .`, - pwshQuote(src), pwshQuote(box.vfoxExe)), box.vars...) - t.Logf("PASS built vfox from main with Go %s (%s)", goWindowsVersion, goArch) +// buildVfoxMain downloads vfox's main branch into dir and builds vfox.exe +// there for arch with the runner's Go, returning its path. It runs +// Windows' own curl and tar: CI runners also put Git's GNU tar on the PATH, +// which reads the colon of a drive letter as a remote host. +func buildVfoxMain(ctx context.Context, dir, arch string) (string, error) { + tarball := filepath.Join(dir, "vfox-main.tar.gz") + exe := filepath.Join(dir, "vfox.exe") + system32 := filepath.Join(os.Getenv("SystemRoot"), "System32") + download := exec.CommandContext(ctx, filepath.Join(system32, "curl.exe"), + append(strings.Fields(curlRetry), "-fsSL", "-o", tarball, vfoxMainSource)...) + extract := exec.CommandContext(ctx, filepath.Join(system32, "tar.exe"), "-xzf", tarball, "-C", dir) + build := exec.CommandContext(ctx, "go", "build", "-trimpath", "-o", exe, ".") + build.Dir = filepath.Join(dir, "vfox-main") + build.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=windows", "GOARCH="+arch) + for _, cmd := range []*exec.Cmd{download, extract, build} { + if out, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("%s: %w\n%s", strings.Join(cmd.Args, " "), err, out) + } + } + return exe, nil } -// setupVfox installs vfox (release, or builds main) and registers the local -// java plugin. +// installVfoxMain stages the vfox.exe built from main that the main combos' +// image carries (see vfoxMainImage). +func installVfoxMain(ctx context.Context, t *testing.T, ctr tc.Container, box slotBox) { + t.Helper() + execOK(ctx, t, ctr, fmt.Sprintf("Copy-Item -Path %s -Destination %s", + pwshQuote(mainVfoxExe), pwshQuote(box.vfoxExe)), box.vars...) + t.Logf("PASS staged vfox built from main") +} + +// setupVfox installs vfox (release, or the build from main) and registers +// the local java plugin. func setupVfox(ctx context.Context, t *testing.T, ctr tc.Container, box slotBox, vfoxVersion string) { t.Helper() - procArch := procArchOfContainer(ctx, t, ctr) if vfoxVersion == "main" { - installVfoxMain(ctx, t, ctr, box, procArch) + installVfoxMain(ctx, t, ctr, box) } else { - installVfoxRelease(ctx, t, ctr, box, procArch) + installVfoxRelease(ctx, t, ctr, box, procArchOfContainer(ctx, t, ctr)) } execOK(ctx, t, ctr, fmt.Sprintf( "tar -a -cf %s -C 'C:\\e2e' metadata.lua hooks lib", @@ -210,26 +268,28 @@ func activatedEnv(activation, script string) string { "if ($__vfoxExport.Trim()) { Invoke-Expression -Command $__vfoxExport }\r\n" + script } -// checkMissingVersionRejected proves an unpublished release fails loudly -// instead of installing something else: PreInstall finds no Foojay package -// (zip or tar.gz) and names the platform it searched, which must be the -// runner's (RUNTIME.osType/archType). +// checkMissingVersionRejected proves an unpublished release, and an +// unpublished build of the pinned one, fail loudly instead of installing +// something else: PreInstall finds no Foojay package (zip or tar.gz) and +// names the platform it searched, which must be the runner's +// (RUNTIME.osType/archType). func checkMissingVersionRejected(ctx context.Context, t *testing.T, ctr tc.Container, box slotBox, distribution, version, arch string) { t.Helper() - spec := common.MissingSpec(distribution, version) - want := "No JDK found for " + spec - out := common.Retry(ctx, t, "vfox install java@"+spec+" (must fail)", func() (int, string) { - code, out := execRun(ctx, t, ctr, "vfox install "+pwshQuote("java@"+spec), box.vars...) - if code == 0 { - t.Fatalf("installing the unpublished %s unexpectedly succeeded\n--- output ---\n%s", spec, out) - } - // Any other failure (e.g. Foojay unreachable) is retried like the - // other network-touching calls. - if strings.Contains(out, want) { - return 0, out - } - return code, out - }) - require.Contains(t, out, " on windows/"+arch, "the lookup must target the runner platform") - t.Logf("PASS java@%s is rejected on windows/%s", spec, arch) + for _, spec := range []string{common.MissingSpec(distribution, version), common.MissingBuildSpec(distribution, version)} { + want := "No JDK found for " + spec + out := common.Retry(ctx, t, "vfox install java@"+spec+" (must fail)", func() (int, string) { + code, out := execRun(ctx, t, ctr, "vfox install "+pwshQuote("java@"+spec), box.vars...) + if code == 0 { + t.Fatalf("installing the unpublished %s unexpectedly succeeded\n--- output ---\n%s", spec, out) + } + // Any other failure (e.g. Foojay unreachable) is retried like the + // other network-touching calls. + if strings.Contains(out, want) { + return 0, out + } + return code, out + }) + require.Contains(t, out, " on windows/"+arch, "the lookup must target the runner platform") + t.Logf("PASS java@%s is rejected on windows/%s", spec, arch) + } } diff --git a/tests/e2e/windows/use-local-disk.ps1 b/tests/e2e/windows/use-local-disk.ps1 new file mode 100644 index 0000000..9c12d57 --- /dev/null +++ b/tests/e2e/windows/use-local-disk.ps1 @@ -0,0 +1,84 @@ +<# + Copyright 2026 Han Li and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#> + +# CI only: moves Docker's data root off C:, which on GitHub-hosted Windows +# runners is a network-attached OS disk where every file flush takes +# milliseconds, onto the runner's local disk. Extracting image layers and +# whatever the e2e containers write flush file by file, so on C: the +# servercore pull alone took five minutes. Go's caches and the temp +# directory follow, for the suite and the vfox build it runs on the runner. +# windows-latest has a local D:; windows-11-arm carries an unformatted local +# NVMe disk, which this formats. A runner with neither keeps all on C:. + +$ErrorActionPreference = 'Stop' +$PSNativeCommandUseErrorActionPreference = $true + +if (-not $env:GITHUB_ACTIONS) { throw 'FAIL this script formats a raw disk; it is for CI runners only' } + +Get-PhysicalDisk | Format-Table -AutoSize DeviceId, FriendlyName, MediaType, BusType, Size | Out-String | Write-Output +Get-Volume | Where-Object DriveLetter | Format-Table -AutoSize DriveLetter, FileSystem, Size, SizeRemaining | Out-String | Write-Output + +$raw = Get-Disk | Where-Object PartitionStyle -eq 'RAW' | Sort-Object Size -Descending | Select-Object -First 1 +# A D: on the disk C: is on would be no faster. +$dDisk = (Get-Partition -DriveLetter D -ErrorAction SilentlyContinue).DiskNumber +if ($raw) { + $volume = Initialize-Disk -Number $raw.Number -PartitionStyle GPT -PassThru | + New-Partition -UseMaximumSize -AssignDriveLetter | + Format-Volume -FileSystem NTFS -NewFileSystemLabel docker -Confirm:$false -Force + $drive = "$($volume.DriveLetter):" +} +elseif ($null -ne $dDisk -and $dDisk -ne (Get-Partition -DriveLetter C).DiskNumber) { + $drive = 'D:' +} +else { + Write-Output 'PASS no local disk here; Docker, Go and the temp directory stay on C:' + exit 0 +} +$dataRoot = "$drive\docker" +$go = "$drive\go" +$temp = "$drive\temp" +New-Item -ItemType Directory -Force -Path $dataRoot, "$go\mod", "$go\build", $temp | Out-Null +"GOMODCACHE=$go\mod", "GOCACHE=$go\build", "TEMP=$temp", "TMP=$temp" | Add-Content -Path $env:GITHUB_ENV + +if (Get-Service docker -ErrorAction SilentlyContinue) { + # The engine windows-latest preinstalls: point its config at the local + # disk and restart it. It holds no images yet, so nothing has to move. + $config = 'C:\ProgramData\docker\config\daemon.json' + $settings = $null + if (Test-Path $config) { $settings = Get-Content $config -Raw | ConvertFrom-Json } + if (-not $settings) { $settings = [pscustomobject]@{} } + $settings | Add-Member -NotePropertyName 'data-root' -NotePropertyValue $dataRoot -Force + Stop-Service docker + New-Item -ItemType Directory -Force -Path (Split-Path $config) | Out-Null + $settings | ConvertTo-Json -Depth 8 | Set-Content -Path $config + Start-Service docker + + $PSNativeCommandUseErrorActionPreference = $false + $root = '' + for ($attempt = 1; $attempt -le 60; $attempt++) { + $root = (& docker info --format '{{.DockerRootDir}}' 2>$null) -join '' + if ($LASTEXITCODE -eq 0) { break } + Start-Sleep -Seconds 2 + } + $PSNativeCommandUseErrorActionPreference = $true + if ($root -ne $dataRoot) { throw "FAIL the docker engine keeps its data in '$root', not $dataRoot" } +} +else { + # windows-11-arm has no engine yet; install-docker-arm64.ps1 starts one + # with this data root. + "E2E_DOCKER_DATA_ROOT=$dataRoot" | Add-Content -Path $env:GITHUB_ENV +} +Write-Output "PASS Docker keeps its data on $dataRoot, Go its caches in $go, and the temp directory is $temp" diff --git a/tests/pre_install_test.lua b/tests/pre_install_test.lua index 29b6fc8..8844fe9 100644 --- a/tests/pre_install_test.lua +++ b/tests/pre_install_test.lua @@ -43,7 +43,11 @@ local foojay = { ["openjdk 21"] = { pkg("21+35") }, ["openjdk 21 available"] = { pkg("21.0.2+13") }, ["openjdk 25.0.2"] = { pkg("25.0.2+10") }, - ["openjdk 28-ea"] = { pkg("28-ea+19"), pkg("28-ea+18") }, + ["openjdk 28-ea"] = { pkg("28-ea+19"), pkg("28-ea+18"), pkg("28-ea+15") }, + ["openjdk 28-ea+15"] = { pkg("28-ea+15") }, + -- JetBrains Runtime has nine 21.0.5 builds; abridged. + ["jetbrains 21.0.5"] = { pkg("21.0.5+79248"), pkg("21.0.5+76240"), pkg("21.0.5+63130"), pkg("21.0.5+6318") }, + ["jetbrains 21.0.5+63130"] = { pkg("21.0.5+63130") }, -- Searching asks for no version; abridged. ["openjdk"] = { pkg("25.0.2+10"), pkg("21.0.2+13") }, ["temurin"] = { pkg("25.0.2+10"), pkg("21.0.8+9") }, @@ -72,14 +76,20 @@ package.preload.http = function() local params = {} local archive_types = {} for key, value in request.url:sub(#JDKS + 1):gmatch("([^&=]+)=([^&]*)") do + -- Foojay decodes the query string, reading a bare + as a space. + value = value:gsub("%+", " "):gsub("%%(%x%x)", function(hex) + return string.char(tonumber(hex, 16)) + end) params[key] = value if key == "archive_type" then archive_types[value] = true end end + -- A version stops at a space: 21.0.5 63130 matches every 21.0.5 build. + local version = params.version:match("^[^ ]*") local key = params.distribution - if params.version ~= "" then - key = key .. " " .. params.version + if version ~= "" then + key = key .. " " .. version end if params.latest then key = key .. " " .. params.latest @@ -151,6 +161,9 @@ check("17-graal", "17.0.12-graal", true) -- Anything longer than a major still matches exactly. check("21.0.8-tem", "21.0.8+9-tem", false) check("28.ea", "28-ea+19", false) +-- A build number installs that build, not the newest one of its release. +check("28.ea.15", "28-ea+15", false) +check("21.0.5+63130-jb", "21.0.5+63130-jb", false) -- OpenJDK versions carry no suffix, however the input spells the distribution. check("25.0.2-open", "25.0.2+10", false) check("openjdk-25.0.2", "25.0.2+10", false)