Locked mode with a Podman machine on Windows

.github/workflows/ci.yml

@@ -118,12 +118,66 @@ jobs:
           & $podman machine start
           if ($LASTEXITCODE -ne 0) { throw "podman machine start failed" }
           & $podman version
-      # Open network only. In locked mode `mudroom network check` hangs on
-      # this runner after "checking locked network with podman..." (no output
-      # for 6+ minutes), and WSL's mirrored networking, which would let the
-      # machine reach the proxy on the runner's 127.0.0.1, fails here with
-      # 0x803b0015. Locked mode still needs a run with Docker Desktop on a PC.
       - name: Sandbox session (podman, open network)
         timeout-minutes: 15
         shell: pwsh
         run: scripts/windows-sandbox-check.ps1 $env:MUDROOM_EXE podman -Network open
+      # Locked mode: the Podman machine reaches the proxy on the WSL network's
+      # gateway, which Windows Firewall blocks until a rule allows it. Without
+      # the rule the check has to stop quickly and say which rule to add.
+      - name: Locked network check without a firewall rule
+        timeout-minutes: 5
+        shell: pwsh
+        run: |
+          $out = & $env:MUDROOM_EXE network check --backend podman --mode locked 2>&1 | Out-String
+          Write-Host $out
+          if ($LASTEXITCODE -eq 0) { throw "the locked check passed without a firewall rule" }
+          if ($out -notmatch 'New-NetFirewallRule') { throw "no firewall hint in the error" }
+          # The failure above is the expected result; without this the
+          # runner's pwsh wrapper exits with mudroom's status.
+          exit 0
+      - name: Allow Mudroom's proxy through Windows Firewall
+        shell: pwsh
+        run: |
+          New-NetFirewallRule -DisplayName 'Mudroom proxy' -Direction Inbound -Action Allow -Protocol TCP -Program $env:MUDROOM_EXE | Out-Null
+          # Log dropped packets, so a failure below names the filter.
+          auditpol /set /subcategory:"Filtering Platform Packet Drop" /failure:enable
+          auditpol /set /subcategory:"Filtering Platform Connection" /failure:enable
+      - name: Sandbox session (podman, locked network)
+        timeout-minutes: 15
+        shell: pwsh
+        run: scripts/windows-sandbox-check.ps1 $env:MUDROOM_EXE podman -SkipImageBuild
+      # What the firewall and WSL's network looked like when locked mode
+      # failed, which filter dropped the connection, and whether a plain
+      # listener on the WSL gateway is reachable.
+      - name: Firewall and network state
+        if: failure()
+        timeout-minutes: 10
+        shell: pwsh
+        run: |
+          $ErrorActionPreference = 'Continue'
+          Get-NetFirewallRule -Direction Inbound | Where-Object { ($_ | Get-NetFirewallApplicationFilter).Program -like '*mudroom*' } |
+            Format-Table DisplayName, Action, Enabled, Profile -AutoSize | Out-String -Width 200
+          Get-NetFirewallRule -Direction Inbound -Action Block -Enabled True -ErrorAction SilentlyContinue | Format-Table DisplayName, Profile -AutoSize | Out-String -Width 200
+          $rows = Get-WinEvent -FilterHashtable @{LogName='Security'; Id=5152,5157} -MaxEvents 500 -ErrorAction SilentlyContinue | ForEach-Object {
+            $d = @{}; ([xml]$_.ToXml()).Event.EventData.Data | ForEach-Object { $d[$_.Name] = $_.'#text' }
+            [pscustomobject]@{ Time = $_.TimeCreated.ToString('HH:mm:ss'); Id = $_.Id; App = $d.Application; Dir = $d.Direction
+              Src = "$($d.SourceAddress):$($d.SourcePort)"; Dst = "$($d.DestAddress):$($d.DestPort)"; Filter = $d.FilterRTID; Layer = $d.LayerName }
+          } | Where-Object { $_.Src -like '172.*' -or $_.Dst -like '172.*' }
+          $rows | Select-Object -First 40 | Format-Table -AutoSize | Out-String -Width 300
+          netsh wfp show filters file="$env:RUNNER_TEMP\filters.xml" | Out-Null
+          $ids = @($rows.Filter | Sort-Object -Unique)
+          ([xml](Get-Content "$env:RUNNER_TEMP\filters.xml")).SelectNodes('//filters/item') | Where-Object { $ids -contains $_.filterId } |
+            ForEach-Object { "filter $($_.filterId): $($_.displayData.name) / $($_.displayData.description) / $($_.action.type)" }
+          $routes = podman machine ssh "ip -4 route"
+          $routes
+          $gw = ([regex]'default via (\S+)').Match(($routes | Out-String)).Groups[1].Value
+          New-NetFirewallRule -DisplayName 'diag listener' -Direction Inbound -Action Allow -Protocol TCP -LocalPort 18080 | Out-Null
+          $l = [Net.Sockets.TcpListener]::new([Net.IPAddress]::Parse($gw), 18080); $l.Start()
+          podman machine ssh "timeout 5 bash -c 'echo > /dev/tcp/$gw/18080' && echo 'machine to gateway: ok' || echo 'machine to gateway: failed'"
+          $js = "for (const h of ['$gw', 'host.containers.internal']) { const s = require('net').connect({host: h, port: 18080}); const t = setTimeout(() => { console.log(h, 'timeout'); s.destroy(); }, 5000); s.on('connect', () => { clearTimeout(t); console.log(h, 'ok'); s.destroy(); }); s.on('error', e => { clearTimeout(t); console.log(h, e.code); }); }"
+          podman run --rm --network podman mudroom/agent-base:latest cat /etc/hosts
+          podman run --rm --network podman mudroom/agent-base:latest node -e $js
+          $l.Stop()
+          & $env:MUDROOM_EXE network check --backend podman --mode locked
+          exit 0

README.md

@@ -767,10 +767,13 @@ address; `mudroom network check` will tell you.
 On Windows the CLI works the same way, with Docker Desktop or a Podman
 machine running Linux containers in a VM. Host paths are passed to
 `docker run --mount` as Windows paths (`C:\Users\...`), which Docker Desktop
-shares into its VM. Locked mode is set up as on macOS: the proxy listens
-on 127.0.0.1 and the forwarder reaches it through `host.docker.internal`.
-Run `mudroom network check --backend docker` before relying on it. What
-is different from macOS and Linux:
+shares into its VM. With Docker Desktop, locked mode is set up as on
+macOS: the proxy listens on 127.0.0.1 and the forwarder reaches it through
+`host.docker.internal`. A Podman machine on WSL 2 can't reach Windows'
+127.0.0.1, so there the proxy listens on the WSL network's gateway (the
+"vEthernet (WSL)" address, which `host.containers.internal` points to) and
+accepts only the WSL network. Run `mudroom network check` before relying
+on it. What is different from macOS and Linux:
 
 - **No permission bits.** Windows files have no Unix mode, so every file
   reads as 0644 and every folder as 0755. A script the agent makes
@@ -801,15 +804,28 @@ is different from macOS and Linux:
   sandbox (create, edit, diff, apply, undo). It also unzips each release
   zip on a runner that never had Swift installed and runs the same check
   there. On x86_64 it runs a real sandbox with a Podman machine (WSL 2):
-  the open-network check and a short session, then diff and apply. Locked
-  mode, the allowlist proxy, hasn't passed in CI yet: with a rootless
-  Podman machine on the runner, `mudroom network check` hangs in locked
-  mode. Docker Desktop hasn't been run by CI, and the arm64 runner can't
-  run a VM at all. Running the Linux build inside WSL2 is untested.
+  the network check and a short session, then diff and apply, with the open
+  and the locked network. Docker Desktop hasn't been run by CI, and the
+  arm64 runner can't run a VM at all. Running the Linux build inside WSL2
+  is untested.
 - **Podman machine.** If Podman says ``controller `pids` is not
   available``, the machine's user has no pids cgroup controller; set
   `pids_limit = 0` under `[containers]` in `~/.config/containers/containers.conf`
   inside the machine (`podman machine ssh`), then restart it.
+- **Locked mode with Podman needs a firewall rule.** Windows Firewall drops
+  connections from WSL to Windows, so the machine can't reach the proxy
+  until a rule allows it. Mudroom checks before the sandbox starts and, if
+  the proxy is out of reach, stops with the rule to add, once, from an
+  administrator PowerShell:
+
+  ```powershell
+  New-NetFirewallRule -DisplayName 'Mudroom proxy' -Direction Inbound -Action Allow -Protocol TCP -Program (Get-Command mudroom).Source
+  ```
+
+  It also needs the machine on WSL's own network: with user-mode
+  networking (`podman machine set --user-mode-networking`)
+  `host.containers.internal` doesn't lead to Windows, and locked mode
+  can't work.
 
 ## How it works
 

Sources/MudroomCore/DockerBackend.swift

@@ -331,7 +331,10 @@ public struct DockerBackend: SandboxBackend {
     /// that forwards `host.docker.internal` to the Mac's loopback, so the
     /// proxy listens on 127.0.0.1 only. On Linux the sidecar reaches the
     /// host at the default bridge's gateway, so the proxy listens there and
-    /// accepts only that bridge.
+    /// accepts only that bridge. On Windows, Docker Desktop works as on
+    /// macOS; a Podman machine on WSL 2 can't reach Windows' loopback, so
+    /// the proxy listens on the WSL network's gateway instead (see
+    /// `podmanMachineGateway`).
     public func proxyListen(for plan: NetworkPlan) throws -> ProxyListen {
         let blocked = plan.clientSubnet.map { [$0] } ?? []
         #if os(Linux)
@@ -340,10 +343,116 @@ public struct DockerBackend: SandboxBackend {
         }
         return ProxyListen(bindHost: bridge.gateway, clientSubnets: [subnet], blockedSubnets: blocked)
         #else
+        #if os(Windows)
+        if let wsl = try podmanMachineGateway() {
+            return ProxyListen(bindHost: wsl.gateway, clientSubnets: [wsl.subnet], blockedSubnets: blocked)
+        }
+        #endif
         return ProxyListen(bindHost: "127.0.0.1", clientSubnets: [], blockedSubnets: blocked)
         #endif
     }
 
+    /// The default gateway in `ip -4 route` output and the on-link subnet
+    /// it belongs to: ("172.20.224.1", 172.20.224.0/20) for
+    /// "default via 172.20.224.1 dev eth0" and
+    /// "172.20.224.0/20 dev eth0 proto kernel scope link src 172.20.234.217".
+    /// Tolerates what a Windows console can add: CRLF line ends, a byte
+    /// order mark, tabs.
+    public static func defaultGateway(routes: String) -> (gateway: String, subnet: IPv4Subnet)? {
+        let lines = routes.replacingOccurrences(of: "\u{FEFF}", with: "").split(whereSeparator: \.isNewline)
+            .map { $0.split(whereSeparator: \.isWhitespace).map(String.init) }
+        guard let route = lines.first(where: { $0.first == "default" }), let via = route.firstIndex(of: "via"),
+              via + 1 < route.count, let gateway = IPAddress(route[via + 1]), gateway.asIPv4 != nil else { return nil }
+        let subnet = lines.lazy.compactMap { $0.first.flatMap(IPv4Subnet.init) }.first { $0.prefix > 0 && $0.contains(gateway) }
+        return subnet.map { (route[via + 1], $0) }
+    }
+
+    /// `text` with line breaks, tabs, byte order marks and other control
+    /// characters spelled out, for an error message.
+    static func visible(_ text: String) -> String {
+        text.unicodeScalars.map { c -> String in
+            switch c {
+            case "\n": return "\\n"
+            case "\r": return "\\r"
+            case "\t": return "\\t"
+            case _ where c.value < 0x20 || c.value == 0x7f || c.value == 0xfeff:
+                return "\\u{" + String(c.value, radix: 16) + "}"
+            default: return String(c)
+            }
+        }.joined()
+    }
+
+    /// A path with Windows' separators, for a command to paste into
+    /// PowerShell: Foundation reports the executable's path with `/`.
+    public static func windowsPath(_ path: String) -> String {
+        path.replacingOccurrences(of: "/", with: "\\")
+    }
+
+    #if os(Windows)
+    /// A Podman machine on WSL 2 reaches Windows only through WSL's NAT
+    /// network, at the machine's default gateway (Windows' "vEthernet (WSL)"
+    /// address), which is also what `host.containers.internal` resolves to.
+    /// Windows' 127.0.0.1 is out of reach from there. Returns that gateway
+    /// and the subnet the machine's connections come from, or nil for
+    /// Docker. Throws when the routes can't be read, or when the gateway
+    /// isn't an address on this PC (user-mode networking, where it is
+    /// gvproxy's): a proxy on 127.0.0.1 would be out of the machine's reach.
+    func podmanMachineGateway() throws -> (gateway: String, subnet: IPv4Subnet)? {
+        guard flavor == .podman, let exe = executable else { return nil }
+        let command = "ip -4 route"
+        let out = try ProcessRunner.capture(exe, ["machine", "ssh", command], timeout: 30)
+        guard out.status == 0, !out.timedOut else {
+            let how = out.timedOut ? "didn't finish within 30 seconds" : "exited with \(out.status)"
+            throw MudroomError.backendUnavailable(
+                "couldn't read the Podman machine's network for the proxy: `podman machine ssh \"\(command)\"` \(how) (\(Self.failureDetail(out))). Is the machine running (`podman machine start`)?")
+        }
+        guard let found = Self.defaultGateway(routes: out.stdout) else {
+            let shown = Self.visible(out.stdout)
+            throw MudroomError.backendUnavailable(
+                "couldn't find the Podman machine's default gateway for the proxy in the output of `podman machine ssh \"\(command)\"`: \(shown.isEmpty ? "nothing" : String(shown.prefix(600)))")
+        }
+        do {
+            let bound = try Sock.listen(host: found.gateway, port: 0)
+            Sock.close(bound.0)
+        } catch {
+            throw MudroomError.backendUnavailable(
+                "the Podman machine's gateway, \(found.gateway), isn't an address Mudroom's proxy can listen on (\(error)). Locked mode needs the machine on WSL's own network; with user-mode networking it can't reach Windows (`podman machine set --user-mode-networking=false` turns that off).")
+        }
+        return found
+    }
+
+    /// Node one-liner run in the sidecar: one connection to the proxy.
+    /// Prints "ok", or what went wrong and the address it tried.
+    static let upstreamCheckScript = """
+    require('dns').lookup(process.env.MUDROOM_UPSTREAM_HOST, {family: 4}, (e, a) => {
+      if (e) { console.log(e.code || 'error'); process.exit(0); }
+      const s = require('net').connect({host: a, port: +process.env.MUDROOM_UPSTREAM_PORT});
+      setTimeout(() => { console.log('timeout connecting to ' + a); process.exit(0); }, 5000);
+      s.on('connect', () => { console.log('ok'); process.exit(0); });
+      s.on('error', e => { console.log((e.code || 'error') + ' connecting to ' + a); process.exit(0); });
+    });
+    """
+
+    /// Checks the sidecar reaches the proxy. Windows Firewall drops
+    /// connections from WSL to Windows unless a rule allows them, and the
+    /// sandbox would otherwise see every request time out, with no hint why.
+    func checkUpstream(_ container: String, port: UInt16, _ exe: String) throws {
+        let out = try ProcessRunner.capture(exe, ["exec", container, "node", "-e", Self.upstreamCheckScript], timeout: 30)
+        let result = out.stdout.trimmingCharacters(in: .whitespacesAndNewlines)
+        if result == "ok" { return }
+        let why = out.timedOut ? "timeout" : result.isEmpty ? Self.failureDetail(out) : result
+        let program = Self.windowsPath(Bundle.main.executablePath ?? CommandLine.arguments[0])
+        throw MudroomError.backendUnavailable("""
+            the Podman machine can't reach Mudroom's proxy at \(hostAlias):\(port) (\(why)). \
+            Windows Firewall blocks connections from WSL to Windows unless a rule allows them. \
+            Allow Mudroom's proxy once, from an administrator PowerShell:
+              New-NetFirewallRule -DisplayName 'Mudroom proxy' -Direction Inbound -Action Allow -Protocol TCP -Program '\(program)'
+            Locked mode also needs the machine on WSL's own network: it doesn't work with user-mode networking \
+            (`podman machine set --user-mode-networking=false` turns that off).
+            """)
+    }
+    #endif
+
     /// Node one-liner for the sidecar: accept on 3128, pipe each connection
     /// to the host proxy. Half-closes pass through.
     static let forwarderScript = """
@@ -400,6 +509,9 @@ public struct DockerBackend: SandboxBackend {
             }
             let ip = try sidecarAddress(sidecar, network: internalNet, exe)
             try waitForSidecar(sidecar, exe)
+            #if os(Windows)
+            if flavor == .podman { try checkUpstream(sidecar, port: port, exe) }
+            #endif
             return ProxyRoute(host: ip, port: Self.sidecarPort, teardown: teardown)
         } catch {
             teardown()

Sources/MudroomCore/ProcessRunner.swift

@@ -57,7 +57,23 @@ public enum ProcessRunner {
         }
         process.standardOutput = outHandle
         process.standardError = errHandle
+        #if os(Windows)
+        // An empty file, not FileHandle.nullDevice: Foundation gives the
+        // child a NUL handle it doesn't mark inheritable, so the child's
+        // stdin is invalid. Most programs never notice, but one that hands
+        // its stdin on fails: `podman machine ssh` stops with "fork/exec
+        // ssh.exe: The handle is invalid".
+        let inURL = tmp.appendingPathComponent("mudroom-in-\(UUID().uuidString)")
+        _ = FileManager.default.createFile(atPath: inURL.path, contents: nil)
+        let inHandle = try FileHandle(forReadingFrom: inURL)
+        defer {
+            try? inHandle.close()
+            try? FileManager.default.removeItem(at: inURL)
+        }
+        process.standardInput = inHandle
+        #else
         process.standardInput = FileHandle.nullDevice
+        #endif
         try start(process)
         let timedOut = LockedBox(false)
         if let timeout {

Sources/MudroomCore/SessionRunner.swift

@@ -146,6 +146,13 @@ public final class NetworkSetup: @unchecked Sendable {
                 route = try backend.attachProxy(port: p.port, plan: plan)
             } catch {
                 p.stop()
+                #if os(Windows)
+                // Where the proxy was, so a connection that never arrives
+                // can be told apart from one the firewall drops.
+                if let e = error as? MudroomError, case .backendUnavailable(let why) = e {
+                    throw MudroomError.backendUnavailable(why + "\n(Mudroom's proxy was listening on \(listen.bindHost ?? "all addresses"):\(p.port).)")
+                }
+                #endif
                 throw error
             }
             teardown.value = route.teardown