diff --git a/internal/cli/pivot.go b/internal/cli/pivot.go index 250942c..94df135 100644 --- a/internal/cli/pivot.go +++ b/internal/cli/pivot.go @@ -251,6 +251,28 @@ func runPivot(ctx context.Context, cfg *config.Config, stdout interface { slog.Info("systemd-mode: skipping init coordination + process termination") } + // Arm the watchdog now, right before the pivot, so the window until the + // supervisor adopts it (just the mount sequence + exec) is tiny. It's + // opened here on the host /dev; the fd is inherited across pivot_root and + // the exec, and pet post-pivot via WatchdogFDEnv. pivot_root keeps the + // running kernel, so it's the same live watchdog throughout — no softdog, + // no reopen by path in the pivoted rootfs. (EnsureWatchdogAvailable above + // already validated it while the old OS was fully alive.) + var watchdogFile *os.File + if cfg.WatchdogTimeout > 0 { + wf, err := postpivot.ArmWatchdogPrePivot(cfg.WatchdogTimeout) + if err != nil { + return fmt.Errorf("arm watchdog: %w", err) + } + watchdogFile = wf + // Runs only on an abort before exec (unix.Exec never returns on + // success): disarm so a box that's staying on the old OS isn't reset. + defer func() { + _, _ = watchdogFile.Write([]byte{'V'}) // magic close = disarm + _ = watchdogFile.Close() + }() + } + // All goroutines below this point share a single OS thread so the // new mount namespace is consistent across what we do. runtime.LockOSThread() @@ -300,9 +322,15 @@ func runPivot(ctx context.Context, cfg *config.Config, stdout interface { } // Exec the post-pivot supervisor: /usr/local/bin/xmorph --init . + // Hand the pre-armed watchdog fd across the exec so the supervisor pets + // the same live device instead of reopening it in the pivoted /dev. + env := os.Environ() + if watchdogFile != nil { + env = append(env, fmt.Sprintf("%s=%d", postpivot.WatchdogFDEnv, watchdogFile.Fd())) + } slog.Info("exec post-pivot supervisor", "binary", postpivot.BinaryPath, "entrypoint", entrypoint) supervisorArgv := append([]string{postpivot.BinaryPath, "--init", entrypoint}, entryArgs...) - if err := unix.Exec(postpivot.BinaryPath, supervisorArgv, os.Environ()); err != nil { + if err := unix.Exec(postpivot.BinaryPath, supervisorArgv, env); err != nil { return fmt.Errorf("exec post-pivot supervisor: %w", err) } return nil // unreachable diff --git a/internal/postpivot/run.go b/internal/postpivot/run.go index 2f939d3..aff0d79 100644 --- a/internal/postpivot/run.go +++ b/internal/postpivot/run.go @@ -7,6 +7,7 @@ import ( "log/slog" "os" "path/filepath" + "strconv" "time" "github.com/ananthb/xmorph/internal/tsnetauth" @@ -65,11 +66,18 @@ func Run(argv []string) int { } if cfg != nil && cfg.WatchdogTimeoutSeconds > 0 { - wd, err := StartWatchdog(time.Duration(cfg.WatchdogTimeoutSeconds) * time.Second) - if err != nil { - slog.Error("watchdog: cannot arm post-pivot", "err", err) + timeout := time.Duration(cfg.WatchdogTimeoutSeconds) * time.Second + // The watchdog was opened and armed pre-pivot; its fd is inherited + // here via WatchdogFDEnv. Adopt and pet it — don't reopen + // /dev/watchdog by path, which the pivoted /dev may not expose. + if s := os.Getenv(WatchdogFDEnv); s != "" { + if fd, err := strconv.Atoi(s); err == nil { + defer AdoptWatchdog(fd, timeout).Close() + } else { + slog.Error("watchdog: invalid inherited fd", "value", s, "err", err) + } } else { - defer wd.Close() + slog.Warn("watchdog: no inherited fd from pre-pivot; not armed post-pivot") } } diff --git a/internal/postpivot/watchdog_linux.go b/internal/postpivot/watchdog_linux.go index 276e968..60402be 100644 --- a/internal/postpivot/watchdog_linux.go +++ b/internal/postpivot/watchdog_linux.go @@ -16,9 +16,11 @@ import ( "golang.org/x/sys/unix" ) -// Watchdog holds /dev/watchdog open and pets it from a goroutine. -// Kernel-only — no userspace fallback. If /dev/watchdog is missing, -// StartWatchdog auto-loads softdog and errors out if that fails. +// Watchdog holds a kernel watchdog fd open and pets it from a goroutine. +// The fd is normally armed pre-pivot (ArmWatchdogPrePivot) and inherited +// across the pivot, then adopted here (AdoptWatchdog). StartWatchdog is the +// standalone path that opens /dev/watchdog directly (loading softdog if it's +// missing) — usable only where the device node is reachable by path. type Watchdog struct { timeout time.Duration fd *os.File @@ -37,6 +39,13 @@ const wdMagicClose byte = 'V' // watchdogDev is the kernel watchdog device path. var so tests can swap it. var watchdogDev = "/dev/watchdog" +// WatchdogFDEnv carries the pre-pivot watchdog fd number across the exec +// into the post-pivot supervisor. pivot_root keeps the running kernel, so a +// watchdog opened before the pivot keeps ticking after it — we inherit the +// open fd rather than reopening /dev/watchdog by path in the pivoted rootfs, +// whose /dev may not expose the node. +const WatchdogFDEnv = "XMORPH_WATCHDOG_FD" + // modprobeCmd auto-loads softdog when /dev/watchdog is absent. var // so tests can swap it for a fake. var modprobeCmd = func() error { @@ -152,6 +161,60 @@ func StartWatchdog(timeout time.Duration) (*Watchdog, error) { return w, nil } +// ArmWatchdogPrePivot opens and arms the kernel watchdog BEFORE pivot_root, +// while the host /dev still exposes it, and clears FD_CLOEXEC so the open fd +// survives the exec into the post-pivot supervisor. The caller passes the fd +// number to the supervisor via WatchdogFDEnv; the supervisor pets it with +// AdoptWatchdog. Because pivot_root keeps the running kernel, this is the +// same live watchdog throughout — no reopen by path in the pivoted rootfs +// (where the node may be absent) and no softdog (which can't load for the +// running kernel from an in-RAM rootfs). Non-positive timeout is a no-op. +// +// The returned *os.File must stay referenced until the exec so its finalizer +// doesn't close the fd; on any abort before exec the caller must disarm it +// (write wdMagicClose) and close it, else the box resets while staying on the +// old OS. +func ArmWatchdogPrePivot(timeout time.Duration) (*os.File, error) { + if timeout <= 0 { + return nil, nil + } + f, err := openWatchdog() + if err != nil { + return nil, err + } + secs := int(timeout / time.Second) + if secs < 1 { + secs = 1 + } + if err := unix.IoctlSetPointerInt(int(f.Fd()), _WDIOC_SETTIMEOUT, secs); err != nil { + slog.Warn("watchdog: set timeout failed; using kernel default", "err", err) + } + // Clear FD_CLOEXEC (Go sets it on every fd it opens) so the fd is + // inherited across unix.Exec into the supervisor. + if _, err := unix.FcntlInt(f.Fd(), unix.F_SETFD, 0); err != nil { + _, _ = f.Write([]byte{wdMagicClose}) + _ = f.Close() + return nil, fmt.Errorf("clear FD_CLOEXEC on watchdog fd: %w", err) + } + slog.Info("watchdog: armed pre-pivot (fd inherits across pivot)", "timeout", timeout, "fd", int(f.Fd())) + return f, nil +} + +// AdoptWatchdog takes over an already-open, already-armed watchdog fd +// inherited (via WatchdogFDEnv) from the pre-pivot ArmWatchdogPrePivot call +// and pets it from a goroutine. Use this post-pivot instead of StartWatchdog, +// which would try to reopen /dev/watchdog by path. +func AdoptWatchdog(fd int, timeout time.Duration) *Watchdog { + w := &Watchdog{ + timeout: timeout, + fd: os.NewFile(uintptr(fd), watchdogDev), + done: make(chan struct{}), + } + slog.Info("watchdog: adopted inherited fd", "fd", fd, "timeout", timeout) + go w.pet() + return w +} + func (w *Watchdog) pet() { tick := time.NewTicker(w.timeout / 3) defer tick.Stop() diff --git a/internal/postpivot/watchdog_other.go b/internal/postpivot/watchdog_other.go index c2308e5..d0dc438 100644 --- a/internal/postpivot/watchdog_other.go +++ b/internal/postpivot/watchdog_other.go @@ -4,9 +4,13 @@ package postpivot import ( "errors" + "os" "time" ) +// WatchdogFDEnv mirrors the linux constant so cross-platform callers compile. +const WatchdogFDEnv = "XMORPH_WATCHDOG_FD" + // Watchdog is a no-op on non-Linux platforms. type Watchdog struct{} @@ -20,5 +24,15 @@ func StartWatchdog(_ time.Duration) (*Watchdog, error) { return nil, errors.New("watchdog: not supported on this platform") } +// ArmWatchdogPrePivot always fails — no watchdog on this platform. +func ArmWatchdogPrePivot(_ time.Duration) (*os.File, error) { + return nil, errors.New("watchdog: not supported on this platform") +} + +// AdoptWatchdog is a no-op stub on non-Linux platforms. +func AdoptWatchdog(_ int, _ time.Duration) *Watchdog { + return &Watchdog{} +} + // Close is a no-op. func (*Watchdog) Close() {}