diff --git a/docs/rescue.md b/docs/rescue.md index 04fd7ce..add1413 100644 --- a/docs/rescue.md +++ b/docs/rescue.md @@ -210,6 +210,23 @@ box is worse than a clear "watchdog can't be armed on this host, drop `--watchdog-timeout` or fix the kernel." Sub-second timeouts are rejected. +## Persistent logs + +xmorph writes its own log and the entrypoint's stdout+stderr to +`/var/log/xmorph/` on the old rootfs by default. Since the pivoted +rootfs is tmpfs and gone on the next reboot, this is where you look +after a failed install. Change the location with `--log-persist-path` +and `--log-persist-device`; empty `--log-persist-path` disables. The +pivot aborts pre-pivot if the resolved directory isn't writable. + +```sh +sudo xmorph pivot --image alpine --rootfs ./overlay/ \ + --entrypoint /install.sh \ + --log-persist-device /srv --log-persist-path xmorph +# Post-reboot, on the recovered OS: +tail /srv/xmorph/xmorph.log /srv/xmorph/entrypoint.log +``` + ## Recovery and exit - **Back to the old OS**: `reboot`. The tmpfs is lost; the system comes diff --git a/internal/cli/dryrun.go b/internal/cli/dryrun.go index 4500b75..abffb1f 100644 --- a/internal/cli/dryrun.go +++ b/internal/cli/dryrun.go @@ -3,6 +3,7 @@ package cli import ( "fmt" "io" + "path/filepath" "strings" "github.com/ananthb/xmorph/internal/config" @@ -41,6 +42,9 @@ func printDryRun(w io.Writer, cfg *config.Config) { if cfg.WatchdogTimeout > 0 { fmt.Fprintf(w, "Watchdog: %s (reset if hung; /dev/watchdog or userspace timer)\n", cfg.WatchdogTimeout) } + if cfg.LogPersistPath != "" { + fmt.Fprintf(w, "Persistent logs: %s\n", filepath.Join("/", cfg.LogPersistDevice, cfg.LogPersistPath)) + } if cfg.Headless { fmt.Fprintf(w, "Mode: headless (will fork and detach, log to /var/log/xmorph.log)\n") } diff --git a/internal/cli/pivot.go b/internal/cli/pivot.go index a88ea1f..e445416 100644 --- a/internal/cli/pivot.go +++ b/internal/cli/pivot.go @@ -150,6 +150,19 @@ func runPivot(ctx context.Context, cfg *config.Config, stdout interface { slog.Info("watchdog pre-pivot check ok") } + // Persistent log dir pre-check. Resolved path lives on the pre-pivot + // root today; post-pivot it moves under KeepOldRoot. + if cfg.LogPersistPath != "" { + if cfg.KeepOldRoot == "" { + return fmt.Errorf("--log-persist-path requires --keep-old-root (need the old root mounted post-pivot)") + } + prePivotDir := filepath.Join("/", cfg.LogPersistDevice, cfg.LogPersistPath) + if err := ensureLogDirWritable(prePivotDir); err != nil { + return fmt.Errorf("--log-persist-path %s: %w", prePivotDir, err) + } + slog.Info("log-persist pre-pivot check ok", "path", prePivotDir) + } + // Memory headroom: warn before we commit to the pivot. // (sysmem usage will be wired through when M5's RAM telemetry is // surfaced via the slog output — for now we just verify the file @@ -241,6 +254,9 @@ func buildPostpivotConfig(cfg *config.Config, entrypoint string, entryArgs []str KeepOldRoot: cfg.KeepOldRoot, Entrypoint: append([]string{entrypoint}, entryArgs...), } + if cfg.LogPersistPath != "" && cfg.KeepOldRoot != "" { + pc.LogPersistDir = filepath.Join(cfg.KeepOldRoot, cfg.LogPersistDevice, cfg.LogPersistPath) + } if cfg.SSHEnabled() { ssh := &postpivot.SSHConfig{ Password: cfg.SSHPassword, @@ -295,6 +311,20 @@ func runContain(cfg *config.Config) error { }) } +// ensureLogDirWritable makes dir and confirms we can write into it. +func ensureLogDirWritable(dir string) error { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("mkdir: %w", err) + } + probe, err := os.CreateTemp(dir, ".xmorph-probe-") + if err != nil { + return fmt.Errorf("write probe: %w", err) + } + name := probe.Name() + probe.Close() + return os.Remove(name) +} + // resolveEntrypoint picks the effective entrypoint + args + env from the // CLI config and the merged ImageConfig. Mirrors src/cmd/pivot.zig:194-236. func resolveEntrypoint(cfg *config.Config, ic *oci.ImageConfig) (entrypoint string, args, env []string) { diff --git a/internal/config/config.go b/internal/config/config.go index 3ab95e9..d02d70c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -46,14 +46,15 @@ type Layer struct { // Default values mirrored from src/config.zig:22-105 with paths renamed to xmorph. const ( - DefaultImage = "docker.io/library/alpine:latest" - DefaultEntrypoint = "/bin/sh" - DefaultKeepOldRoot = "/mnt/oldroot" - DefaultTimeout = 30 - DefaultCacheDir = "/var/cache/xmorph" - DefaultWorkDir = "/run/xmorph/rootfs" - DefaultLogDir = "/var/log" - DefaultTailscaleImg = "docker.io/tailscale/tailscale:latest" + DefaultImage = "docker.io/library/alpine:latest" + DefaultEntrypoint = "/bin/sh" + DefaultKeepOldRoot = "/mnt/oldroot" + DefaultTimeout = 30 + DefaultCacheDir = "/var/cache/xmorph" + DefaultLogPersistPath = "/var/log/xmorph" + DefaultWorkDir = "/run/xmorph/rootfs" + DefaultLogDir = "/var/log" + DefaultTailscaleImg = "docker.io/tailscale/tailscale:latest" ) // Config holds the parsed CLI configuration. Field order and zero-values @@ -97,6 +98,12 @@ type Config struct { // it. Prefers /dev/watchdog (kernel), falls back to a userspace timer. WatchdogTimeout time.Duration + // LogPersistDevice + LogPersistPath compose the pre-pivot absolute + // path where xmorph writes durable logs. Post-pivot the same path + // is visible under KeepOldRoot. Empty LogPersistPath disables. + LogPersistDevice string + LogPersistPath string + // SSH: tri-state Enable (nil = auto from other ssh.* flags), explicit // fields below. SSHEnable *bool @@ -121,6 +128,7 @@ func New() Config { KeepOldRoot: DefaultKeepOldRoot, Timeout: DefaultTimeout, CacheDir: DefaultCacheDir, + LogPersistPath: DefaultLogPersistPath, WorkDir: DefaultWorkDir, LogDir: DefaultLogDir, TailscaleImage: DefaultTailscaleImg, diff --git a/internal/config/flags.go b/internal/config/flags.go index 7dddb79..e7da390 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -72,6 +72,8 @@ func bindPivotOnly(fs *pflag.FlagSet, cfg *Config) { fs.BoolVar(&cfg.Headless, "headless", false, "detach from terminal (implies --force)") fs.BoolVar(&cfg.KeepFirewall, "keep-firewall", false, "keep existing firewall rules (default: flush before starting services)") fs.DurationVar(&cfg.WatchdogTimeout, "watchdog-timeout", 0, "reset the box if the pivoted entrypoint hangs longer than this (0: disabled; uses /dev/watchdog when available, otherwise a userspace timer)") + fs.StringVar(&cfg.LogPersistDevice, "log-persist-device", "", "pre-pivot mount path holding the log directory (empty: old root)") + fs.StringVar(&cfg.LogPersistPath, "log-persist-path", DefaultLogPersistPath, "path under --log-persist-device where xmorph writes durable logs (empty: disabled)") } func bindBuildOnly(fs *pflag.FlagSet, cfg *Config) { diff --git a/internal/initsys/coordinator.go b/internal/initsys/coordinator.go index e5abc60..3596da8 100644 --- a/internal/initsys/coordinator.go +++ b/internal/initsys/coordinator.go @@ -31,7 +31,10 @@ func NewCoordinator(timeout time.Duration) Coordinator { func (c Coordinator) TransitionToRescue() error { switch c.Kind { case Systemd: - return c.run("systemctl", "isolate", "rescue.target") + // basic.target keeps journal + dbus up but stops all user + // services. rescue.target would additionally start sulogin + // on the console. + return c.run("systemctl", "isolate", "basic.target") case OpenRC: return c.run("openrc", "single") case SysVinit: diff --git a/internal/postpivot/configwrite.go b/internal/postpivot/configwrite.go index 07ced55..8c30a07 100644 --- a/internal/postpivot/configwrite.go +++ b/internal/postpivot/configwrite.go @@ -30,11 +30,14 @@ type Config struct { // KeepOldRoot is the pre-pivot root's mount point (default // /mnt/oldroot). Reboot path unmounts it before restart. Empty // means the pivot step already unmounted. - KeepOldRoot string `json:"keep_old_root,omitempty"` - SSH *SSHConfig `json:"ssh,omitempty"` - Tailscale *TSConfig `json:"tailscale,omitempty"` - Entrypoint []string `json:"entrypoint,omitempty"` - Command []string `json:"command,omitempty"` + KeepOldRoot string `json:"keep_old_root,omitempty"` + // LogPersistDir is the post-pivot absolute path (typically under + // KeepOldRoot) where xmorph writes durable logs. Empty disables. + LogPersistDir string `json:"log_persist_dir,omitempty"` + SSH *SSHConfig `json:"ssh,omitempty"` + Tailscale *TSConfig `json:"tailscale,omitempty"` + Entrypoint []string `json:"entrypoint,omitempty"` + Command []string `json:"command,omitempty"` } // SSHConfig describes the in-rootfs SSH setup (dropbear for now). diff --git a/internal/postpivot/configwrite_test.go b/internal/postpivot/configwrite_test.go index 57a18da..cce0bc5 100644 --- a/internal/postpivot/configwrite_test.go +++ b/internal/postpivot/configwrite_test.go @@ -14,6 +14,7 @@ func TestWriteConfigRoundTrip(t *testing.T) { RebootOnFailure: true, WatchdogTimeoutSeconds: 300, KeepOldRoot: "/mnt/oldroot", + LogPersistDir: "/mnt/oldroot/var/log/xmorph", SSH: &SSHConfig{ Port: 22, AuthorizedKeys: "ssh-ed25519 AAAA", @@ -47,6 +48,9 @@ func TestWriteConfigRoundTrip(t *testing.T) { if got.KeepOldRoot != "/mnt/oldroot" { t.Errorf("KeepOldRoot = %q, want /mnt/oldroot", got.KeepOldRoot) } + if got.LogPersistDir != "/mnt/oldroot/var/log/xmorph" { + t.Errorf("LogPersistDir = %q", got.LogPersistDir) + } if got.SSH == nil || got.SSH.Port != 22 { t.Errorf("SSH = %+v", got.SSH) } @@ -66,7 +70,7 @@ func TestWriteConfigOmitsEmptyOptional(t *testing.T) { } data, _ := os.ReadFile(filepath.Join(dir, ConfigPath)) s := string(data) - for _, omitted := range []string{"\"ssh\"", "\"tailscale\"", "\"entrypoint\"", "\"watchdog_timeout_seconds\"", "\"keep_old_root\""} { + for _, omitted := range []string{"\"ssh\"", "\"tailscale\"", "\"entrypoint\"", "\"watchdog_timeout_seconds\"", "\"keep_old_root\"", "\"log_persist_dir\""} { if contains := indexOfStr(s, omitted) >= 0; contains { t.Errorf("expected %q to be omitted; got %s", omitted, s) } diff --git a/internal/postpivot/run.go b/internal/postpivot/run.go index d2fa2a0..fa82fca 100644 --- a/internal/postpivot/run.go +++ b/internal/postpivot/run.go @@ -3,8 +3,10 @@ package postpivot import ( "context" "fmt" + "io" "log/slog" "os" + "path/filepath" "time" "github.com/ananthb/xmorph/internal/tsnetauth" @@ -35,6 +37,27 @@ func Run(argv []string) int { FlushFirewall() } + var entrypointLog io.Writer + if cfg != nil && cfg.LogPersistDir != "" { + if err := os.MkdirAll(cfg.LogPersistDir, 0o755); err != nil { + slog.Error("log-persist mkdir", "dir", cfg.LogPersistDir, "err", err) + } else { + if xf, err := os.OpenFile(filepath.Join(cfg.LogPersistDir, "xmorph.log"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644); err == nil { + defer xf.Close() + slog.SetDefault(slog.New(slog.NewTextHandler(io.MultiWriter(os.Stderr, xf), nil))) + slog.Info("persistent log opened", "dir", cfg.LogPersistDir) + } else { + slog.Error("open xmorph.log", "err", err) + } + if ef, err := os.OpenFile(filepath.Join(cfg.LogPersistDir, "entrypoint.log"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644); err == nil { + defer ef.Close() + entrypointLog = ef + } else { + slog.Error("open entrypoint.log", "err", err) + } + } + } + if cfg != nil && cfg.WatchdogTimeoutSeconds > 0 { wd, err := StartWatchdog(time.Duration(cfg.WatchdogTimeoutSeconds) * time.Second) if err != nil { @@ -88,6 +111,7 @@ func Run(argv []string) int { Argv: supervised, RebootOnFailure: rebootOnFailure, OldRootPath: oldRoot, + LogWriter: entrypointLog, }) if err != nil { fmt.Fprintf(os.Stderr, "xmorph --init: %v\n", err) diff --git a/internal/postpivot/supervise.go b/internal/postpivot/supervise.go index 94ad4b3..f29d890 100644 --- a/internal/postpivot/supervise.go +++ b/internal/postpivot/supervise.go @@ -3,6 +3,7 @@ package postpivot import ( "errors" "fmt" + "io" "os" "os/exec" "os/signal" @@ -22,6 +23,8 @@ type SuperviseOptions struct { RebootOnFailure bool // OldRootPath is unmounted before reboot; empty skips. OldRootPath string + // LogWriter, if non-nil, tees the child's stdout + stderr. + LogWriter io.Writer } // Supervise spawns Argv as a child process and forwards @@ -40,6 +43,10 @@ func Supervise(opts SuperviseOptions) (exitCode int, err error) { cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr + if opts.LogWriter != nil { + cmd.Stdout = io.MultiWriter(os.Stdout, opts.LogWriter) + cmd.Stderr = io.MultiWriter(os.Stderr, opts.LogWriter) + } if opts.Env != nil { cmd.Env = opts.Env } diff --git a/internal/postpivot/supervise_test.go b/internal/postpivot/supervise_test.go new file mode 100644 index 0000000..39695db --- /dev/null +++ b/internal/postpivot/supervise_test.go @@ -0,0 +1,25 @@ +package postpivot + +import ( + "bytes" + "strings" + "testing" +) + +func TestSuperviseTeesToLogWriter(t *testing.T) { + var buf bytes.Buffer + code, err := Supervise(SuperviseOptions{ + Argv: []string{"/bin/sh", "-c", "echo hi; echo bad >&2"}, + LogWriter: &buf, + }) + if err != nil { + t.Fatalf("Supervise: %v", err) + } + if code != 0 { + t.Errorf("exit code = %d, want 0", code) + } + out := buf.String() + if !strings.Contains(out, "hi") || !strings.Contains(out, "bad") { + t.Errorf("log buffer %q missing stdout or stderr", out) + } +}