[WIP] Support immutable hosts and configurable installation paths

cmd/agent/internal/bootstrap/coordinator.go

@@ -16,7 +16,17 @@ import (
 	"github.com/Azure/unbounded/cmd/agent/internal/installstate"
 )
 
-type Identity struct{ MachineName, ConfigFingerprint string }
+// Identity is what makes one installation distinguishable from another.
+//
+// HostPrefix is the resolved installation prefix. It is carried here so the
+// record written before the first host mutation knows where this installation
+// puts its files, which is the only thing teardown can consult after a
+// bootstrap that failed before the node started.
+type Identity struct {
+	MachineName       string
+	ConfigFingerprint string
+	HostPrefix        string
+}
 
 type Stages interface {
 	EnsureHostClean(context.Context) error
@@ -80,7 +90,7 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) {
 			return Outcome{}, err
 		}
 
-		r, err = installstate.NewRecord(id.MachineName, id.ConfigFingerprint)
+		r, err = installstate.NewRecord(id.MachineName, id.ConfigFingerprint, id.HostPrefix)
 		if err != nil {
 			return Outcome{}, err
 		}

cmd/agent/internal/bootstrap/coordinator_test.go

@@ -104,7 +104,7 @@ func TestCompletedRecoveryDoesNotResolveRetiredBootstrapInputs(t *testing.T) {
 	for _, repair := range []bool{false, true} {
 		dir := t.TempDir()
 		store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock"))
-		r, err := installstate.NewRecord("machine", "fingerprint")
+		r, err := installstate.NewRecord("machine", "fingerprint", "")
 		require.NoError(t, err)
 
 		r.Phase = installstate.Complete
@@ -140,7 +140,7 @@ func TestAdmissionFailurePreventsAllStageWork(t *testing.T) {
 		t.Run(mode, func(t *testing.T) {
 			dir := t.TempDir()
 			store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock"))
-			r, err := installstate.NewRecord("machine", "fingerprint")
+			r, err := installstate.NewRecord("machine", "fingerprint", "")
 			require.NoError(t, err)
 
 			if mode == "resetting" {
@@ -170,7 +170,7 @@ func TestAdmissionFailurePreventsAllStageWork(t *testing.T) {
 
 func TestInterruptedRepairRemainsCompleteAndRetries(t *testing.T) {
 	store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock"))
-	r, err := installstate.NewRecord("machine", "fingerprint")
+	r, err := installstate.NewRecord("machine", "fingerprint", "")
 	require.NoError(t, err)
 	require.NoError(t, store.MarkComplete(r))
 	stages := &fakeStages{store: store, fail: "repair", verifyErr: errInjected}

cmd/agent/internal/cmd/agentupgrade.go

@@ -42,10 +42,17 @@ type hostAgentUpgradeHandler struct {
 
 func newCmdHostAgentUpgrade(cmdCtx *CommandContext) *cobra.Command {
 	handler := &hostAgentUpgradeHandler{
-		cmdCtx:       cmdCtx,
-		writer:       os.Stdout,
-		executable:   os.Executable,
-		resolvedPath: goalstates.ResolvedAgentUpgradePaths,
+		cmdCtx:     cmdCtx,
+		writer:     os.Stdout,
+		executable: os.Executable,
+		// Wrapped rather than referenced directly so the prefix is read when
+		// the command runs, not when it is constructed. This runs on the host
+		// rather than under systemd, but the applied config is still the
+		// authority: the prefix belongs to the installation, not to whatever
+		// environment happens to be invoking the upgrade.
+		resolvedPath: func() (goalstates.AgentUpgradePaths, error) {
+			return goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig())
+		},
 		geteuid:      os.Geteuid,
 		installation: installstate.DefaultStore(),
 	}

cmd/agent/internal/cmd/bootstrap.go

@@ -64,16 +64,52 @@ func canonicalImageIdentity(image string) string {
 func bootstrapIdentity(cfg *provision.UnboundedAgentConfig) (bootstrap.Identity, error) {
 	// Keep identity tied to the cluster and installed rootfs, while allowing
 	// credentials and artifact locations to be refreshed for a retry.
+	//
+	// HostPrefix enters the hash only when it resolves somewhere other than the
+	// default, and carries omitempty so that at the default it contributes
+	// nothing at all. Every host already in the field was fingerprinted without
+	// this input; if the default hashed as a value, each of them would read as a
+	// different installation and demand an explicit reset on upgrade, for a
+	// field they never set. TestBootstrapV1CompatibilityFixtures catches that.
+	//
+	// It is the resolved prefix that matters, not how it was written. Leaving it
+	// unset and naming /usr/local explicitly put the files in the same place, so
+	// they are the same installation and must hash alike.
+	//
+	// A prefix that resolves elsewhere does belong in the identity. The agent's
+	// own files live under it, so starting with a different one is not a retry:
+	// it would leave the first installation behind and build a second one
+	// beside it.
+	resolvedPrefix := goalstates.HostPrefixOrDefault(cfg.HostPrefix)
+
+	fingerprintedPrefix := resolvedPrefix
+	if fingerprintedPrefix == goalstates.DefaultHostPrefix {
+		fingerprintedPrefix = ""
+	}
+
 	data, err := json.Marshal(struct {
 		KubernetesVersion string
 		OCIImage          string
 		APIServer         string
-	}{strings.TrimPrefix(cfg.Cluster.Version, "v"), canonicalImageIdentity(cfg.OCIImage), cfg.Kubelet.ApiServer})
+		HostPrefix        string `json:",omitempty"`
+	}{
+		strings.TrimPrefix(cfg.Cluster.Version, "v"),
+		canonicalImageIdentity(cfg.OCIImage),
+		cfg.Kubelet.ApiServer,
+		fingerprintedPrefix,
+	})
 	if err != nil {
 		return bootstrap.Identity{}, err
 	}
 
-	return bootstrap.Identity{MachineName: cfg.MachineName, ConfigFingerprint: installstate.Fingerprint(data)}, nil
+	return bootstrap.Identity{
+		MachineName:       cfg.MachineName,
+		ConfigFingerprint: installstate.Fingerprint(data),
+		// Resolved rather than configured, so the record names a real directory
+		// instead of an empty string meaning "wherever the default was at the
+		// time", which is what teardown would have to guess from.
+		HostPrefix: resolvedPrefix,
+	}, nil
 }
 
 func (s *agentStages) EnsureHostClean(ctx context.Context) error {

cmd/agent/internal/cmd/bootstrap_test.go

@@ -256,3 +256,68 @@ func TestClassifyNodeStartFailure(t *testing.T) {
 		})
 	}
 }
+
+// TestBootstrapFingerprintTracksTheInstallationPrefix covers both halves of how
+// the prefix enters installation identity, because the two pull in opposite
+// directions.
+//
+// Configuring a prefix has to change the fingerprint. The agent's own binaries
+// live under it, so a start with a different prefix is not a retry of the same
+// installation: continuing would leave the first installation's files behind
+// and build a second one beside them. Admission must refuse and ask for a
+// reset, which is what a changed fingerprint does.
+//
+// Configuring nothing has to change nothing. Every host already in the field
+// was fingerprinted without this input, and if the default hashed differently
+// each of them would read as a different installation and demand an explicit
+// reset on upgrade, for a field they never set.
+func TestBootstrapFingerprintTracksTheInstallationPrefix(t *testing.T) {
+	load := func(t *testing.T) *provision.UnboundedAgentConfig {
+		t.Helper()
+
+		cfg, err := loadConfigFromFile(filepath.Join("testdata", "bootstrap-v1", "input.json"))
+		require.NoError(t, err)
+
+		return cfg
+	}
+
+	baseline, err := bootstrapIdentity(load(t))
+	require.NoError(t, err)
+
+	// Whitespace is not a configuration choice, so it must not be one here
+	// either; otherwise a stray space rewrites the identity of a default host.
+	for _, blank := range []string{"", " ", "\t"} {
+		cfg := load(t)
+		cfg.HostPrefix = blank
+
+		unset, err := bootstrapIdentity(cfg)
+		require.NoError(t, err)
+		require.Equal(t, baseline.ConfigFingerprint, unset.ConfigFingerprint,
+			"an unset prefix must hash as it did before the field existed, got %q", blank)
+	}
+
+	// Naming the default explicitly puts the files in the same place as leaving
+	// it unset, so the two are the same installation. Hashing them differently
+	// would tell an operator who wrote down what was already true that they
+	// must reset the host.
+	explicit := load(t)
+	explicit.HostPrefix = goalstates.DefaultHostPrefix
+
+	explicitID, err := bootstrapIdentity(explicit)
+	require.NoError(t, err)
+	require.Equal(t, baseline.ConfigFingerprint, explicitID.ConfigFingerprint,
+		"identity follows where the files land, not how the prefix was spelled")
+
+	moved := load(t)
+	moved.HostPrefix = "/opt/unbounded"
+
+	movedID, err := bootstrapIdentity(moved)
+	require.NoError(t, err)
+	require.NotEqual(t, baseline.ConfigFingerprint, movedID.ConfigFingerprint,
+		"moving the installation prefix must not read as a retry of the same installation")
+	require.Equal(t, "/opt/unbounded", movedID.HostPrefix)
+
+	// The record needs a real directory, not an empty string standing for
+	// whatever the default was when it was written.
+	require.Equal(t, goalstates.DefaultHostPrefix, baseline.HostPrefix)
+}

cmd/agent/internal/daemon/agentupgrade.go

@@ -61,7 +61,7 @@ func parseAgentUpgradeRequest(parameters map[string]string) (agentUpgradeRequest
 }
 
 func upgradeDaemonBinary(ctx context.Context, log *slog.Logger, request agentUpgradeRequest) error {
-	paths, err := goalstates.ResolvedAgentUpgradePaths()
+	paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig())
 	if err != nil {
 		return fmt.Errorf("resolve current daemon binary symlink: %w", err)
 	}
@@ -85,7 +85,7 @@ func upgradeDaemonBinary(ctx context.Context, log *slog.Logger, request agentUpg
 }
 
 func newAgentUpgradeSignalOperator() (agentUpgradeSignalOperator, error) {
-	paths, err := goalstates.ResolvedAgentUpgradePaths()
+	paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig())
 	if err != nil {
 		return nil, fmt.Errorf("resolve AgentUpgrade signal path: %w", err)
 	}

cmd/agent/internal/daemon/lifecycle.go

@@ -51,7 +51,7 @@ func EnableDaemon(log *slog.Logger) phases.Task {
 func (d *enableDaemon) Name() string { return "enable-daemon" }
 
 func (d *enableDaemon) Do(ctx context.Context) error {
-	paths, err := goalstates.ResolvedAgentUpgradePaths()
+	paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig())
 	if err != nil {
 		return fmt.Errorf("resolve current daemon binary symlink: %w", err)
 	}
@@ -158,7 +158,7 @@ func usableDaemonBinary(path string) bool {
 }
 
 func renderDaemonAsset(name string, content []byte) ([]byte, error) {
-	paths, err := goalstates.ResolvedAgentUpgradePaths()
+	paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig())
 	if err != nil {
 		return nil, err
 	}
@@ -339,7 +339,7 @@ func removeOwnedFile(path string) error {
 // active daemon already proves it resolved an applied config at startup, so the
 // applied-config check belongs to RepairDaemon rather than here.
 func VerifyDaemonInstalled(ctx context.Context, log *slog.Logger) error {
-	paths, err := goalstates.ResolvedAgentUpgradePaths()
+	paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig())
 	if err != nil {
 		return err
 	}

cmd/agent/internal/daemon/migration_test.go

@@ -61,7 +61,7 @@ func TestStartupStandsDownWhileInstallationUnfinished(t *testing.T) {
 	t.Parallel()
 	store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock"))
 
-	record, err := installstate.NewRecord("machine-1", "fingerprint")
+	record, err := installstate.NewRecord("machine-1", "fingerprint", "")
 	require.NoError(t, err)
 	require.NoError(t, store.Save(record))
 

cmd/agent/internal/daemon/reset.go

@@ -72,7 +72,7 @@ func recordForTeardown(log *slog.Logger, store *installstate.Store) (installstat
 		log.Warn("installation record is unreadable; replacing it for teardown", "error", err)
 	}
 
-	return installstate.NewRecord("legacy-reset", "legacy-reset")
+	return installstate.NewRecord("legacy-reset", "legacy-reset", "")
 }
 
 func resetUnderLock(ctx context.Context, log *slog.Logger, store *installstate.Store, inner phases.Task) error {

cmd/agent/internal/daemon/reset_test.go

@@ -35,7 +35,7 @@ func TestResetRetainsOwnershipUntilTeardownAndSyncSucceed(t *testing.T) {
 		t.Run(failure, func(t *testing.T) {
 			dir := t.TempDir()
 			store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock"))
-			r, err := installstate.NewRecord("machine", "f")
+			r, err := installstate.NewRecord("machine", "f", "")
 			require.NoError(t, err)
 
 			r.Phase = installstate.Resetting
@@ -128,7 +128,7 @@ func TestTeardownKeepsAReadableRecord(t *testing.T) {
 	dir := t.TempDir()
 	store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock"))
 
-	saved, err := installstate.NewRecord("machine-1", "fingerprint-1")
+	saved, err := installstate.NewRecord("machine-1", "fingerprint-1", "")
 	require.NoError(t, err)
 	require.NoError(t, store.Save(saved))
 

cmd/agent/internal/installstate/store.go

@@ -52,6 +52,21 @@ type Record struct {
 	MachineName       string `json:"machineName"`
 	ConfigFingerprint string `json:"configFingerprint"`
 	Phase             Phase  `json:"phase"`
+
+	// HostPrefix is the resolved installation prefix, recorded so teardown can
+	// find the agent's own files without being told where they are.
+	//
+	// It is written before the first host mutation, which makes it the only
+	// source that survives a bootstrap that failed before the node started. The
+	// applied config carries the same prefix but does not exist until then, so
+	// reset on a half-built host has nothing else to go on.
+	//
+	// Optional, and absent means the default. The schema version does not move
+	// for it: a record written by an agent that knows about the prefix stays
+	// readable by one that does not, because unknown fields are ignored, and a
+	// record written before it existed is read here as the default, which is
+	// what such a host actually has on disk.
+	HostPrefix string `json:"hostPrefix,omitempty"`
 }
 
 func (r Record) Validate() error {
@@ -157,15 +172,24 @@ func (s *Store) Remove() error {
 	return err
 }
 
-func NewRecord(machine, fingerprint string) (Record, error) {
+// NewRecord returns a record for a fresh installation.
+//
+// hostPrefix is a parameter rather than a field callers set afterwards because
+// forgetting it is silent and only surfaces at teardown, on a host whose files
+// are somewhere reset would not look. An empty prefix means the default.
+//
+// The value is stored as given and not validated here. This package deals in
+// stdlib and durability only, and pulling in config validation to re-check a
+// string this agent wrote from an already validated config would buy little.
+func NewRecord(machine, fingerprint, hostPrefix string) (Record, error) {
 	id := make([]byte, 16)
 	if _, err := rand.Read(id); err != nil {
 		return Record{}, err
 	}
 
 	return Record{
 		SchemaVersion: schemaVersion, InstallID: hex.EncodeToString(id), MachineName: machine,
-		ConfigFingerprint: fingerprint, Phase: Installing,
+		ConfigFingerprint: fingerprint, Phase: Installing, HostPrefix: hostPrefix,
 	}, nil
 }