fix(proxmox): deliver snippets over SSH and preserve cloud-image disk options

README.md

@@ -176,6 +176,11 @@ providers:
         endpoint: https://pve.home.local:8006
         node: pve1
         credentials: homelab-token
+        snippetSSH:
+          hosts:
+            pve1: pve.home.local
+          user: root
+          identityFile: ~/.ssh/proxmox
     credentials:
       homelab-token:
         tokenId: root@pam!openctl
@@ -185,6 +190,17 @@ providers:
       network: vmbr0
 ```
 
+Cloud-init vendor snippets (`packages` / `runcmd`, plus guest-agent enablement)
+are written through SSH: Proxmox's REST upload endpoint does not accept snippets.
+Map each Proxmox node explicitly under `snippetSSH.hosts`; the SSH account must
+be able to run `pvesm path` and write the selected storage's snippet directory.
+`identityFile` is a path on the machine running openctl/controller, not key
+material in a manifest. Omit it to use normal OpenSSH agent/default identities.
+Uploads use batch authentication, strict host-key checking, and atomic file
+replacement. Verify and trust the node's host-key fingerprint beforehand;
+unknown or changed host keys fail rather than being accepted automatically.
+
+
 Full controller behavior config:
 
 ```yaml
@@ -205,6 +221,11 @@ providers:
         endpoint: https://pve.home.local:8006
         node: pve1
         credentials: homelab-token
+        snippetSSH:
+          hosts:
+            pve1: pve.home.local
+          user: root
+          identityFile: ~/.ssh/proxmox
     credentials:
       homelab-token:
         tokenId: root@pam!openctl

docs/homelab-verification.md

@@ -89,3 +89,31 @@ commands, but its **first real run against your hardware is itself part of the
 verification** — that run both exercises openctl and shakes out any
 environment-specific rough edges here. Report failures with `openctl ctl op
 list` / `openctl ctl op get <id>` output.
+
+## Cloud-image disks and cloud-init
+
+Cloud-image creation must apply disk options after cloning and resizing, before
+starting the guest. Check `qm config <vmid>` on the Proxmox node: a requested
+300G SSD with TRIM must retain its volume reference and show `size=300G`,
+`ssd=1`, and `discard=on`. This path was exercised on Proxmox 9.1.1; the
+regression test also checks that a rejected option update prevents startup.
+
+Proxmox's HTTP upload API does not accept `content=snippets`, even when the
+storage supports snippets. Vendor-data delivery uses the context's explicit
+`snippetSSH` node-to-host mapping instead. Verify the node's SSH host key and
+configure an identity available to the controller process; the SSH user needs
+`pvesm path` and write access to the selected snippet directory. Uploads replace
+the destination atomically. See the [provider configuration](../README.md) for
+the context fields.
+
+A successful apply proves VM provisioning, not completion of first-boot
+cloud-init. Check cloud-init's final status and bootstrap results inside the
+guest, including its network and installed tools, before making a recovery
+snapshot. Keep warnings distinct from bootstrap failures; never infer readiness
+from the VM merely being running.
+
+For example, cloud-init 26.1 reports `degraded done` and exits 2 for the deprecated
+string `user` field generated by Proxmox 9.1.1, even with no bootstrap errors.
+Its JSON status distinguishes `status: done`, an empty `errors` list, and
+`recoverable_errors.DEPRECATED` warnings. Retain and report those warnings;
+accept a completed bootstrap only after checking its own readiness results.

internal/config/config.go

@@ -295,9 +295,10 @@ type TerraformResourceMap struct {
 
 // Context represents a provider context (like a cluster or endpoint)
 type Context struct {
-	Endpoint    string `yaml:"endpoint"`
-	Node        string `yaml:"node"`
-	Credentials string `yaml:"credentials"`
+	Endpoint    string                     `yaml:"endpoint"`
+	Node        string                     `yaml:"node"`
+	Credentials string                     `yaml:"credentials"`
+	SnippetSSH  *protocol.SnippetSSHConfig `yaml:"snippetSSH,omitempty"`
 }
 
 // Credential represents credentials for a provider
@@ -373,6 +374,16 @@ func (c *Config) GetProviderConfig(providerName, contextName string) (*protocol.
 		Defaults: provider.Defaults,
 	}
 
+	if ctx.SnippetSSH != nil {
+		snippetSSH := *ctx.SnippetSSH
+		identityFile, err := ExpandPath(snippetSSH.IdentityFile)
+		if err != nil {
+			return nil, fmt.Errorf("resolve snippet SSH identity file: %w", err)
+		}
+		snippetSSH.IdentityFile = identityFile
+		cfg.SnippetSSH = &snippetSSH
+	}
+
 	if ctx.Credentials != "" {
 		cred, ok := provider.Credentials[ctx.Credentials]
 		if !ok {

internal/config/config_test.go

@@ -3,6 +3,7 @@ package config
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -287,6 +288,107 @@ providers:
 	}
 }
 
+func TestGetProviderConfig_SnippetSSHIdentityRemainsPortable(t *testing.T) {
+	dir := t.TempDir()
+	configFile := filepath.Join(dir, "config.yaml")
+	content := `
+providers:
+  proxmox:
+    default-context: home
+    contexts:
+      home:
+        endpoint: https://pve.home:8006
+        snippetSSH:
+          hosts:
+            pve1: pve.home
+          identityFile: ~/.ssh/proxmox
+      remote:
+        endpoint: https://pve.remote:8006
+`
+	if err := os.WriteFile(configFile, []byte(content), 0600); err != nil {
+		t.Fatal(err)
+	}
+	cfg, err := LoadFromFile(configFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// The key need not exist: resolution passes a path to OpenSSH, not key bytes.
+	firstHome := filepath.Join(dir, "first-home")
+	t.Setenv("HOME", firstHome)
+	first, err := cfg.GetProviderConfig("proxmox", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if first.SnippetSSH == nil || first.SnippetSSH.IdentityFile != filepath.Join(firstHome, ".ssh/proxmox") {
+		t.Fatalf("identity was not resolved for this machine: %+v", first.SnippetSSH)
+	}
+	if err := cfg.SaveToFile(configFile); err != nil {
+		t.Fatal(err)
+	}
+	saved, err := os.ReadFile(configFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(string(saved), firstHome) {
+		t.Fatal("resolving a provider persisted a machine-specific identity path")
+	}
+
+	secondHome := filepath.Join(dir, "second-home")
+	t.Setenv("HOME", secondHome)
+	reloaded, err := LoadFromFile(configFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := reloaded.GetProviderConfig("proxmox", "home")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if second.SnippetSSH == nil || second.SnippetSSH.IdentityFile != filepath.Join(secondHome, ".ssh/proxmox") {
+		t.Fatalf("saved config is not portable to another home: %+v", second.SnippetSSH)
+	}
+	remote, err := reloaded.GetProviderConfig("proxmox", "remote")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if remote.SnippetSSH != nil {
+		t.Fatal("SSH access leaked into a context with no snippet transport configured")
+	}
+}
+
+func TestGetProviderConfig_SnippetSSHIdentityExpansionFailure(t *testing.T) {
+	dir := t.TempDir()
+	configFile := filepath.Join(dir, "config.yaml")
+	content := `
+providers:
+  proxmox:
+    contexts:
+      explicit:
+        snippetSSH:
+          hosts:
+            pve1: pve.home
+          identityFile: ~/.ssh/proxmox
+      agent:
+        snippetSSH:
+          hosts:
+            pve1: pve.home
+`
+	if err := os.WriteFile(configFile, []byte(content), 0600); err != nil {
+		t.Fatal(err)
+	}
+	cfg, err := LoadFromFile(configFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("HOME", "")
+	if resolved, err := cfg.GetProviderConfig("proxmox", "explicit"); err == nil || resolved != nil {
+		t.Fatalf("unresolvable explicit identity must fail, got config=%+v err=%v", resolved, err)
+	}
+	if _, err := cfg.GetProviderConfig("proxmox", "agent"); err != nil {
+		t.Fatalf("ambient SSH credentials must not require a home expansion: %v", err)
+	}
+}
+
 func TestExpandPath(t *testing.T) {
 	homeDir, err := os.UserHomeDir()
 	if err != nil {

pkg/protocol/request.go

@@ -24,6 +24,14 @@ type ProviderConfig struct {
 	TokenID     string            `json:"tokenId,omitempty"`
 	TokenSecret string            `json:"tokenSecret,omitempty"`
 	Defaults    map[string]string `json:"defaults,omitempty"`
+	SnippetSSH  *SnippetSSHConfig `json:"snippetSSH,omitempty"`
+}
+
+// SnippetSSHConfig configures SSH uploads to explicitly mapped Proxmox nodes.
+type SnippetSSHConfig struct {
+	Hosts        map[string]string `json:"hosts" yaml:"hosts"`
+	User         string            `json:"user,omitempty" yaml:"user,omitempty"`
+	IdentityFile string            `json:"identityFile,omitempty" yaml:"identityFile,omitempty"`
 }
 
 // Action constants

pkg/proxmox/client/client.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"io"
 	"maps"
-	"mime/multipart"
 	"net"
 	"net/http"
 	"net/url"
@@ -41,10 +40,13 @@ var ErrNotFound = errors.New("not found")
 
 // Client is a Proxmox API client
 type Client struct {
-	endpoint    string
-	tokenID     string
-	tokenSecret string
-	httpClient  *http.Client
+	endpoint               string
+	tokenID                string
+	tokenSecret            string
+	httpClient             *http.Client
+	snippetSSHHosts        map[string]string
+	snippetSSHUser         string
+	snippetSSHIdentityFile string
 }
 
 // New creates a new Proxmox client with API token authentication
@@ -1123,66 +1125,6 @@ func (c *Client) SnippetExists(ctx context.Context, node, storage, filename stri
 	return false, nil
 }
 
-// UploadSnippet uploads a snippet file to storage
-// This uses multipart form upload to the Proxmox API
-func (c *Client) UploadSnippet(ctx context.Context, node, storage, filename, content string) error {
-	path := fmt.Sprintf("/api2/json/nodes/%s/storage/%s/upload", node, storage)
-
-	debugf("UploadSnippet: uploading %s to %s:snippets/", filename, storage)
-
-	// Create multipart form body
-	var body bytes.Buffer
-	writer := multipart.NewWriter(&body)
-
-	// Add content type field (must be "snippets" for snippet files)
-	if err := writer.WriteField("content", "snippets"); err != nil {
-		return fmt.Errorf("failed to write content field: %w", err)
-	}
-
-	// Add file content as the "filename" form field
-	// Proxmox expects the file to be in the "filename" field with the filename as the form file name
-	part, err := writer.CreateFormFile("filename", filename)
-	if err != nil {
-		return fmt.Errorf("failed to create form file: %w", err)
-	}
-	if _, writeErr := part.Write([]byte(content)); writeErr != nil {
-		return fmt.Errorf("failed to write file content: %w", writeErr)
-	}
-
-	if closeErr := writer.Close(); closeErr != nil {
-		return fmt.Errorf("failed to close multipart writer: %w", closeErr)
-	}
-
-	// Create request
-	reqURL := c.endpoint + path
-	req, err := http.NewRequestWithContext(ctx, "POST", reqURL, &body)
-	if err != nil {
-		return fmt.Errorf("failed to create request: %w", err)
-	}
-
-	req.Header.Set("Authorization", fmt.Sprintf("PVEAPIToken=%s=%s", c.tokenID, c.tokenSecret))
-	req.Header.Set("Content-Type", writer.FormDataContentType())
-
-	resp, err := c.httpClient.Do(req)
-	if err != nil {
-		return fmt.Errorf("upload request failed: %w", err)
-	}
-	defer resp.Body.Close()
-
-	respBody, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return fmt.Errorf("failed to read response: %w", err)
-	}
-
-	debugf("UploadSnippet: response status %d, body: %s", resp.StatusCode, truncate(string(respBody), 200))
-
-	if resp.StatusCode >= 400 {
-		return fmt.Errorf("upload failed (status %d): %s", resp.StatusCode, string(respBody))
-	}
-
-	return nil
-}
-
 // QemuAgentSnippetName is the standard name for the qemu-agent enablement snippet
 const QemuAgentSnippetName = "openctl-qemu-agent.yaml"