diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dbac62..7c46a8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,37 @@ jobs: - name: Unit tests run: go test -race -count=1 ./... + - name: Portable check + run: | + go build -o bin/k3helper ./cmd/k3helper + scripts/portable-check.sh ./bin/k3helper + + # macOS is a first-class target — it is what most operators drive a cluster + # from — and nothing else in CI runs there. Darwin differs where it matters + # for this binary: its own temp directory layout, a different libc for the + # cgo-free build, and BSD userland for anything shelled out to. + macos: + name: macOS build + tests + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: go vet + run: go vet ./... + + - name: Unit tests + run: go test -race -count=1 ./... + + - name: Portable check + run: | + go build -o bin/k3helper ./cmd/k3helper + scripts/portable-check.sh ./bin/k3helper + build: name: Cross-compile runs-on: ubuntu-latest @@ -57,12 +88,95 @@ jobs: - name: Build all platforms run: make build-all + # The Windows assets must carry .exe or Windows will not execute them, + # and that suffix is applied in a shell loop rather than by the Go + # toolchain — so assert it rather than trusting the loop. + - name: Windows binaries carry .exe and are PE executables + run: | + for arch in amd64 arm64; do + f="bin/k3helper-windows-$arch.exe" + [ -f "$f" ] || { echo "::error::$f was not built"; exit 1; } + case "$(file -b "$f")" in + *"PE32"*|*"MS Windows"*) echo "✓ $f" ;; + *) echo "::error::$f is not a Windows executable: $(file -b "$f")"; exit 1 ;; + esac + done + - uses: actions/upload-artifact@v4 with: name: k3helper-binaries path: bin/ retention-days: 7 + # Windows is not just another cross-compile target here. The kubeconfig + # transport exists partly because Windows has no /bin/sh: it runs kubectl by + # argument rather than through a shell, and it stages manifests in the OS + # temp directory rather than /tmp. None of that is exercised by building on + # Linux — it has to actually run. + windows: + name: Windows build + tests + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: go vet + run: go vet ./... + + # -race needs a C toolchain; windows-latest has one, but the point of + # this job is the platform-specific paths rather than the race detector, + # which the Linux job already covers. + - name: Unit tests + run: go test -count=1 ./... + + - name: Build the .exe + run: go build -o k3helper.exe ./cmd/k3helper + + - name: The .exe actually runs + run: | + .\k3helper.exe version + .\k3helper.exe --help + + # The same portable check Linux and macOS run, so the three platforms are + # held to one standard rather than to whatever each job happened to test. + - name: Portable check + shell: bash + run: scripts/portable-check.sh ./k3helper.exe + + # The portable check above runs under the runner's Git Bash. This one + # deliberately does not: it drives the .exe from PowerShell, with native + # Windows paths and no POSIX shell anywhere in the picture, which is the + # environment a client actually double-clicks into. Native paths are also + # where a path check tends to quietly pass while being wrong. + - name: Runs from a bare directory under PowerShell + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path C:\portable | Out-Null + Copy-Item k3helper.exe C:\portable\ + Set-Location C:\portable + + & .\k3helper.exe init --kubeconfig C:\portable\missing.yaml --cluster demo + if ($LASTEXITCODE -eq 0) { throw "init accepted a kubeconfig that does not exist" } + if (Test-Path targets.yaml) { throw "init wrote a targets file after failing" } + + "apiVersion: v1" | Out-File -Encoding ascii C:\portable\kube.yaml + & .\k3helper.exe init --kubeconfig C:\portable\kube.yaml --cluster demo + if ($LASTEXITCODE -ne 0) { throw "init failed on a valid kubeconfig with a native path" } + Get-Content targets.yaml + + & .\k3helper.exe ctx + if ($LASTEXITCODE -ne 0) { throw "ctx failed" } + + - uses: actions/upload-artifact@v4 + with: + name: k3helper-windows-exe + path: k3helper.exe + retention-days: 7 + # Sandbox E2E on the container driver — three privileged systemd containers # standing in for three Linux hosts, which is what a GitHub-hosted runner can # actually provide. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5836f05..0fe3353 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -70,6 +70,26 @@ jobs: *) echo "::error::binary reports '$got', expected ${{ steps.tag.outputs.version }}"; exit 1 ;; esac + # Every asset install.sh can ask for has to be in the release, and the + # Windows ones have to carry .exe or Windows will not execute them. A + # missing or misnamed asset only surfaces when somebody's download 404s. + - name: Every platform asset is present and listed in checksums.txt + run: | + for asset in \ + k3helper-linux-amd64 k3helper-linux-arm64 \ + k3helper-darwin-amd64 k3helper-darwin-arm64 \ + k3helper-windows-amd64.exe k3helper-windows-arm64.exe + do + [ -f "dist/$asset" ] || { echo "::error::dist/$asset missing"; exit 1; } + grep -q " $asset\$" dist/checksums.txt \ + || { echo "::error::$asset not listed in checksums.txt"; exit 1; } + echo "✓ $asset" + done + case "$(file -b dist/k3helper-windows-amd64.exe)" in + *"PE32"*|*"MS Windows"*) ;; + *) echo "::error::the Windows asset is not a Windows executable"; exit 1 ;; + esac + # Pull the section for this version out of CHANGELOG.md. Falling back to # generated notes would quietly ship a commit list as the release notes, # so a missing section is an error instead. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4caf73b..9db0e5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,83 @@ Notable changes per release. The release workflow publishes the section matching the tag it is building, so this file is the source of the release notes on GitHub. +## v0.5.0 + +Clusters you cannot SSH into. k3helper now reaches a cluster either by its +nodes or by a kubeconfig, which is what a managed cluster (EKS, GKE, AKS, +Rancher, anything else that hands you credentials and keeps the machines) +actually gives you. + +### Install + +```bash +curl -fsSL https://raw.githubusercontent.com/solutionforest/k3helper/main/install.sh | sh +``` + +No breaking changes from v0.4.0. Existing targets files keep working exactly +as they did. + +### Added + +- **`kubeconfig:` clusters.** A targets file entry with `kubeconfig:` (and an + optional `kube_context:`) instead of `nodes:` drives a local `kubectl` + against the API server. `k3helper init --kubeconfig ~/.kube/config + --kube-context prod` writes one. +- **Everything that works through the API server works there**: `doctor`'s + cluster and workload signatures, `deploy`, `verify --dry-run-server`, `gen`, + and the TUI including logs, describe and port-forward. +- **`k3helper ctx` gained a REACHED column**, so `0 nodes` reads as "reached by + kubeconfig" rather than as a broken file. One file can mix both kinds. +- **Windows binaries, released.** `k3helper-windows-amd64.exe` and + `k3helper-windows-arm64.exe` are built by `make release`, listed in + `checksums.txt`, and published with every tag. One file, no installer, + nothing written outside the folder you put it in. The kubeconfig transport + runs kubectl by argument rather than through a shell, so it needs no + `/bin/sh`. +- **`make portable-check`**, and CI runs it on Linux, macOS and Windows. + "Single portable binary" is a claim about the runtime rather than the build, + so it is checked by running the binary from a directory it has never seen: + it must start, explain a missing targets file, refuse a kubeconfig that is + not there without writing anything, complete the kubeconfig and SSH init + flows, generate and verify YAML with no cluster and no network, and leave + nothing behind in `$HOME`. +- **A macOS CI job.** macOS is what most operators drive a cluster from and + nothing in CI ran there before; it now gets vet, race tests and the portable + check on every push, the same as Linux. + +### Fixed + +- **`doctor` now catches a crash loop it happens to sample mid-restart.** A + container in backoff only reads as CrashLoopBackOff while it is waiting + between attempts; the moment it starts again the pod is Running with no + reason attached, and the diagnosis saw nothing worse than an unready pod. + Restart counts were already parsed and thrown away — they are now kept, and a + pod that is not ready after three or more restarts is called a crash loop + whichever half of the cycle we caught. This is why the fault matrix could + catch that fault or miss it depending on timing. + +### Changed + +- **`doctor` separates scope notes from faults.** A finding that describes what + could *not* be looked at — incomplete evidence, or a missing host layer — is + printed under "Notes on what was looked at" and no longer sets the exit code. + A healthy managed cluster exits 0 instead of 2. +- **`check` and the TUI build their host check list from one place** + (`check.HostChecks`), so a check added to one cannot go missing from the + other. + +### Not available on a kubeconfig cluster + +By nature, not by omission — a kubeconfig reaches the API server, not the +machines behind it: + +- Host-layer checks: disk, memory, swap, cgroups, systemd units, container + runtime, clock skew. `doctor` reports the gap rather than implying those are + fine; `check` says it has nothing to check. +- `vm setup` and `registry apply`. Both write files on the nodes, and both + refuse with an explanation rather than doing half the job. +- Certificate expiry, which reads `k3s certificate check` on the node. + ## v0.4.0 Private registries: declared once, applied to every node, and diagnosed diff --git a/Makefile b/Makefile index 068dde2..008a10f 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,14 @@ BINARY := k3helper -VERSION := 0.4.0 +VERSION := 0.5.0 VERPKG := github.com/solutionforest/k3helper/internal/cli.version LDFLAGS := -ldflags "-X $(VERPKG)=$(VERSION)" # -s -w strips the symbol table and DWARF: ~25% smaller downloads, and Go # panics keep their function names because the runtime carries its own tables. RELFLAGS := -ldflags "-s -w -X $(VERPKG)=$(VERSION)" -PLATFORMS := linux/amd64 linux/arm64 darwin/arm64 darwin/amd64 +PLATFORMS := linux/amd64 linux/arm64 darwin/arm64 darwin/amd64 windows/amd64 windows/arm64 +# Windows will not execute a downloaded file without the extension, so those +# two assets carry .exe. Everything else stays extensionless; install.sh builds +# the asset name the same way. DIST := dist TARGETS := test/sandbox/targets.sandbox.yaml SSH_KEY := test/sandbox/ssh/id_ed25519 @@ -14,7 +17,7 @@ SSH_OPTS := -i $(SSH_KEY) -o StrictHostKeyChecking=no -o UserKnownHostsFile=/d # Sandbox nodes are OrbStack Linux VMs (NOT Docker containers — OrbStack # containers share the macOS kernel and kubelet PLEG kills pods falsely). -.PHONY: help build build-all test test-integration clean e2e e2e-fast e2e-quick \ +.PHONY: help build build-all test test-integration portable-check clean e2e e2e-fast e2e-quick \ sandbox-up sandbox-down sandbox-reset sandbox-verify sandbox-ssh \ bootstrap check doctor fault-clean fault-list fault-check-all release release-upload bundle @@ -42,21 +45,30 @@ help: @echo " make fault-list list faults + the signature each should trigger" @echo " make fault- inject one fault (e.g. make fault-oom)" @echo " make fault-check-all inject every fault, assert doctor catches each" + @echo " make portable-check run the built binary from a bare directory" @echo " make fault-clean undo injected faults" build: go build $(LDFLAGS) -o bin/$(BINARY) ./cmd/k3helper build-all: - @for platform in linux/amd64 linux/arm64 darwin/arm64 darwin/amd64; do \ - GOOS=$${platform%/*} GOARCH=$${platform#*/} go build $(LDFLAGS) \ - -o bin/$(BINARY)-$${platform%/*}-$${platform#*/} ./cmd/k3helper; \ - echo "✓ bin/$(BINARY)-$${platform%/*}-$${platform#*/}"; \ + @for platform in $(PLATFORMS); do \ + os=$${platform%/*}; arch=$${platform#*/}; ext=""; \ + [ "$$os" = windows ] && ext=".exe"; \ + GOOS=$$os GOARCH=$$arch go build $(LDFLAGS) \ + -o bin/$(BINARY)-$$os-$$arch$$ext ./cmd/k3helper || exit 1; \ + echo "✓ bin/$(BINARY)-$$os-$$arch$$ext"; \ done test: go test ./... +# "Single portable binary" is a runtime claim, so it is checked by running the +# binary from a directory it has never seen. CI runs this on Linux, macOS and +# Windows against the same script. +portable-check: build + @scripts/portable-check.sh ./bin/$(BINARY) + test-integration: go test -tags=integration ./... @@ -79,9 +91,10 @@ e2e-quick: release: @rm -rf $(DIST) && mkdir -p $(DIST) @for platform in $(PLATFORMS); do \ - os=$${platform%/*}; arch=$${platform#*/}; \ - GOOS=$$os GOARCH=$$arch go build $(RELFLAGS) -o $(DIST)/$(BINARY)-$$os-$$arch ./cmd/k3helper || exit 1; \ - echo "✓ $(DIST)/$(BINARY)-$$os-$$arch"; \ + os=$${platform%/*}; arch=$${platform#*/}; ext=""; \ + [ "$$os" = windows ] && ext=".exe"; \ + GOOS=$$os GOARCH=$$arch go build $(RELFLAGS) -o $(DIST)/$(BINARY)-$$os-$$arch$$ext ./cmd/k3helper || exit 1; \ + echo "✓ $(DIST)/$(BINARY)-$$os-$$arch$$ext"; \ done @cd $(DIST) && (command -v sha256sum >/dev/null && sha256sum $(BINARY)-* || shasum -a 256 $(BINARY)-*) > checksums.txt @cp install.sh $(DIST)/install.sh diff --git a/PLAN.md b/PLAN.md index 409dcfc..1cf9b5a 100644 --- a/PLAN.md +++ b/PLAN.md @@ -395,10 +395,50 @@ scripts/bundle.sh 11. **Linux CI sandbox** — the container driver runs the full E2E. ✅ 12. **Private registries** — targets-file declaration, `registry apply`, pull-secret generation, node-side check, three diagnosis signatures. ✅ +13. **Kubeconfig transport** — clusters reached through their API server + instead of by SSH, so managed Kubernetes is in scope. ✅ Remaining: nothing from the original scope. Open items are new asks, tracked under "Open questions" below. +### 8. Kubeconfig transport (#13) + +**Why.** Every layer below the cluster assumed SSH into a node and a kubectl +living there. That covers k3s and kubeadm and excludes every managed cluster, +which is what most clients run: EKS/GKE/AKS hand out a kubeconfig and keep the +machines. The cluster layer never needed the machines — only the transport did. + +**Shape.** A `Targets` entry carries either `nodes:` or `kubeconfig:`, and +`Targets.Mode()` says which. `internal/transport` is the one place that opens +connections; `internal/kubectl` is the local transport. + +**Why the local transport parses instead of shelling out.** The codebase builds +shell command strings. `internal/ssh/local.go` runs them through `/bin/sh`, +which Windows does not have — and the portable `.exe` is a deliverable. So +`kubectl.Local.Run` asserts the command is one of its own kubectl invocations, +strips the trailing redirection, splits the arguments honouring the single +quoting `kube.shellQuote` produces, and execs kubectl directly. Anything else — +`df`, `systemctl`, `free`, the k3s probes — is refused with exit 127. That +refusal is the feature: host probes fail where they are written, with a reason, +instead of somewhere further down. + +**Layers.** `check.HostChecks` is the single host-check list both `check` and +the TUI build from. `Evidence.HostLayerUnavailable` is deliberately neither a +`ProbeError` nor an `UnreachableNode`: both of those mean "we tried and failed", +which is a fault, while this means "there was never anything to try", which is +how managed clusters work. Conflating them reports every healthy EKS cluster as +degraded. `Diagnosis.Informational()` keeps those notes out of the exit code. + +**Refusals.** `vm setup` and `registry apply` need machines. They stop with an +explanation rather than doing half the job — a registry configured on some +nodes and not others is the exact failure the cluster-level registry block +exists to prevent. + +**Proof.** Unit tests parse the real command strings from `gather.go` and +`deploy.go`; `exec_test.go` runs the whole path against a stub kubectl on PATH; +the integration suite reads the sandbox twice, once over SSH and once through +its kubeconfig, and asserts the two agree. + ## Research — similar tools | Tool | What it does | Covers our reqs | Gap | diff --git a/README.md b/README.md index 2e5a89a..fc84f7f 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,32 @@ chmod +x k3helper ./k3helper version ``` -Assets: `k3helper-linux-amd64`, `k3helper-linux-arm64`, `k3helper-darwin-amd64`, `k3helper-darwin-arm64`. Every release also ships `checksums.txt`. +Assets: `k3helper-linux-amd64`, `k3helper-linux-arm64`, `k3helper-darwin-amd64`, `k3helper-darwin-arm64`, `k3helper-windows-amd64.exe`, `k3helper-windows-arm64.exe`. Every release also ships `checksums.txt`. + +### Windows — one .exe, nothing installed + +Download it, put it where you like, run it. No installer, no registry keys, no +runtime, nothing written outside the folder you put it in. + +```powershell +curl.exe -fLo k3helper.exe ^ + https://github.com/solutionforest/k3helper/releases/latest/download/k3helper-windows-amd64.exe +.\k3helper.exe version +``` + +On Windows on ARM take `k3helper-windows-arm64.exe` instead. Verify it against +the release manifest if you want to: + +```powershell +curl.exe -fLO https://github.com/solutionforest/k3helper/releases/latest/download/checksums.txt +Get-FileHash k3helper.exe -Algorithm SHA256 # compare with the line in checksums.txt +``` + +Windows reaches clusters through a kubeconfig rather than over SSH — see +[Clusters you cannot SSH into](#clusters-you-cannot-ssh-into) — so it needs +`kubectl` on PATH and nothing else. SSH mode works too if your nodes are +reachable, but the host-layer commands (`vm setup`, `registry apply`) target +Linux nodes. ### From source @@ -632,18 +657,74 @@ Back on the server, `k3s kubectl get nodes` should show them joining. You lose ` ## k3s vs k8s support -| Area | k3s | kubeadm / other k8s | -|---|---|---| -| verify / gen | ✅ | ✅ (pure YAML) | -| deploy / doctor cluster layer | ✅ | ✅ (kubectl-based) | -| check host service layer | ✅ (`k3s`/`k3s-agent`) | ✅ (`kubelet` + `containerd`) | -| kubeconfig discovery | ✅ `/etc/rancher/k3s/k3s.yaml` | ✅ `/etc/kubernetes/admin.conf` | -| certificate expiry | ✅ (`k3s certificate check`) | ❌ k3s-specific command | -| vm setup | ✅ (incl. HA) | ✅ single control plane (`--distro kubeadm`) | +| Area | k3s | kubeadm / other k8s | managed (EKS/GKE/AKS) | +|---|---|---|---| +| verify / gen | ✅ | ✅ (pure YAML) | ✅ | +| deploy / doctor cluster layer | ✅ | ✅ (kubectl-based) | ✅ | +| TUI browser, logs, port-forward | ✅ | ✅ | ✅ | +| check host service layer | ✅ (`k3s`/`k3s-agent`) | ✅ (`kubelet` + `containerd`) | ❌ no host access | +| kubeconfig discovery | ✅ `/etc/rancher/k3s/k3s.yaml` | ✅ `/etc/kubernetes/admin.conf` | ✅ your own kubeconfig | +| certificate expiry | ✅ (`k3s certificate check`) | ❌ k3s-specific command | ❌ provider's job | +| registry apply | ✅ | ✅ | ❌ writes files on nodes | +| vm setup | ✅ (incl. HA) | ✅ single control plane (`--distro kubeadm`) | ❌ provider's job | + +For self-managed clusters the distribution is detected per node from its unit +files, so a mixed inventory works: kubectl is invoked through whichever +kubeconfig exists, and the service check looks for the units that distribution +actually installs. + +### Clusters you cannot SSH into + +A managed cluster hands you a kubeconfig and nothing else — no control-plane +node to log into, no `admin.conf` to read. Describe it by that kubeconfig +instead of by nodes: + +```bash +k3helper init --kubeconfig ~/.kube/config --kube-context prod-admin --cluster client-prod +``` + +```yaml +cluster: client-prod +kubeconfig: ~/.kube/config +kube_context: prod-admin # optional; defaults to the file's current-context +``` + +Then everything that works through the API server works normally: + +```bash +k3helper doctor # cluster + workload signatures, ranked, with fixes +k3helper deploy -f app.yaml +k3helper verify --dry-run-server app.yaml +k3helper tui # browser, logs, describe, port-forward +``` + +Two notes on what this cannot do: + +- **The host layer is not there to look at.** Disk pressure, swap, cgroups, + systemd units, the container runtime and certificate expiry are not checked. + `doctor` says so as a note on the scope of the diagnosis rather than staying + quiet, and it does not affect the exit code — a healthy managed cluster exits + 0. `check` explains that it has nothing to check and stops. +- **`vm setup` and `registry apply` refuse.** Both write files on the machines + themselves. On a managed cluster that is the provider's job. + +`kube_context` is deliberately not called `context`: `--context` already +selects which cluster to use from a multi-cluster targets file, and one file +can hold both kinds. + +```yaml +clusters: + - cluster: client-prod # managed, reached through its API server + kubeconfig: ~/.kube/client-prod.yaml + - cluster: lab # self-managed, reached over SSH + nodes: + - {name: server, role: server, host: 10.0.0.10, user: ubuntu, key: ~/.ssh/id_ed25519} +current: client-prod +``` -The distribution is detected per node from its unit files, so a mixed -inventory works: kubectl is invoked through whichever kubeconfig exists, and -the service check looks for the units that distribution actually installs. +`k3helper ctx` prints which is which. This mode drives your local `kubectl`, so +it has to be installed — k3helper says so at the front door rather than part +way through a diagnosis. ## Testing @@ -657,6 +738,7 @@ make e2e # full lifecycle, ~10-15 min make e2e-fast # reuse running sandbox, ~8 min make e2e-quick # reuse sandbox, skip the fault sweep, ~4 min make test # unit tests +make portable-check # run the built binary from a bare directory make test-integration make fault-list # every fault + the signature it should trigger make fault- # inject one fault, e.g. make fault-oom @@ -761,6 +843,12 @@ Current, and worth knowing before pointing this at production: - **Host-layer checks assume systemd.** Nodes without it (some minimal or container-based images) get cluster-layer findings only. +- **A kubeconfig cluster has no host layer at all.** That is the nature of a + managed cluster rather than a gap here, but it means roughly half the + signatures cannot fire. See "Clusters you cannot SSH into". +- **Kubeconfig mode shells out to `kubectl`.** It is driven by argument, not + through a shell, so it works the same on Windows — but kubectl has to be on + PATH. - **`doctor` reads the cluster through the first server node.** If that node is down, cluster-layer evidence is unavailable even when other servers are up. - **Certificate expiry is read via `k3s certificate check`**, so it is not diff --git a/install.sh b/install.sh index b43e6b2..19138ae 100755 --- a/install.sh +++ b/install.sh @@ -49,6 +49,19 @@ fi OS="${K3HELPER_OS:-$(uname -s | tr '[:upper:]' '[:lower:]')}" case "$OS" in linux|darwin) ;; + # Git Bash, MSYS2 and Cygwin all report their own kernel name. They can run + # this script, but "install to /usr/local/bin" means something different + # under each of them and none of it is on the Windows PATH. Windows ships a + # single self-contained .exe instead — say so rather than installing + # something the user cannot then run. + mingw*|msys*|cygwin*|windows*) + log "k3helper on Windows is a single .exe — no installer needed." + log "" + log " https://github.com/$REPO/releases/latest/download/k3helper-windows-amd64.exe" + log "" + log "Download it, put it anywhere, and run it: k3helper-windows-amd64.exe version" + log "(On Windows on ARM, take k3helper-windows-arm64.exe instead.)" + exit 0 ;; *) fail "unsupported OS: $OS" ;; esac @@ -75,6 +88,10 @@ else fi ASSET="${BIN_NAME}-${OS}-${ARCH}" +# The Windows assets carry an extension because Windows will not execute a file +# without one. Reachable only through K3HELPER_OS=windows, since a native +# Windows shell does not run this script. +[ "$OS" = windows ] && ASSET="${ASSET}.exe" TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT INT TERM diff --git a/internal/check/distro_test.go b/internal/check/distro_test.go index 18efe1c..24b54c4 100644 --- a/internal/check/distro_test.go +++ b/internal/check/distro_test.go @@ -141,20 +141,55 @@ func TestK3sServiceCheckDistinguishesStoppedFromMissing(t *testing.T) { } } -// check and the TUI must use the distro-aware ServiceCheck. Wiring the -// k3s-only predecessor told the operator of a healthy kubeadm node to install -// a second Kubernetes distribution. -func TestCheckWiringUsesServiceCheck(t *testing.T) { +// The host runner must use the distro-aware ServiceCheck. Wiring the k3s-only +// predecessor told the operator of a healthy kubeadm node to install a second +// Kubernetes distribution. +// +// HostChecks is now the one list both `check` and the TUI build from, so this +// asserts against the list itself rather than grepping each call site. +func TestHostChecksUseDistroAwareServiceCheck(t *testing.T) { + var found bool + for _, c := range HostChecks("server") { + if _, isK3sOnly := c.(K3sServiceCheck); isK3sOnly { + t.Error("HostChecks wires the k3s-only K3sServiceCheck; use ServiceCheck") + } + if _, ok := c.(ServiceCheck); ok { + found = true + } + } + if !found { + t.Error("HostChecks does not run a service check at all") + } +} + +// Both callers must build their runner from HostChecks, or a check added to +// the list silently goes missing from one of them. +func TestCheckWiringUsesHostChecks(t *testing.T) { for _, path := range []string{"../cli/check_cmd.go", "../tui/tui.go"} { src, err := os.ReadFile(path) if err != nil { t.Fatalf("read %s: %v", path, err) } - if strings.Contains(string(src), "K3sServiceCheck{") { - t.Errorf("%s still wires the k3s-only K3sServiceCheck; use ServiceCheck", path) + if !strings.Contains(string(src), "check.HostChecks(") { + t.Errorf("%s builds its own check list instead of using check.HostChecks", path) + } + } +} + +// A kubeconfig cluster reports every host check as skipped, with the reason. +// Reporting nothing would read as a clean bill of health for a layer that was +// never looked at. +func TestSkippedHostResultsCoverEveryHostCheck(t *testing.T) { + results := SkippedHostResults("server") + if len(results) != len(HostChecks("server")) { + t.Fatalf("got %d skipped results for %d host checks", len(results), len(HostChecks("server"))) + } + for _, r := range results { + if r.Status != Skip { + t.Errorf("%s: status %v, want Skip — a managed cluster is not unhealthy for having no SSH", r.ID, r.Status) } - if !strings.Contains(string(src), "ServiceCheck{") { - t.Errorf("%s does not run a service check at all", path) + if !strings.Contains(r.Summary, "kubeconfig") { + t.Errorf("%s: summary %q does not say why it was skipped", r.ID, r.Summary) } } } diff --git a/internal/check/layers.go b/internal/check/layers.go new file mode 100644 index 0000000..472e6ea --- /dev/null +++ b/internal/check/layers.go @@ -0,0 +1,45 @@ +package check + +// This file decides which checks can run against which kind of cluster. +// +// Every check in this package reads the machine: disks, memory, swap, cgroups, +// systemd units, the registry configuration files. A cluster reached through a +// kubeconfig has no machine to read — the API server answers, and the nodes +// behind it belong to a cloud provider. Rather than let those checks run and +// fail, they are reported as skipped with the reason attached. + +// HostReason explains why the host layer was not checked. It is a Skip, never +// a Fail: a managed cluster is not unhealthy for declining to give out SSH. +const HostReason = "host layer not available on a kubeconfig cluster (no SSH to the nodes)" + +// HostChecks are the checks that need a machine to run on. This is the single +// list; `check` and the TUI both build their runner from it, so a check added +// in one place cannot go missing from the other. +func HostChecks(role string) []Check { + return []Check{ + DiskUsageCheck{}, + MemoryCheck{}, + SwapCheck{}, + CgroupCheck{}, + ServiceCheck{Role: role}, + RegistryCheck{}, + } +} + +// SkippedHostResults returns one Skip result per host check, so that a +// kubeconfig cluster reports the same rows as an SSH one with an honest +// verdict in them. Printing nothing at all would read as "all clear". +func SkippedHostResults(role string) []Result { + checks := HostChecks(role) + out := make([]Result, 0, len(checks)) + for _, c := range checks { + out = append(out, Result{ + ID: c.ID(), + Category: c.Category(), + Name: c.Name(), + Status: Skip, + Summary: HostReason, + }) + } + return out +} diff --git a/internal/cli/check_cmd.go b/internal/cli/check_cmd.go index 5b3b6de..e11c65d 100644 --- a/internal/cli/check_cmd.go +++ b/internal/cli/check_cmd.go @@ -6,6 +6,7 @@ import ( "text/tabwriter" "github.com/solutionforest/k3helper/internal/check" + "github.com/solutionforest/k3helper/internal/config" "github.com/solutionforest/k3helper/internal/ssh" "github.com/spf13/cobra" ) @@ -29,6 +30,16 @@ dead k3s does.`, if err != nil { return err } + if targets.Mode() == config.ModeKubeconfig { + // Every check in this command reads the machine. Say so once, + // plainly, instead of printing a screen of skipped rows for a + // cluster that was never going to have them. + fmt.Fprintf(cmd.OutOrStdout(), + "cluster %q is reached through a kubeconfig: %s\n\n"+ + "Run `k3helper doctor` for the checks that work through the API server.\n", + targets.Cluster, check.HostReason) + return nil + } anyFail, anyWarn := false, false for _, node := range targets.Nodes { client, err := ssh.Dial(node.SSH()) @@ -37,14 +48,7 @@ dead k3s does.`, anyFail = true continue } - runner := check.NewRunner( - check.DiskUsageCheck{}, - check.MemoryCheck{}, - check.SwapCheck{}, - check.CgroupCheck{}, - check.ServiceCheck{Role: node.Role}, - check.RegistryCheck{}, - ) + runner := check.NewRunner(check.HostChecks(node.Role)...) results := runner.RunAll(check.Context{Exec: execAdapter{client}, Node: node.Name}) client.Close() diff --git a/internal/cli/ctx_cmd.go b/internal/cli/ctx_cmd.go index 85fd7a6..6fbfe8a 100644 --- a/internal/cli/ctx_cmd.go +++ b/internal/cli/ctx_cmd.go @@ -24,7 +24,10 @@ func newCtxCmd() *cobra.Command { return err } w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 4, 2, ' ', 0) - fmt.Fprintln(w, "\tCLUSTER\tNODES\tSERVER") + // REACHED is worth a column of its own: it decides which commands + // are available, and "0 nodes" alone looks like a broken file + // rather than a kubeconfig cluster. + fmt.Fprintln(w, "\tCLUSTER\tREACHED\tNODES\tSERVER") for i := range f.Clusters { c := &f.Clusters[i] marker := " " @@ -32,13 +35,18 @@ func newCtxCmd() *cobra.Command { marker = "*" } server := "-" - if s, err := c.Server(); err == nil { + if c.Mode() == config.ModeKubeconfig { + server = c.Kubeconfig + if c.KubeContext != "" { + server += " (" + c.KubeContext + ")" + } + } else if s, err := c.Server(); err == nil { server = s.Host if s.Local { server = "local" } } - fmt.Fprintf(w, "%s\t%s\t%d\t%s\n", marker, c.Cluster, len(c.Nodes), server) + fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\n", marker, c.Cluster, c.Mode(), len(c.Nodes), server) } w.Flush() if len(f.Clusters) > 1 { diff --git a/internal/cli/deploy_cmd.go b/internal/cli/deploy_cmd.go index ddc1596..0360716 100644 --- a/internal/cli/deploy_cmd.go +++ b/internal/cli/deploy_cmd.go @@ -8,7 +8,7 @@ import ( "github.com/charmbracelet/lipgloss" "github.com/solutionforest/k3helper/internal/deploy" - "github.com/solutionforest/k3helper/internal/ssh" + "github.com/solutionforest/k3helper/internal/transport" "github.com/spf13/cobra" ) @@ -32,14 +32,10 @@ func newDeployCmd() *cobra.Command { if err != nil { return err } - srv, err := targets.Server() + server, err := transport.Server(targets) if err != nil { return err } - server, err := ssh.Dial(toSSHNode(*srv)) - if err != nil { - return fmt.Errorf("connect to server: %w", err) - } defer server.Close() res, err := deploy.Deploy(server, manifest, deploy.Options{ diff --git a/internal/cli/doctor_cmd.go b/internal/cli/doctor_cmd.go index cd3235e..bcd58f2 100644 --- a/internal/cli/doctor_cmd.go +++ b/internal/cli/doctor_cmd.go @@ -12,7 +12,8 @@ import ( "text/tabwriter" "time" - "github.com/solutionforest/k3helper/internal/ssh" + "github.com/solutionforest/k3helper/internal/config" + "github.com/solutionforest/k3helper/internal/transport" "github.com/solutionforest/k3helper/internal/troubleshoot" "github.com/spf13/cobra" ) @@ -62,10 +63,8 @@ func newDoctorCmd() *cobra.Command { if err := enc.Encode(report); err != nil { return err } - for _, d := range diagnoses { - if d.SignatureID != "cluster.partial-evidence" { - os.Exit(2) - } + if !troubleshoot.OnlyInformational(diagnoses) { + os.Exit(2) } return nil } @@ -94,24 +93,26 @@ func newDoctorCmd() *cobra.Command { fmt.Fprintln(out, "✓ no issues detected — cluster looks healthy") return nil } + // Scope notes are not faults. Leading with "1 likely issue" for a + // healthy managed cluster is how a working setup gets reported to + // the client as broken. + if troubleshoot.OnlyInformational(diagnoses) { + w.Flush() + fmt.Fprintln(out, "✓ no issues detected — cluster looks healthy") + fmt.Fprintln(out, "\nNotes on what was looked at:") + for _, d := range diagnoses { + fmt.Fprintf(w, " \t%s\n", d.Title) + fmt.Fprintf(w, " \t %s\n", wrapText(d.Remediation, " \t ")) + } + w.Flush() + return nil + } fmt.Fprintf(out, "Found %d likely issue(s), ranked by confidence:\n\n", len(diagnoses)) for i, d := range diagnoses { fmt.Fprintf(w, "%d.\t[%d%%]\t%s\n", i+1, d.Confidence, d.Title) fmt.Fprintf(w, " \tfix:\t%s\n", wrapText(d.Remediation, " \t ")) } w.Flush() - // Incomplete evidence on its own is a caveat, not a fault: exiting - // non-zero for it alone would turn every RBAC-scoped kubeconfig - // into a red CI run. - onlyPartial := true - for _, d := range diagnoses { - if d.SignatureID != "cluster.partial-evidence" { - onlyPartial = false - } - } - if onlyPartial { - return nil - } os.Exit(2) return nil }, @@ -216,34 +217,22 @@ func diagnoseOnce(targetsPath string) (diagnosis, error) { if err != nil { return diagnosis{}, err } - srvNode, err := targets.Server() + server, err := transport.Server(targets) if err != nil { return diagnosis{}, err } - server, err := ssh.Dial(srvNode.SSH()) - if err != nil { - return diagnosis{}, fmt.Errorf("connect to server: %w", err) - } defer server.Close() - // Every node gets its own connection except the one already dialled for + // Every node gets its own connection except the one already opened for // kubectl. A node we cannot reach is evidence, not something to skip: // silently dropping it lets doctor report a healthy cluster while a node - // is down. - hosts := map[string]ssh.Executor{} + // is down. A kubeconfig cluster has no nodes here at all, which is a + // different thing and is recorded as such. + hosts, failures, closeHosts := transport.Hosts(targets, server, transport.ServerName(targets)) + defer closeHosts() var unreachable []troubleshoot.UnreachableNode - for _, n := range targets.Nodes { - if n.Name == srvNode.Name { - hosts[n.Name] = server - continue - } - c, err := ssh.Dial(n.SSH()) - if err != nil { - unreachable = append(unreachable, troubleshoot.UnreachableNode{Name: n.Name, Reason: err.Error()}) - continue - } - defer c.Close() - hosts[n.Name] = c + for _, f := range failures { + unreachable = append(unreachable, troubleshoot.UnreachableNode{Name: f.Name, Reason: f.Reason}) } var serverNames []string @@ -252,6 +241,7 @@ func diagnoseOnce(targetsPath string) (diagnosis, error) { } g := troubleshoot.Gatherer{ Server: server, Hosts: hosts, Unreachable: unreachable, ServerNodes: serverNames, + NoHostLayer: targets.Mode() == config.ModeKubeconfig, } evidence := g.Collect() return diagnosis{ diff --git a/internal/cli/init_cmd.go b/internal/cli/init_cmd.go index 6e1aff5..915f190 100644 --- a/internal/cli/init_cmd.go +++ b/internal/cli/init_cmd.go @@ -7,21 +7,24 @@ import ( "strings" "github.com/solutionforest/k3helper/internal/config" + "github.com/solutionforest/k3helper/internal/transport" "github.com/spf13/cobra" "sigs.k8s.io/yaml" ) func newInitCmd() *cobra.Command { var ( - outPath string - cluster string - server string - agents []string - user string - key string - local bool - force bool - insecure bool + outPath string + cluster string + server string + agents []string + user string + key string + local bool + force bool + insecure bool + kubeconfig string + kubeContext string ) cmd := &cobra.Command{ Use: "init", @@ -36,6 +39,9 @@ Examples: # this machine, no SSH (provider browser console) k3helper init --local + # a managed cluster (EKS/GKE/AKS): no SSH, driven through kubectl + k3helper init --kubeconfig ~/.kube/config --kube-context prod + # a template to fill in by hand k3helper init`, RunE: func(cmd *cobra.Command, args []string) error { @@ -43,14 +49,14 @@ Examples: return fmt.Errorf("%s already exists; pass --force to overwrite it", outPath) } - targets, err := buildTargets(cluster, server, agents, user, key, local, insecure) + targets, err := buildTargets(cluster, server, agents, user, key, local, insecure, kubeconfig, kubeContext) if err != nil { return err } // Render before validating: Validate fills in defaults (it sets a // local node's host to "localhost"), and echoing those back into // the file would clutter it with values the user never chose. - body := initHeader(local) + renderTargets(targets) + body := initHeader(local, kubeconfig != "") + renderTargets(targets) // A targets file that cannot be loaded back is worse than none, so // validate a copy and then re-parse what we are about to write. @@ -70,6 +76,11 @@ Examples: out := cmd.OutOrStdout() fmt.Fprintf(out, "✓ wrote %s\n\n", outPath) + if targets.Mode() == config.ModeKubeconfig { + fmt.Fprintf(out, " %-8s %s\n", targets.Cluster, transport.ServerName(targets)) + fmt.Fprintf(out, "\nNext: k3helper doctor -t %s\n", outPath) + return nil + } for _, n := range targets.Nodes { where := n.Host if n.Local { @@ -88,6 +99,10 @@ Examples: cmd.Flags().StringVarP(&user, "user", "u", "root", "SSH user for every node") cmd.Flags().StringVarP(&key, "key", "k", "~/.ssh/id_ed25519", "SSH private key for every node") cmd.Flags().BoolVar(&local, "local", false, "single node: the machine k3helper runs on, no SSH") + cmd.Flags().StringVar(&kubeconfig, "kubeconfig", "", + "reach the cluster through this kubeconfig instead of over SSH (managed clusters)") + cmd.Flags().StringVar(&kubeContext, "kube-context", "", + "context to use inside --kubeconfig (default: the file's current-context)") cmd.Flags().BoolVar(&insecure, "insecure-host-key", false, "skip SSH host key verification for these nodes") cmd.Flags().BoolVar(&force, "force", false, "overwrite an existing file") return cmd @@ -95,9 +110,20 @@ Examples: // buildTargets assembles the cluster from the flags, falling back to a // placeholder template when no addresses were given. -func buildTargets(cluster, server string, agents []string, user, key string, local, insecure bool) (*config.Targets, error) { +func buildTargets(cluster, server string, agents []string, user, key string, local, insecure bool, + kubeconfig, kubeContext string) (*config.Targets, error) { t := &config.Targets{Cluster: cluster} + if kubeconfig != "" { + if local || server != "" || len(agents) > 0 { + return nil, fmt.Errorf("--kubeconfig reaches the cluster through its API server; " + + "do not combine it with --local/--server/--agent, which describe machines to SSH into") + } + t.Kubeconfig = kubeconfig + t.KubeContext = kubeContext + return t, nil + } + if local { if server != "" || len(agents) > 0 { return nil, fmt.Errorf("--local describes a single node; do not combine it with --server/--agent") @@ -180,6 +206,13 @@ func splitHostPort(s string) (string, int, error) { func renderTargets(t *config.Targets) string { var b strings.Builder fmt.Fprintf(&b, "cluster: %s\n", t.Cluster) + if t.Kubeconfig != "" { + fmt.Fprintf(&b, "kubeconfig: %s\n", t.Kubeconfig) + if t.KubeContext != "" { + fmt.Fprintf(&b, "kube_context: %s\n", t.KubeContext) + } + return b.String() + } b.WriteString("nodes:\n") for _, n := range t.Nodes { fmt.Fprintf(&b, " - name: %s\n", n.Name) @@ -201,7 +234,7 @@ func renderTargets(t *config.Targets) string { return b.String() } -func initHeader(local bool) string { +func initHeader(local, kubeconfig bool) string { h := `# k3helper targets — the machines this cluster runs on. # # role: server | agent (at least one server) @@ -230,6 +263,31 @@ func initHeader(local bool) string { h += `# # local: true means "the machine k3helper is running on" — commands run # through /bin/sh instead of SSH, so no host, user or key is needed. +` + } + if kubeconfig { + h = `# k3helper targets — a cluster reached through its API server. +# +# kubeconfig: path on THIS machine; ~ is expanded +# kube_context: which context inside that file (default: its current-context) +# +# There are no nodes here on purpose. A kubeconfig reaches the API server, not +# the machines behind it, so the host layer is not available: no disk/swap/ +# cgroup checks, no systemd or container-runtime state, and no "vm setup" or +# "registry apply". Everything that works through kubectl still works — +# verify, gen, deploy, doctor's cluster and workload signatures, and the TUI. +# +# For a self-managed cluster you can SSH into, describe its nodes instead and +# get the host layer as well. +# +# Several clusters can live in one file, mixing both kinds: +# clusters: +# - cluster: prod +# kubeconfig: ~/.kube/prod.yaml +# - cluster: lab +# nodes: [...] +# current: prod +# then select one with --context . ` } return h diff --git a/internal/cli/kyaml_cmds.go b/internal/cli/kyaml_cmds.go index 962d2cb..f7dafb8 100644 --- a/internal/cli/kyaml_cmds.go +++ b/internal/cli/kyaml_cmds.go @@ -7,7 +7,7 @@ import ( "text/tabwriter" "github.com/solutionforest/k3helper/internal/kyaml" - "github.com/solutionforest/k3helper/internal/ssh" + "github.com/solutionforest/k3helper/internal/transport" "github.com/spf13/cobra" ) @@ -25,20 +25,16 @@ func newVerifyCmd() *cobra.Command { // --dry-run=server adds validation layer 3: submit each manifest to // the real API server, which catches unknown fields, admission // rejections and CRD schemas that offline rules cannot. - var server *ssh.Client + var server transport.Cluster if serverDry { targets, err := loadTargets(targetsPath) if err != nil { return err } - srvNode, err := targets.Server() + server, err = transport.Server(targets) if err != nil { return err } - server, err = ssh.Dial(toSSHNode(*srvNode)) - if err != nil { - return fmt.Errorf("connect to server: %w", err) - } defer server.Close() } diff --git a/internal/cli/registry_cmd.go b/internal/cli/registry_cmd.go index 5ac2e1e..d86d8bb 100644 --- a/internal/cli/registry_cmd.go +++ b/internal/cli/registry_cmd.go @@ -7,6 +7,7 @@ import ( "github.com/solutionforest/k3helper/internal/config" "github.com/solutionforest/k3helper/internal/registry" "github.com/solutionforest/k3helper/internal/ssh" + "github.com/solutionforest/k3helper/internal/transport" "github.com/spf13/cobra" ) @@ -46,6 +47,9 @@ func newRegistryApplyCmd() *cobra.Command { if err != nil { return err } + if err := transport.RequireHosts(targets, "registry apply"); err != nil { + return err + } regs, err := regFlags.registriesFor(targets) if err != nil { return err diff --git a/internal/cli/root.go b/internal/cli/root.go index 2164462..3f766de 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -11,7 +11,7 @@ import ( // version is overridden at build time by the Makefile's ldflags. The literal // here is the fallback for a bare `go build`, so keep it in step with // Makefile's VERSION or `go run` reports a stale number. -var version = "0.4.0" +var version = "0.5.0" // contextName is the --context value: which cluster to use from a // multi-cluster targets file. Empty means the file's `current`, or its only diff --git a/internal/cli/vm_cmd.go b/internal/cli/vm_cmd.go index 3946a59..b765658 100644 --- a/internal/cli/vm_cmd.go +++ b/internal/cli/vm_cmd.go @@ -7,6 +7,7 @@ import ( "github.com/solutionforest/k3helper/internal/config" "github.com/solutionforest/k3helper/internal/registry" "github.com/solutionforest/k3helper/internal/ssh" + "github.com/solutionforest/k3helper/internal/transport" "github.com/solutionforest/k3helper/internal/vm" "github.com/spf13/cobra" ) @@ -86,6 +87,9 @@ func newVMSetupCmd() *cobra.Command { if err != nil { return err } + if err := transport.RequireHosts(targets, "vm setup"); err != nil { + return err + } // Servers first, then agents. Multiple servers switch k3s to // embedded etcd; the first one initialises the cluster. var servers, agents []vm.Target diff --git a/internal/config/config.go b/internal/config/config.go index fd72de5..10f956e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,6 +3,7 @@ package config import ( "fmt" "os" + "path/filepath" "strings" "github.com/solutionforest/k3helper/internal/ssh" @@ -44,10 +45,42 @@ func (n Node) SSH() ssh.Node { } } -// Targets is one cluster: a name and the machines that make it up. +// Mode is how k3helper reaches a cluster. +type Mode int + +const ( + // ModeSSH reaches the cluster by connecting to its nodes and running the + // kubectl that lives on them. It is the only mode that can see the host + // layer — disks, systemd units, the container runtime. + ModeSSH Mode = iota + // ModeKubeconfig reaches the cluster through the API server with a local + // kubectl and a kubeconfig. It is the only mode that works against a + // managed cluster (EKS/GKE/AKS and friends), where there is no SSH to a + // control-plane node and no admin.conf to read. + ModeKubeconfig +) + +func (m Mode) String() string { + if m == ModeKubeconfig { + return "kubeconfig" + } + return "ssh" +} + +// Targets is one cluster: a name, and either the machines that make it up or +// the kubeconfig that reaches it. type Targets struct { Cluster string `json:"cluster"` - Nodes []Node `json:"nodes"` + Nodes []Node `json:"nodes,omitempty"` + // Kubeconfig is a path on *this* machine. Setting it selects + // ModeKubeconfig: commands run through a local kubectl instead of over + // SSH, and the host layer is not available at all. + Kubeconfig string `json:"kubeconfig,omitempty"` + // KubeContext names a context inside that kubeconfig. Empty uses the + // file's current-context. It is deliberately not called "context": that + // word already means "which cluster in this targets file" everywhere else + // in k3helper, including the --context flag. + KubeContext string `json:"kube_context,omitempty"` // Registries are the image registries every node in this cluster pulls // from. Cluster-level rather than per-node: a mirror configured on some // nodes and not others produces pods that run on two machines out of @@ -55,6 +88,45 @@ type Targets struct { Registries []Registry `json:"registries,omitempty"` } +// Mode reports how this cluster is reached. +func (t *Targets) Mode() Mode { + if t.Kubeconfig != "" { + return ModeKubeconfig + } + return ModeSSH +} + +// KubeconfigPath is the kubeconfig with a leading ~ expanded. Tilde expansion +// is done here rather than left to a shell: the path is handed to os.Stat and +// to kubectl as an argument, and neither expands it. +func (t *Targets) KubeconfigPath() string { + return expandHome(t.Kubeconfig) +} + +// expandHome expands a leading ~ or ~/ to the user's home directory. A path +// that does not start with ~ is returned unchanged, as is one whose home +// cannot be determined — the caller's os.Stat will report that better than a +// guess would. +// +// The result is joined rather than concatenated so it comes back in the +// platform's own separators: "~/.kube/config" on Windows should become +// C:\Users\you\.kube\config, not C:\Users\you/.kube/config. Go accepts the +// mixed form, but the path is printed in errors and passed to kubectl, and +// half-and-half reads like a bug. +func expandHome(p string) string { + if p != "~" && !strings.HasPrefix(p, "~/") { + return p + } + home, err := os.UserHomeDir() + if err != nil || home == "" { + return p + } + if p == "~" { + return home + } + return filepath.Join(home, filepath.FromSlash(p[2:])) +} + // Registry is a container image registry the nodes pull from — a private // registry, or a mirror in front of a public one. type Registry struct { @@ -144,6 +216,10 @@ type File struct { // single-cluster form Cluster string `json:"cluster,omitempty"` Nodes []Node `json:"nodes,omitempty"` + // Kubeconfig/KubeContext are the single-cluster form of the same fields on + // Targets: one cluster reached through its API server rather than by SSH. + Kubeconfig string `json:"kubeconfig,omitempty"` + KubeContext string `json:"kube_context,omitempty"` // multi-cluster form Clusters []Targets `json:"clusters,omitempty"` Current string `json:"current,omitempty"` @@ -174,16 +250,27 @@ func Load(path string) (*File, error) { if err := yaml.Unmarshal(data, f); err != nil { return nil, fmt.Errorf("parse targets %s: %w", path, err) } - if len(f.Clusters) > 0 && (f.Cluster != "" || len(f.Nodes) > 0) { + if len(f.Clusters) > 0 && (f.Cluster != "" || len(f.Nodes) > 0 || f.Kubeconfig != "") { return nil, fmt.Errorf("invalid targets %s: use either top-level cluster/nodes or a clusters list, not both", path) } if len(f.Clusters) == 0 { - f.Clusters = []Targets{{Cluster: f.Cluster, Nodes: f.Nodes, Registries: f.Registries}} - f.Cluster, f.Nodes = "", nil + single := Targets{ + Cluster: f.Cluster, Nodes: f.Nodes, + Kubeconfig: f.Kubeconfig, KubeContext: f.KubeContext, + } + // Registries at the top of a single-cluster file are that cluster's + // own, not a default to inherit — but a kubeconfig cluster cannot act + // on them, and Validate says so rather than dropping them silently. + single.Registries = f.Registries + f.Clusters = []Targets{single} + f.Cluster, f.Nodes, f.Kubeconfig, f.KubeContext = "", nil, "", "" } else { // File-level registries are the default for clusters that declare none. + // Kubeconfig clusters are skipped: registry config is written on the + // nodes, and inheriting a block they cannot act on would turn a + // perfectly good file into a validation error. for i := range f.Clusters { - if len(f.Clusters[i].Registries) == 0 { + if len(f.Clusters[i].Registries) == 0 && f.Clusters[i].Mode() == ModeSSH { f.Clusters[i].Registries = f.Registries } } @@ -251,8 +338,11 @@ func (t *Targets) Validate() error { if t.Cluster == "" { return fmt.Errorf("cluster name is required") } + if t.Mode() == ModeKubeconfig { + return t.validateKubeconfig() + } if len(t.Nodes) == 0 { - return fmt.Errorf("at least one node is required") + return fmt.Errorf("at least one node is required (or set kubeconfig: to reach the cluster through its API server)") } seen := map[string]bool{} servers := 0 @@ -299,6 +389,37 @@ func (t *Targets) Validate() error { return t.validateRegistries() } +// validateKubeconfig checks a kubeconfig-mode cluster. +// +// Nodes and a kubeconfig are rejected together rather than merged. The two +// describe the same cluster through different doors, and a file that lists +// both leaves every command guessing which door to use — including the ones +// that write to hosts, where guessing wrong writes to the wrong machine. +func (t *Targets) validateKubeconfig() error { + if len(t.Nodes) > 0 { + return fmt.Errorf("cluster %q: set either nodes or kubeconfig, not both "+ + "(nodes: reach the cluster over SSH; kubeconfig: reach it through its API server)", t.Cluster) + } + path := t.KubeconfigPath() + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("cluster %q: no kubeconfig at %s", t.Cluster, path) + } + return fmt.Errorf("cluster %q: kubeconfig %s: %w", t.Cluster, path, err) + } + if info.IsDir() { + return fmt.Errorf("cluster %q: kubeconfig %s is a directory", t.Cluster, path) + } + if len(t.Registries) > 0 { + // Registry configuration is written to files on the nodes. There are no + // nodes here, so accepting the block would silently do nothing. + return fmt.Errorf("cluster %q: registries cannot be configured on a kubeconfig cluster — "+ + "k3helper writes registry config on the nodes themselves, which needs SSH access", t.Cluster) + } + return nil +} + // validateRegistries rejects registry entries that cannot mean what they say. func (t *Targets) validateRegistries() error { seen := map[string]bool{} @@ -341,6 +462,12 @@ func (t *Targets) Server() (*Node, error) { return &t.Nodes[i], nil } } + if t.Mode() == ModeKubeconfig { + // A caller that reached here wants a machine to run something on, and + // this cluster has none. Say that rather than "no server node", which + // reads like a malformed file. + return nil, fmt.Errorf("cluster %q is reached through a kubeconfig and has no nodes to connect to", t.Cluster) + } return nil, fmt.Errorf("no server node in targets") } diff --git a/internal/config/kubeconfig_test.go b/internal/config/kubeconfig_test.go new file mode 100644 index 0000000..70daea1 --- /dev/null +++ b/internal/config/kubeconfig_test.go @@ -0,0 +1,186 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// writeKubeconfig creates a stand-in kubeconfig. Nothing parses it — the +// transport hands the path to kubectl — so its contents only have to exist. +func writeKubeconfig(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config") + if err := os.WriteFile(path, []byte("apiVersion: v1\nkind: Config\n"), 0o600); err != nil { + t.Fatalf("write kubeconfig: %v", err) + } + return path +} + +func writeTargets(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "targets.yaml") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatalf("write targets: %v", err) + } + return path +} + +func TestModeFollowsKubeconfig(t *testing.T) { + ssh := &Targets{Cluster: "a", Nodes: []Node{{Name: "s", Role: "server", Host: "h", User: "u"}}} + if got := ssh.Mode(); got != ModeSSH { + t.Errorf("nodes-only cluster: mode = %v, want ModeSSH", got) + } + kc := &Targets{Cluster: "a", Kubeconfig: "/k/c.yaml"} + if got := kc.Mode(); got != ModeKubeconfig { + t.Errorf("kubeconfig cluster: mode = %v, want ModeKubeconfig", got) + } +} + +func TestLoadKubeconfigCluster(t *testing.T) { + kube := writeKubeconfig(t) + path := writeTargets(t, "cluster: prod\nkubeconfig: "+kube+"\nkube_context: admin\n") + + got, err := LoadTargets(path) + if err != nil { + t.Fatalf("LoadTargets: %v", err) + } + if got.Mode() != ModeKubeconfig { + t.Errorf("mode = %v, want ModeKubeconfig", got.Mode()) + } + if got.KubeContext != "admin" { + t.Errorf("kube_context = %q, want admin", got.KubeContext) + } + if len(got.Nodes) != 0 { + t.Errorf("got %d nodes, want none", len(got.Nodes)) + } +} + +// Both together leave every command guessing which door to use — including +// the ones that write files on hosts. +func TestKubeconfigAndNodesTogetherRejected(t *testing.T) { + kube := writeKubeconfig(t) + path := writeTargets(t, "cluster: prod\nkubeconfig: "+kube+"\nnodes:\n - name: s\n role: server\n host: 10.0.0.1\n user: root\n") + + _, err := LoadTargets(path) + if err == nil { + t.Fatal("a file with both nodes and kubeconfig was accepted") + } + if !strings.Contains(err.Error(), "not both") { + t.Errorf("error %q does not explain the conflict", err) + } +} + +func TestKubeconfigMustExist(t *testing.T) { + path := writeTargets(t, "cluster: prod\nkubeconfig: /nowhere/kube.yaml\n") + _, err := LoadTargets(path) + if err == nil { + t.Fatal("a missing kubeconfig was accepted") + } + if !strings.Contains(err.Error(), "no kubeconfig at") { + t.Errorf("error %q does not name the missing file", err) + } +} + +func TestKubeconfigDirectoryRejected(t *testing.T) { + dir := t.TempDir() + path := writeTargets(t, "cluster: prod\nkubeconfig: "+dir+"\n") + if _, err := LoadTargets(path); err == nil { + t.Fatal("a directory was accepted as a kubeconfig") + } +} + +// Registry configuration is written on the nodes. A kubeconfig cluster has +// none, so accepting the block would silently do nothing. +func TestRegistriesRejectedOnKubeconfigCluster(t *testing.T) { + kube := writeKubeconfig(t) + path := writeTargets(t, "cluster: prod\nkubeconfig: "+kube+"\nregistries:\n - host: reg.example.net\n") + _, err := LoadTargets(path) + if err == nil { + t.Fatal("registries were accepted on a cluster with no nodes to write them to") + } + if !strings.Contains(err.Error(), "SSH") { + t.Errorf("error %q does not say why", err) + } +} + +// File-level registries are a default for SSH clusters. Inheriting them into a +// kubeconfig cluster would turn a valid mixed file into a validation error. +func TestFileLevelRegistriesSkipKubeconfigClusters(t *testing.T) { + kube := writeKubeconfig(t) + path := writeTargets(t, `registries: + - host: reg.example.net +clusters: + - cluster: prod + kubeconfig: `+kube+` + - cluster: lab + nodes: + - name: s + role: server + host: 10.0.0.1 + user: root +current: prod +`) + f, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + prod, err := f.Select("prod") + if err != nil { + t.Fatalf("Select prod: %v", err) + } + if len(prod.Registries) != 0 { + t.Errorf("kubeconfig cluster inherited %d registries", len(prod.Registries)) + } + lab, err := f.Select("lab") + if err != nil { + t.Fatalf("Select lab: %v", err) + } + if len(lab.Registries) != 1 { + t.Errorf("SSH cluster got %d registries, want the file-level default", len(lab.Registries)) + } +} + +// A path is handed to os.Stat and to kubectl as an argument; neither expands ~. +func TestKubeconfigPathExpandsHome(t *testing.T) { + home, err := os.UserHomeDir() + if err != nil { + t.Skip("no home directory in this environment") + } + tg := &Targets{Cluster: "a", Kubeconfig: "~/.kube/config"} + want := filepath.Join(home, ".kube", "config") + if got := tg.KubeconfigPath(); got != want { + t.Errorf("KubeconfigPath() = %q, want %q", got, want) + } + // A plain path is left alone. + tg.Kubeconfig = "/etc/kube/config" + if got := tg.KubeconfigPath(); got != "/etc/kube/config" { + t.Errorf("absolute path rewritten to %q", got) + } +} + +// The message a caller gets when it asks a kubeconfig cluster for a machine +// has to say what is actually going on, not "no server node". +func TestServerOnKubeconfigClusterExplainsItself(t *testing.T) { + tg := &Targets{Cluster: "prod", Kubeconfig: "/k/c.yaml"} + _, err := tg.Server() + if err == nil { + t.Fatal("Server() returned a node for a cluster that has none") + } + if !strings.Contains(err.Error(), "kubeconfig") { + t.Errorf("error %q reads like a malformed file rather than a kubeconfig cluster", err) + } +} + +// A nodes-only file must keep validating exactly as before. +func TestSSHClusterStillNeedsNodes(t *testing.T) { + path := writeTargets(t, "cluster: prod\nnodes: []\n") + _, err := LoadTargets(path) + if err == nil { + t.Fatal("a cluster with neither nodes nor a kubeconfig was accepted") + } + if !strings.Contains(err.Error(), "kubeconfig") { + t.Errorf("error %q does not mention the other way to reach a cluster", err) + } +} diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index d6a317c..1b34b52 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -161,7 +161,7 @@ func Deploy(exec Executor, manifest string, opts Options) (*Result, error) { return res, fmt.Errorf("--namespace %s conflicts with the manifest: %s; remove metadata.namespace or drop the flag", opts.Namespace, strings.Join(conflicts, "; ")) } - remote, err := kyaml.RemotePath(manifest) + remote, err := kyaml.RemotePathIn(kyaml.StageDir(exec), manifest) if err != nil { return res, err } diff --git a/internal/kube/kube.go b/internal/kube/kube.go index bde7338..4f9b706 100644 --- a/internal/kube/kube.go +++ b/internal/kube/kube.go @@ -25,6 +25,14 @@ const ( kubeadmBase = `sudo -n kubectl --kubeconfig /etc/kubernetes/admin.conf` ) +// Based is a transport that already knows its own kubectl invocation and must +// not be probed for one. The local kubeconfig transport is the case: it has no +// node to look at, and the probes below would all fail and leave it building +// k3s commands for a cluster that is not k3s. +type Based interface { + Base() string +} + // DetectBase picks the kubectl invocation this node can actually use. // // It probes once rather than chaining the candidates with || on every call. @@ -33,6 +41,9 @@ const ( // the caller would be shown the *second* arm's "no such file" instead of the // validation error it actually needed. func DetectBase(exec Executor) string { + if b, ok := exec.(Based); ok { + return b.Base() + } if _, code, err := exec.Run(`test -x /usr/local/bin/k3s || command -v k3s >/dev/null 2>&1`); err == nil && code == 0 { return k3sBase } diff --git a/internal/kubectl/exec_test.go b/internal/kubectl/exec_test.go new file mode 100644 index 0000000..948a3bc --- /dev/null +++ b/internal/kubectl/exec_test.go @@ -0,0 +1,149 @@ +//go:build !windows + +package kubectl + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/solutionforest/k3helper/internal/kube" + "github.com/solutionforest/k3helper/internal/troubleshoot" +) + +// fakeKubectl puts a stub kubectl at the front of PATH. +// +// The unit tests above stop at the argument list. This runs the whole path — +// command string, parse, argv, child process, output — because the failure +// this transport is most likely to have is an argument that survives parsing +// and then means something different to a real kubectl. +func fakeKubectl(t *testing.T, script string) { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "kubectl") + if err := os.WriteFile(path, []byte("#!/bin/sh\n"+script), 0o755); err != nil { + t.Fatalf("write fake kubectl: %v", err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func TestRunExecutesKubectl(t *testing.T) { + // Echo the arguments back so the test can assert on exactly what kubectl + // was called with. + fakeKubectl(t, `echo "$@"`) + l := Local{Kubeconfig: "/k/c.yaml", Context: "prod"} + + out, code, err := l.Run(l.Base() + " get pods -n 'kube-system' -o json") + if err != nil { + t.Fatalf("Run: %v", err) + } + if code != 0 { + t.Fatalf("exit %d: %s", code, out) + } + got := strings.TrimSpace(out) + want := "--kubeconfig /k/c.yaml --context prod get pods -n kube-system -o json" + if got != want { + t.Errorf("kubectl got %q, want %q", got, want) + } +} + +// A non-zero exit is a result, not an error: `kubectl diff` reports 1 for +// "there are differences" and callers read the code. +func TestRunReportsExitCode(t *testing.T) { + fakeKubectl(t, `echo "boom" >&2; exit 3`) + l := Local{Kubeconfig: "/k/c.yaml"} + out, code, err := l.Run(l.Base() + " diff -f '/tmp/x.yaml' 2>&1") + if err != nil { + t.Fatalf("Run returned an error for a non-zero exit: %v", err) + } + if code != 3 { + t.Errorf("exit code = %d, want 3", code) + } + if !strings.Contains(out, "boom") { + t.Errorf("2>&1 did not merge stderr: %q", out) + } +} + +// 2>/dev/null exists because kubectl writes deprecation warnings to stderr and +// a merged stream puts them in front of the JSON, so every parse fails on a +// perfectly healthy cluster. +func TestStderrIsDroppedWhenAsked(t *testing.T) { + fakeKubectl(t, `echo "Warning: v1 Endpoints is deprecated" >&2; echo '{"items":[]}'`) + l := Local{Kubeconfig: "/k/c.yaml"} + + quiet, _, err := l.Run(l.Base() + " get endpoints -A -o json 2>/dev/null") + if err != nil { + t.Fatalf("Run: %v", err) + } + if !strings.HasPrefix(strings.TrimSpace(quiet), "{") { + t.Errorf("stderr leaked into JSON output: %q", quiet) + } + + merged, _, err := l.Run(l.Base() + " get endpoints -A -o json") + if err != nil { + t.Fatalf("Run: %v", err) + } + if !strings.Contains(merged, "deprecated") { + t.Errorf("stderr was dropped without being asked: %q", merged) + } +} + +// The whole point of the transport: the existing cluster-layer code, unchanged, +// driven through a kubeconfig instead of over SSH. +func TestListPodsThroughTransport(t *testing.T) { + fakeKubectl(t, `cat <<'JSON' +{"items":[ + {"metadata":{"name":"web-1","namespace":"default","labels":{"app":"web"}}, + "spec":{"nodeName":"node-a"}, + "status":{"phase":"Running","podIP":"10.0.0.5","startTime":"2026-01-01T00:00:00Z", + "containerStatuses":[{"ready":true,"restartCount":2}]}} +]} +JSON`) + l := Local{Kubeconfig: "/k/c.yaml"} + pods, err := kube.ListPods(l, "default") + if err != nil { + t.Fatalf("ListPods: %v", err) + } + if len(pods) != 1 { + t.Fatalf("got %d pods, want 1", len(pods)) + } + p := pods[0] + if p.Name != "web-1" || p.Namespace != "default" { + t.Errorf("pod = %s/%s, want default/web-1", p.Namespace, p.Name) + } + if p.Restarts != 2 { + t.Errorf("restarts = %d, want 2", p.Restarts) + } + if p.Node != "node-a" { + t.Errorf("node = %q, want node-a", p.Node) + } +} + +// A gather through a kubeconfig must produce cluster evidence and no host +// evidence, without recording the absent host layer as a failed probe. +func TestGatherThroughTransport(t *testing.T) { + fakeKubectl(t, ` +case "$*" in + *"get nodes"*) echo '{"items":[{"metadata":{"name":"node-a"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}' ;; + *) echo '{"items":[]}' ;; +esac`) + l := Local{Kubeconfig: "/k/c.yaml"} + e := troubleshoot.Gatherer{Server: l, NoHostLayer: true}.Collect() + + if e.KubeconfigError != "" { + t.Errorf("cluster layer failed through the transport: %s", e.KubeconfigError) + } + if !e.HostLayerUnavailable { + t.Error("the missing host layer was not recorded") + } + if len(e.HostMetrics) != 0 || len(e.K3sService) != 0 { + t.Error("host evidence appeared for a cluster with no hosts") + } + // Certificate collection probes for the k3s binary first. The transport + // refuses that with exit 127, which must read as "not applicable" and not + // as a probe that failed. + if _, recorded := e.ProbeErrors["certificates"]; recorded { + t.Error("the k3s certificate probe was recorded as a failure on a non-k3s cluster") + } +} diff --git a/internal/kubectl/forward.go b/internal/kubectl/forward.go new file mode 100644 index 0000000..bc6ebd0 --- /dev/null +++ b/internal/kubectl/forward.go @@ -0,0 +1,134 @@ +package kubectl + +import ( + "bufio" + "fmt" + "io" + "os/exec" + "strconv" + "strings" + "sync" + "time" +) + +// Forward is a `kubectl port-forward` running as a child of this process. +// +// The SSH transport has to do this in two parts — kubectl binds on the cluster +// node, and a tunnel carries the port back here. Through a kubeconfig there is +// only one machine involved, so kubectl binds locally and there is nothing to +// tunnel. +type Forward struct { + cmd *exec.Cmd + local int + once sync.Once +} + +// LocalPort is the port bound on this machine. +func (f *Forward) LocalPort() int { return f.local } + +// Close stops the forward. Safe to call more than once: the TUI closes each +// forward on teardown and again when the operator removes it from the list. +func (f *Forward) Close() error { + f.once.Do(func() { + if f.cmd != nil && f.cmd.Process != nil { + f.cmd.Process.Kill() + // Reap it, or the child stays a zombie for the life of the TUI. + f.cmd.Wait() + } + }) + return nil +} + +// StartForward runs `kubectl port-forward` and returns once it is listening. +// +// The local port is chosen by kubectl (":remote" asks for any free one) and +// read back from its output rather than picked here. Picking a port means +// binding it to find out it is free, closing it, and handing the number to a +// process that races anything else on the machine for it; asking kubectl what +// it bound has no such gap. +func (l Local) StartForward(ns, target string, remotePort int) (*Forward, error) { + if err := Available(); err != nil { + return nil, err + } + args := []string{"--kubeconfig", l.Kubeconfig} + if l.Context != "" { + args = append(args, "--context", l.Context) + } + args = append(args, "port-forward", "-n", ns, target, + fmt.Sprintf(":%d", remotePort), "--address", "127.0.0.1") + + cmd := exec.Command(Binary, args...) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("port-forward: %w", err) + } + // kubectl reports the bound address on stdout and its failures on stderr. + // Both are needed: one to learn the port, the other to explain a refusal + // instead of reporting a bare timeout. + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, fmt.Errorf("port-forward: %w", err) + } + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("start port-forward: %w", err) + } + f := &Forward{cmd: cmd} + + type result struct { + port int + err error + } + ch := make(chan result, 1) + go func() { + sc := bufio.NewScanner(stdout) + for sc.Scan() { + if p, ok := parseForwardingPort(sc.Text()); ok { + ch <- result{port: p} + // Keep draining: kubectl blocks on a full pipe, which would + // wedge the forward as soon as it logged a second line. + go io.Copy(io.Discard, stdout) + return + } + } + ch <- result{err: fmt.Errorf("port-forward exited without binding a port")} + }() + go io.Copy(io.Discard, stderr) + + select { + case r := <-ch: + if r.err != nil { + f.Close() + return nil, r.err + } + f.local = r.port + return f, nil + case <-time.After(10 * time.Second): + f.Close() + return nil, fmt.Errorf("port-forward to %s/%s did not start listening within 10s", ns, target) + } +} + +// parseForwardingPort reads the local port out of kubectl's +// "Forwarding from 127.0.0.1:54321 -> 80". +func parseForwardingPort(line string) (int, bool) { + const marker = "Forwarding from " + i := strings.Index(line, marker) + if i < 0 { + return 0, false + } + rest := line[i+len(marker):] + // Take the address up to the arrow, then the port after the last colon — + // last, not first, because an IPv6 address has several. + if j := strings.Index(rest, " ->"); j >= 0 { + rest = rest[:j] + } + k := strings.LastIndexByte(rest, ':') + if k < 0 { + return 0, false + } + port, err := strconv.Atoi(strings.TrimSpace(rest[k+1:])) + if err != nil || port <= 0 { + return 0, false + } + return port, true +} diff --git a/internal/kubectl/integration_test.go b/internal/kubectl/integration_test.go new file mode 100644 index 0000000..5896a68 --- /dev/null +++ b/internal/kubectl/integration_test.go @@ -0,0 +1,137 @@ +//go:build integration + +package kubectl_test + +import ( + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/solutionforest/k3helper/internal/kube" + "github.com/solutionforest/k3helper/internal/kubectl" + "github.com/solutionforest/k3helper/internal/sandbox" + "github.com/solutionforest/k3helper/internal/troubleshoot" +) + +// This is the proof the transport works: the same sandbox cluster, read twice — +// once over SSH the way k3helper always has, and once through a kubeconfig with +// a local kubectl — asserting the two agree. +// +// Unit tests can only show that a command string parses into the arguments we +// expected. They cannot show that a real kubectl, given those arguments, +// returns what the parsers upstream are expecting. That is what this does. +// +// It needs a kubeconfig for the sandbox on this machine. `make bootstrap` +// writes one to sandbox-kubeconfig.yaml; the test skips rather than fails +// without it, so the suite still runs on a machine that only has SSH to the +// cluster. +func localForSandbox(t *testing.T) kubectl.Local { + t.Helper() + if _, err := exec.LookPath("kubectl"); err != nil { + t.Skip("kubectl not installed on this machine") + } + path := os.Getenv("K3HELPER_TEST_KUBECONFIG") + if path == "" { + root, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + // internal/kubectl → repo root + path = filepath.Join(root, "..", "..", "sandbox-kubeconfig.yaml") + } + if _, err := os.Stat(path); err != nil { + t.Skipf("no sandbox kubeconfig at %s (set K3HELPER_TEST_KUBECONFIG)", path) + } + return kubectl.Local{Kubeconfig: path} +} + +func names(pods []kube.Pod) []string { + out := make([]string, 0, len(pods)) + for _, p := range pods { + out = append(out, p.Namespace+"/"+p.Name) + } + sort.Strings(out) + return out +} + +// The pod list must be the same list whichever door it came through. +func TestPodsMatchBetweenTransports(t *testing.T) { + viaSSH, err := kube.ListPods(sandbox.Dial(t, "server"), "kube-system") + if err != nil { + t.Fatalf("ListPods over SSH: %v", err) + } + viaKubeconfig, err := kube.ListPods(localForSandbox(t), "kube-system") + if err != nil { + t.Fatalf("ListPods through kubeconfig: %v", err) + } + a, b := names(viaSSH), names(viaKubeconfig) + if strings.Join(a, ",") != strings.Join(b, ",") { + t.Errorf("transports disagree about the cluster\n over SSH: %v\n via kubeconfig: %v", a, b) + } +} + +func TestNodesMatchBetweenTransports(t *testing.T) { + viaSSH, err := kube.ListNodes(sandbox.Dial(t, "server")) + if err != nil { + t.Fatalf("ListNodes over SSH: %v", err) + } + viaKubeconfig, err := kube.ListNodes(localForSandbox(t)) + if err != nil { + t.Fatalf("ListNodes through kubeconfig: %v", err) + } + if len(viaSSH) != len(viaKubeconfig) { + t.Fatalf("node count differs: %d over SSH, %d via kubeconfig", len(viaSSH), len(viaKubeconfig)) + } + for i := range viaSSH { + if viaSSH[i].Name != viaKubeconfig[i].Name { + t.Errorf("node %d: %q over SSH, %q via kubeconfig", i, viaSSH[i].Name, viaKubeconfig[i].Name) + } + if viaSSH[i].Version != viaKubeconfig[i].Version { + t.Errorf("node %s: version %q over SSH, %q via kubeconfig", + viaSSH[i].Name, viaSSH[i].Version, viaKubeconfig[i].Version) + } + } +} + +// A gather through the kubeconfig must produce usable cluster evidence — not +// an empty bundle that would make a broken cluster look healthy. +func TestGatherThroughKubeconfigSeesTheCluster(t *testing.T) { + e := troubleshoot.Gatherer{Server: localForSandbox(t), NoHostLayer: true}.Collect() + + if e.KubeconfigError != "" { + t.Fatalf("cluster layer unreadable through the kubeconfig: %s", e.KubeconfigError) + } + if e.CoreDNS == nil { + t.Error("CoreDNS readiness was not gathered; the jsonpath query did not survive the transport") + } + if !e.HostLayerUnavailable { + t.Error("the missing host layer was not recorded") + } + // The host layer must be absent, not merely empty-looking: anything here + // means a host probe ran somewhere it could not have. + if len(e.HostMetrics) != 0 || len(e.K3sService) != 0 || len(e.ClockSkew) != 0 { + t.Error("host evidence appeared for a transport with no hosts") + } + if _, recorded := e.ProbeErrors["certificates"]; recorded { + t.Error("the k3s-only certificate probe was recorded as a failure rather than skipped") + } + t.Logf("probe errors: %v", e.ProbeErrors) +} + +// port-forward through the kubeconfig binds here rather than on a node, so +// there is no tunnel involved and the port must be reachable directly. +func TestPortForwardThroughKubeconfig(t *testing.T) { + l := localForSandbox(t) + fwd, err := l.StartForward("kube-system", "svc/kube-dns", 53) + if err != nil { + t.Fatalf("StartForward: %v", err) + } + defer fwd.Close() + if fwd.LocalPort() <= 0 { + t.Fatalf("LocalPort() = %d, want the port kubectl bound", fwd.LocalPort()) + } + t.Logf("kube-dns forwarded to 127.0.0.1:%d", fwd.LocalPort()) +} diff --git a/internal/kubectl/local.go b/internal/kubectl/local.go new file mode 100644 index 0000000..9c62e9f --- /dev/null +++ b/internal/kubectl/local.go @@ -0,0 +1,281 @@ +// Package kubectl runs kubectl on this machine against a kubeconfig, as an +// alternative to running it on a cluster node over SSH. +// +// It exists for managed Kubernetes — EKS, GKE, AKS, and anything else where +// there is no SSH into a control-plane node and no /etc/kubernetes/admin.conf +// to read. Those clusters hand the operator a kubeconfig and nothing else. +// +// The rest of k3helper builds shell command strings and hands them to an +// Executor. This one does not have a shell to hand them to: the whole point of +// the local transport is that it also works on Windows, where there is no +// /bin/sh. So Run parses the command instead of interpreting it, and accepts +// only the kubectl invocations this package itself produced. +package kubectl + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// Binary is the kubectl executable to run. A name (rather than a path) is +// resolved through PATH by os/exec. +const Binary = "kubectl" + +// ErrHostLayer is returned for commands that are not kubectl invocations — +// disk usage, systemd state, and the rest of the host layer. A kubeconfig +// reaches an API server, not a machine, so these can never work here. +var ErrHostLayer = errors.New("host-layer command not available on a kubeconfig cluster") + +// Local runs kubectl on this machine. +type Local struct { + // Kubeconfig is the path to the kubeconfig file. Required: an empty value + // would silently fall back to whatever ~/.kube/config points at, which is + // how a command meant for staging lands on production. + Kubeconfig string + // Context names a context within that file. Empty uses its current-context. + Context string +} + +// Base is the kubectl invocation every command from this transport starts +// with. kube.Builder uses it instead of probing the node for a distribution: +// there is no node to probe. +func (l Local) Base() string { + b := Binary + " --kubeconfig " + quote(l.Kubeconfig) + if l.Context != "" { + b += " --context " + quote(l.Context) + } + return b +} + +// Available reports whether kubectl can be found and run. Callers check this +// once at startup: "kubectl: executable file not found in $PATH" from the +// middle of a diagnosis is a worse message than one at the front door. +func Available() error { + path, err := exec.LookPath(Binary) + if err != nil { + return fmt.Errorf("kubectl not found in PATH — a kubeconfig cluster is driven through kubectl, "+ + "so install it and try again: %w", err) + } + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("kubectl at %s is not usable: %w", path, err) + } + return nil +} + +// Run executes one command and returns its output, exit code and error, with +// the same contract as ssh.Client.Run: a non-zero exit is reported in the code +// and is not an error. +// +// The command must be a kubectl invocation produced from Base. Anything else +// is refused with exit 127 rather than attempted, so a host-layer probe fails +// where it is written instead of somewhere further down the call stack. +func (l Local) Run(cmd string) (string, int, error) { + args, mode, err := l.parse(cmd) + if err != nil { + if errors.Is(err, ErrHostLayer) { + // 127 is what a shell reports for a command it cannot run, which is + // what callers that probe with `command -v` already expect. + return err.Error(), 127, nil + } + return "", -1, err + } + c := exec.Command(Binary, args...) + var out []byte + if mode == stderrDrop { + // Callers add 2>/dev/null before parsing JSON, because kubectl prints + // deprecation warnings to stderr and a merged stream would put them in + // front of the document. Honour that: drop stderr, keep stdout. + out, err = c.Output() + } else { + out, err = c.CombinedOutput() + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + // A non-zero exit is a result, not a failure: `kubectl diff` reports 1 + // for "there are differences" and callers read the code. + return string(out), exitErr.ExitCode(), nil + } + if err != nil { + return string(out), -1, fmt.Errorf("run kubectl: %w", err) + } + return string(out), 0, nil +} + +// stderrMode is what to do with the child's stderr, decided by the redirection +// the caller wrote at the end of the command. +type stderrMode int + +const ( + stderrMerge stderrMode = iota // default, and `2>&1`: what SSH does + stderrDrop // `2>/dev/null` +) + +// parse turns a command string into kubectl arguments. +func (l Local) parse(cmd string) ([]string, stderrMode, error) { + if l.Kubeconfig == "" { + return nil, stderrMerge, fmt.Errorf("kubectl transport: no kubeconfig configured") + } + rest := strings.TrimSpace(cmd) + + // Strip the trailing redirection first: it is punctuation for the shell we + // are standing in for, not an argument to kubectl. + mode := stderrMerge + switch { + case strings.HasSuffix(rest, "2>/dev/null"): + mode = stderrDrop + rest = strings.TrimSpace(strings.TrimSuffix(rest, "2>/dev/null")) + case strings.HasSuffix(rest, "2>&1"): + rest = strings.TrimSpace(strings.TrimSuffix(rest, "2>&1")) + } + + base := l.Base() + if !strings.HasPrefix(rest, base) { + return nil, mode, fmt.Errorf("%w: %s", ErrHostLayer, firstWord(rest)) + } + rest = strings.TrimSpace(strings.TrimPrefix(rest, base)) + + args, err := split(rest) + if err != nil { + return nil, mode, err + } + // Re-attach the flags from Base as real argv entries. They are quoted in + // the string form and must not reach kubectl with the quotes still on. + full := []string{"--kubeconfig", l.Kubeconfig} + if l.Context != "" { + full = append(full, "--context", l.Context) + } + return append(full, args...), mode, nil +} + +// firstWord is used to name the refused command in an error. +func firstWord(s string) string { + if i := strings.IndexAny(s, " \t"); i >= 0 { + return s[:i] + } + if s == "" { + return "(empty command)" + } + return s +} + +// split breaks a command tail into arguments, honouring the single quoting +// that kube.shellQuote produces: '...' with an embedded quote written '\”. +// +// This is not a general shell parser and deliberately refuses what it does not +// understand. Every command that reaches it was built by k3helper from a small +// set of format strings; a pipe or a subshell here means a caller has grown a +// shape this transport cannot honour, and failing loudly is how that gets +// noticed before it silently drops half a command. +func split(s string) ([]string, error) { + var ( + args []string + cur strings.Builder + started bool + ) + flush := func() { + if started { + args = append(args, cur.String()) + cur.Reset() + started = false + } + } + for i := 0; i < len(s); i++ { + switch c := s[i]; c { + case ' ', '\t': + flush() + case '\'': + started = true + // Consume to the closing quote. '\'' is not an escape inside a + // single-quoted string; it is a close, a literal quote, and a + // reopen — which this loop handles by simply continuing. + i++ + for i < len(s) && s[i] != '\'' { + cur.WriteByte(s[i]) + i++ + } + if i >= len(s) { + return nil, fmt.Errorf("kubectl transport: unbalanced quote in %q", s) + } + case '"': + started = true + i++ + for i < len(s) && s[i] != '"' { + cur.WriteByte(s[i]) + i++ + } + if i >= len(s) { + return nil, fmt.Errorf("kubectl transport: unbalanced quote in %q", s) + } + case '\\': + if i+1 < len(s) { + i++ + cur.WriteByte(s[i]) + started = true + } + case '|', ';', '&', '>', '<', '`', '$', '(', ')': + return nil, fmt.Errorf("kubectl transport: shell metacharacter %q in %q "+ + "(this transport runs kubectl directly and has no shell)", string(c), s) + default: + cur.WriteByte(c) + started = true + } + } + flush() + return args, nil +} + +// quote renders a value for the *string* form of the command, which is what +// gets prefix-matched and printed. Nothing ever hands this to a shell. +func quote(s string) string { + if s == "" || strings.ContainsAny(s, " \t'\"\\") { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" + } + return s +} + +// --- file staging --- +// +// deploy and `verify --live` upload a manifest and then apply it by path, +// because in SSH mode the file the operator names is on their machine and +// kubectl reads it on the cluster's. Here both are the same machine, so +// "upload" is a write to the local temp directory. The interface has to match +// ssh.Client either way, or every caller needs a mode switch. + +// StageDir is this machine's temp directory. kyaml asks the transport for it +// rather than assuming /tmp, which a Windows machine does not have. +func (l Local) StageDir() string { return os.TempDir() } + +// WriteFile writes a staged manifest. +func (l Local) WriteFile(path string, data []byte, mode os.FileMode) error { + if dir := filepath.Dir(path); dir != "" { + if err := os.MkdirAll(dir, 0o700); err != nil { + return fmt.Errorf("create %s: %w", dir, err) + } + } + if err := os.WriteFile(path, data, mode.Perm()); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + // os.WriteFile only applies mode on create, and the umask masks it; chmod + // pins the requested bits either way. Manifests are written 0600 and can + // carry Secrets. + if err := os.Chmod(path, mode.Perm()); err != nil { + return fmt.Errorf("chmod %s: %w", path, err) + } + return nil +} + +// RemoveFile deletes a staged manifest. +func (l Local) RemoveFile(path string) error { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("rm %s: %w", path, err) + } + return nil +} + +// Close exists so callers can treat this like an ssh.Client and defer a close +// without caring which transport they hold. There is nothing to close. +func (l Local) Close() error { return nil } diff --git a/internal/kubectl/local_test.go b/internal/kubectl/local_test.go new file mode 100644 index 0000000..9c1b210 --- /dev/null +++ b/internal/kubectl/local_test.go @@ -0,0 +1,250 @@ +package kubectl + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/solutionforest/k3helper/internal/kube" +) + +func TestBase(t *testing.T) { + tests := []struct { + name string + l Local + want string + }{ + {"no context", Local{Kubeconfig: "/home/a/.kube/config"}, + "kubectl --kubeconfig /home/a/.kube/config"}, + {"with context", Local{Kubeconfig: "/k/c.yaml", Context: "prod"}, + "kubectl --kubeconfig /k/c.yaml --context prod"}, + {"path with spaces is quoted", Local{Kubeconfig: `C:\Users\Al An\.kube\config`}, + `kubectl --kubeconfig 'C:\Users\Al An\.kube\config'`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := tc.l.Base(); got != tc.want { + t.Errorf("Base() = %q, want %q", got, tc.want) + } + }) + } +} + +// The commands here are the real shapes k3helper builds — copied from +// troubleshoot/gather.go, deploy/deploy.go and kube/kube.go. If one of those +// grows a shape this transport cannot parse, this test is where it shows up. +func TestParseRealCommands(t *testing.T) { + l := Local{Kubeconfig: "/k/c.yaml", Context: "prod"} + b := l.Base() + prefix := []string{"--kubeconfig", "/k/c.yaml", "--context", "prod"} + + tests := []struct { + name string + cmd string + want []string + mode stderrMode + }{ + { + name: "nodes json with stderr dropped", + cmd: b + " get nodes -o json 2>/dev/null", + want: []string{"get", "nodes", "-o", "json"}, + mode: stderrDrop, + }, + { + name: "pods all namespaces", + cmd: b + " get pods -A -o json 2>/dev/null", + want: []string{"get", "pods", "-A", "-o", "json"}, + mode: stderrDrop, + }, + { + name: "quoted namespace", + cmd: b + " get pods -n 'kube-system' -o json", + want: []string{"get", "pods", "-n", "kube-system", "-o", "json"}, + mode: stderrMerge, + }, + { + name: "jsonpath keeps its braces and slash", + cmd: b + " get deployment coredns -n kube-system -o jsonpath={.status.readyReplicas}/{.spec.replicas} 2>/dev/null", + want: []string{"get", "deployment", "coredns", "-n", "kube-system", + "-o", "jsonpath={.status.readyReplicas}/{.spec.replicas}"}, + mode: stderrDrop, + }, + { + name: "apply dry-run with merged stderr", + cmd: b + " apply --dry-run=server -n 'default' -f '/tmp/k3helper-ab12-app.yaml' 2>&1", + want: []string{"apply", "--dry-run=server", "-n", "default", "-f", "/tmp/k3helper-ab12-app.yaml"}, + mode: stderrMerge, + }, + { + name: "rollout status", + cmd: b + " rollout status -n 'web' 'deployment/api' --timeout=120s 2>&1", + want: []string{"rollout", "status", "-n", "web", "deployment/api", "--timeout=120s"}, + mode: stderrMerge, + }, + { + name: "label selector", + cmd: b + " get pods -n 'default' -l 'app=web,tier=front' -o json", + want: []string{"get", "pods", "-n", "default", "-l", "app=web,tier=front", "-o", "json"}, + mode: stderrMerge, + }, + { + name: "no arguments at all", + cmd: b, + want: nil, + mode: stderrMerge, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + args, mode, err := l.parse(tc.cmd) + if err != nil { + t.Fatalf("parse(%q) error: %v", tc.cmd, err) + } + if mode != tc.mode { + t.Errorf("stderr mode = %v, want %v", mode, tc.mode) + } + want := append(append([]string{}, prefix...), tc.want...) + if !reflect.DeepEqual(args, want) { + t.Errorf("parse(%q)\n got %q\nwant %q", tc.cmd, args, want) + } + }) + } +} + +// A name with a quote in it is quoted by kube.shellQuote as '\” — three +// tokens of punctuation that must come back as one literal quote. +func TestParseEmbeddedQuote(t *testing.T) { + l := Local{Kubeconfig: "/k/c.yaml"} + cmd := l.Base() + ` describe -n 'ns' ` + `'pod'\''s-name'` + args, _, err := l.parse(cmd) + if err != nil { + t.Fatalf("parse: %v", err) + } + got := args[len(args)-1] + if got != "pod's-name" { + t.Errorf("last arg = %q, want %q", got, "pod's-name") + } +} + +func TestParseRejectsHostCommands(t *testing.T) { + l := Local{Kubeconfig: "/k/c.yaml"} + // Every one of these is a real host-layer probe from check/ or gather.go. + for _, cmd := range []string{ + `df -P / | tail -1 | awk '{print $5}'`, + `free -m | awk '/^Mem:/{print $7}'`, + `systemctl is-active k3s`, + `test -x /usr/local/bin/k3s || command -v k3s >/dev/null 2>&1`, + `test -e /etc/kubernetes/admin.conf`, + `sudo -n k3s certificate check 2>&1`, + `hostname`, + `date +%s`, + // The k3s kubectl base is not this transport's base: a command built + // for a node must not be run here with the node's paths. + `sudo -n k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get nodes -o json`, + } { + if _, _, err := l.parse(cmd); !errors.Is(err, ErrHostLayer) { + t.Errorf("parse(%q) error = %v, want ErrHostLayer", cmd, err) + } + } +} + +// Refusal has to arrive as exit 127, not as a Go error: callers probe with +// `command -v` and read the code, and an error would abort the diagnosis +// instead of recording that one probe found nothing. +func TestRunRefusesHostCommandWithExit127(t *testing.T) { + l := Local{Kubeconfig: "/k/c.yaml"} + out, code, err := l.Run(`systemctl is-active k3s`) + if err != nil { + t.Fatalf("Run returned error %v, want nil", err) + } + if code != 127 { + t.Errorf("exit code = %d, want 127", code) + } + if !strings.Contains(out, "kubeconfig cluster") { + t.Errorf("output %q does not explain why", out) + } +} + +func TestRunWithoutKubeconfigIsAnError(t *testing.T) { + var l Local + if _, _, err := l.Run("kubectl get pods"); err == nil { + t.Fatal("empty kubeconfig accepted; it would silently use ~/.kube/config") + } +} + +func TestSplitRejectsShellMetacharacters(t *testing.T) { + for _, s := range []string{ + `get pods | head -1`, + `get pods; rm -rf /`, + `get pods $(whoami)`, + "get pods `whoami`", + `get pods > /tmp/out`, + `get pods & echo hi`, + } { + if _, err := split(s); err == nil { + t.Errorf("split(%q) accepted a shell construct it cannot run", s) + } + } +} + +func TestSplitUnbalancedQuote(t *testing.T) { + if _, err := split(`get pods -n 'default`); err == nil { + t.Error("unbalanced quote accepted") + } +} + +// kube.Builder must take the transport's own base instead of probing for a +// distribution. Probing would fail every arm and silently produce k3s +// commands for a cluster that is not k3s. +func TestKubeBuilderUsesTransportBase(t *testing.T) { + l := Local{Kubeconfig: "/k/c.yaml", Context: "prod"} + got := kube.Builder(l)("get pods -A") + want := l.Base() + " get pods -A" + if got != want { + t.Errorf("Builder produced %q, want %q", got, want) + } +} + +func TestWriteAndRemoveFile(t *testing.T) { + l := Local{Kubeconfig: "/k/c.yaml"} + path := filepath.Join(t.TempDir(), "nested", "manifest.yaml") + if err := l.WriteFile(path, []byte("kind: Pod\n"), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat: %v", err) + } + // Manifests can carry Secrets; the mode has to survive the umask. + // + // Windows has no POSIX permission bits — Go maps a file mode onto the + // read-only attribute and reports 0666 for anything writable — so there is + // nothing to assert there. The file is still created under the user's own + // profile directory, which is where the protection comes from on that + // platform. + if runtime.GOOS != "windows" { + if perm := info.Mode().Perm(); perm != 0o600 { + t.Errorf("mode = %o, want 600", perm) + } + } + if err := l.RemoveFile(path); err != nil { + t.Fatalf("RemoveFile: %v", err) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Error("file still present after RemoveFile") + } + // Removing what is not there is not an error: callers defer it. + if err := l.RemoveFile(path); err != nil { + t.Errorf("second RemoveFile: %v", err) + } +} + +func TestStageDirIsWritable(t *testing.T) { + if dir := (Local{}).StageDir(); dir == "" { + t.Fatal("StageDir is empty") + } +} diff --git a/internal/kyaml/live.go b/internal/kyaml/live.go index 0fbf286..25c3303 100644 --- a/internal/kyaml/live.go +++ b/internal/kyaml/live.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "os" + "path/filepath" "strings" "github.com/solutionforest/k3helper/internal/kube" @@ -42,7 +43,7 @@ const KubectlBase = `sudo -n k3s kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml` func VerifyLive(t Target, data []byte, name string) (*Result, error) { res := Verify(data) - remote, err := RemotePath(name) + remote, err := RemotePathIn(StageDir(t), name) if err != nil { return res, err } @@ -69,9 +70,29 @@ func VerifyLive(t Target, data []byte, name string) (*Result, error) { return res, nil } +// StageDir is where a manifest is written on the machine kubectl runs on. +// +// It is asked of the transport rather than hardcoded because the local +// kubectl transport stages on this machine, which may be Windows and have no +// /tmp. Transports that do not answer get the SSH default, since every machine +// k3helper connects to over SSH is a Unix host with one. +func StageDir(t any) string { + if s, ok := t.(interface{ StageDir() string }); ok { + if dir := s.StageDir(); dir != "" { + return dir + } + } + return "/tmp" +} + // RemotePath builds a collision-resistant /tmp path from a manifest name, // keeping only characters that are safe inside a single-quoted shell word. func RemotePath(name string) (string, error) { + return RemotePathIn("/tmp", name) +} + +// RemotePathIn is RemotePath with the staging directory named explicitly. +func RemotePathIn(dir, name string) (string, error) { base := name if i := strings.LastIndexByte(base, '/'); i >= 0 { base = base[i+1:] @@ -95,5 +116,18 @@ func RemotePath(name string) (string, error) { if err != nil { return "", err } - return fmt.Sprintf("/tmp/k3helper-%s-%s", suffix, base), nil + return joinPath(dir, fmt.Sprintf("k3helper-%s-%s", suffix, base)), nil +} + +// joinPath joins in the style of the directory it is given. +// +// filepath.Join alone is wrong here: on a Windows machine deploying to a Linux +// node over SSH it would build "\tmp\manifest.yaml" for a path that the node +// has to read. The separator has to follow the target, not the host running +// k3helper. +func joinPath(dir, name string) string { + if strings.ContainsRune(dir, '\\') { + return filepath.Join(dir, name) + } + return strings.TrimSuffix(dir, "/") + "/" + name } diff --git a/internal/ssh/local.go b/internal/ssh/local.go index 802610e..686b397 100644 --- a/internal/ssh/local.go +++ b/internal/ssh/local.go @@ -5,6 +5,7 @@ import ( "io" "os" "os/exec" + "runtime" ) // This file backs nodes marked Local: k3helper running directly on the box it @@ -16,7 +17,26 @@ import ( const localShell = "/bin/sh" +// errNoLocalShell explains a `local: true` node on a platform that has no +// POSIX shell. k3helper builds and runs on Windows, but the local transport +// means "run these commands on the machine k3helper is on" — and every command +// it runs is a Linux one, aimed at a node that hosts k3s. There is nothing +// sensible to do here except say so. +// +// The bare failure is "exec: /bin/sh: executable file not found in %PATH%", +// which reads like a broken install rather than a targets file describing +// something that cannot exist. +func errNoLocalShell() error { + return fmt.Errorf("local: true needs a POSIX shell at %s, which %s does not have — "+ + "a local node is the machine k3helper runs on, and k3s nodes are Linux hosts. "+ + "Describe the node with host/user/key to reach it over SSH instead", + localShell, runtime.GOOS) +} + func runLocal(cmd string) (string, int, error) { + if runtime.GOOS == "windows" { + return "", -1, errNoLocalShell() + } out, err := exec.Command(localShell, "-c", cmd).CombinedOutput() if exitErr, ok := err.(*exec.ExitError); ok { return string(out), exitErr.ExitCode(), nil @@ -28,6 +48,9 @@ func runLocal(cmd string) (string, int, error) { } func streamLocal(cmd string, w io.Writer) (int, error) { + if runtime.GOOS == "windows" { + return -1, errNoLocalShell() + } c := exec.Command(localShell, "-c", cmd) c.Stdout = w c.Stderr = w diff --git a/internal/ssh/local_test.go b/internal/ssh/local_test.go index f3e4507..d57bc7f 100644 --- a/internal/ssh/local_test.go +++ b/internal/ssh/local_test.go @@ -4,10 +4,25 @@ import ( "bytes" "os" "path/filepath" + "runtime" "strings" "testing" ) +// requireLocalShell skips a test that needs the POSIX shell the local +// transport runs commands through. +// +// Local mode means "manage the machine k3helper is on", and the commands it +// runs are Linux ones aimed at a k3s node — so it is a Unix-only path by +// design, not an unimplemented one. Windows gets a clear error instead, which +// TestLocalOnWindowsExplainsItself covers. +func requireLocalShell(t *testing.T) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("local mode needs /bin/sh; on Windows it is refused with an explanation instead") + } +} + // dialLocal is the path a targets file with `local: true` takes. func dialLocal(t *testing.T) *Client { t.Helper() @@ -29,6 +44,7 @@ func TestDialLocalDoesNotConnect(t *testing.T) { } func TestLocalRunExitCodes(t *testing.T) { + requireLocalShell(t) c := dialLocal(t) out, code, err := c.Run("echo hello") @@ -53,6 +69,7 @@ func TestLocalRunExitCodes(t *testing.T) { } func TestLocalStream(t *testing.T) { + requireLocalShell(t) c := dialLocal(t) var buf bytes.Buffer code, err := c.Stream("echo streamed; exit 7", &buf) @@ -68,6 +85,7 @@ func TestLocalStream(t *testing.T) { } func TestLocalSudoPrefix(t *testing.T) { + requireLocalShell(t) c := dialLocal(t) want := "sudo -n " if os.Geteuid() == 0 { @@ -85,6 +103,7 @@ func TestLocalSudoPrefix(t *testing.T) { } func TestLocalWriteAndRemoveFile(t *testing.T) { + requireLocalShell(t) c := dialLocal(t) path := filepath.Join(t.TempDir(), "manifest.yaml") @@ -119,8 +138,28 @@ func TestLocalWriteAndRemoveFile(t *testing.T) { } func TestLocalWriteFileRejectsBadPath(t *testing.T) { + requireLocalShell(t) c := dialLocal(t) if err := c.WriteFile("/tmp/bad'name", []byte("x"), 0o600); err == nil { t.Error("expected quoted path to be rejected on the local transport too") } } + +// On Windows the local transport must explain itself rather than failing with +// "exec: /bin/sh: executable file not found", which reads like a broken +// install rather than a targets file describing something that cannot exist. +func TestLocalOnWindowsExplainsItself(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("the refusal only happens on Windows") + } + c := dialLocal(t) + _, _, err := c.Run("echo hi") + if err == nil { + t.Fatal("local mode ran a command on a platform with no POSIX shell") + } + for _, want := range []string{"local: true", "over SSH"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} diff --git a/internal/transport/forward_ssh.go b/internal/transport/forward_ssh.go new file mode 100644 index 0000000..8d18b8b --- /dev/null +++ b/internal/transport/forward_ssh.go @@ -0,0 +1,61 @@ +package transport + +import ( + "fmt" + "net" + "strconv" + + "github.com/solutionforest/k3helper/internal/kube" + "github.com/solutionforest/k3helper/internal/ssh" +) + +// sshForward is a port-forward opened in two parts: kubectl binds a port on +// the cluster node, and an SSH tunnel carries it to this machine. Both halves +// are needed — without the tunnel the port is open on the node and nowhere the +// operator can reach. +// +// This is the code that used to live in the TUI. It moved here so the TUI can +// ask for "a forward" without knowing which transport it holds. +type sshForward struct { + pf *kube.PortForward + tunnel *ssh.Tunnel + local int +} + +func (f *sshForward) LocalPort() int { return f.local } + +func (f *sshForward) Close() error { + if f.tunnel != nil { + f.tunnel.Close() + f.tunnel = nil + } + if f.pf != nil { + f.pf.Stop() + f.pf = nil + } + return nil +} + +func startSSHForward(client *ssh.Client, ns, target string, remotePort, seq int) (Forward, error) { + if client == nil { + return nil, fmt.Errorf("no server connection") + } + // A node-side port distinct from the local one, so several forwards can + // coexist and neither side collides with something already bound. + nodePort := 39000 + seq + pf, err := kube.StartPortForward(client, ns, target, remotePort, nodePort) + if err != nil { + return nil, err + } + // :0 lets the OS choose a free local port and report which. + tunnel, err := client.Forward("127.0.0.1:0", pf.NodeAddr()) + if err != nil { + pf.Stop() + return nil, err + } + local := 0 + if _, portStr, e := net.SplitHostPort(tunnel.LocalAddr); e == nil { + local, _ = strconv.Atoi(portStr) + } + return &sshForward{pf: pf, tunnel: tunnel, local: local}, nil +} diff --git a/internal/transport/transport.go b/internal/transport/transport.go new file mode 100644 index 0000000..d796ecf --- /dev/null +++ b/internal/transport/transport.go @@ -0,0 +1,160 @@ +// Package transport opens the connections k3helper works through, and is the +// one place that decides how a cluster is reached. +// +// A cluster is described either by its nodes (SSH to a machine, run the +// kubectl that lives there) or by a kubeconfig (run kubectl here, talk to the +// API server). Both answer the same interface, so callers ask for a cluster +// executor and stop caring — except where they genuinely need a machine, which +// is what RequireHosts is for. +// +// Centralising this is not tidiness. Each command used to dial for itself, and +// that is how `local` and the host-key policy each got silently dropped from +// one call site; a second way to reach a cluster would multiply that. +package transport + +import ( + "fmt" + "os" + + "github.com/solutionforest/k3helper/internal/config" + "github.com/solutionforest/k3helper/internal/kubectl" + "github.com/solutionforest/k3helper/internal/ssh" +) + +// Cluster is a machine kubectl runs on: it executes commands and can stage a +// file for kubectl to read. Satisfied by *ssh.Client and by kubectl.Local. +// +// The file methods are here because `deploy` and `verify --live` name a +// manifest on the operator's machine and kubectl reads it on the cluster's. +// Over SSH those differ; through a kubeconfig they are the same machine, and +// the write is a local one. +type Cluster interface { + Run(cmd string) (string, int, error) + WriteFile(path string, data []byte, mode os.FileMode) error + RemoveFile(path string) error + Close() error +} + +// NodeFailure is a node named in the targets file that could not be contacted. +type NodeFailure struct { + Name string + Reason string +} + +// Server opens the connection cluster-layer commands run through. +// +// In SSH mode that is a connection to the first server node. In kubeconfig +// mode it is a local kubectl, and nothing is dialled at all. +func Server(t *config.Targets) (Cluster, error) { + if t.Mode() == config.ModeKubeconfig { + return Local(t) + } + srv, err := t.Server() + if err != nil { + return nil, err + } + c, err := ssh.Dial(srv.SSH()) + if err != nil { + return nil, fmt.Errorf("connect to server: %w", err) + } + return c, nil +} + +// Local builds the kubeconfig transport for a cluster, checking up front that +// kubectl exists. Finding that out mid-diagnosis produces a much worse message +// than finding it out at the front door. +func Local(t *config.Targets) (Cluster, error) { + if err := kubectl.Available(); err != nil { + return nil, err + } + return kubectl.Local{Kubeconfig: t.KubeconfigPath(), Context: t.KubeContext}, nil +} + +// ServerName is the name to print for whatever Server connected to. +func ServerName(t *config.Targets) string { + if t.Mode() == config.ModeKubeconfig { + if t.KubeContext != "" { + return t.KubeContext + " (kubeconfig)" + } + return "kubeconfig" + } + if srv, err := t.Server(); err == nil { + return srv.Name + } + return t.Cluster +} + +// Hosts opens a connection to every node for host-layer probes, reusing the +// one already open for the server node. +// +// A kubeconfig cluster has no hosts and returns an empty map with no failures: +// that is the absence of a layer, not a fault, and callers distinguish the two. +// A node that cannot be dialled is returned as a NodeFailure rather than +// skipped, because silently dropping it lets a diagnosis report a healthy +// cluster while a machine is down. +func Hosts(t *config.Targets, server Cluster, serverName string) (map[string]ssh.Executor, []NodeFailure, func()) { + hosts := map[string]ssh.Executor{} + var failures []NodeFailure + var open []*ssh.Client + closeAll := func() { + for _, c := range open { + c.Close() + } + } + if t.Mode() == config.ModeKubeconfig { + return hosts, nil, closeAll + } + for _, n := range t.Nodes { + if n.Name == serverName { + if exec, ok := server.(ssh.Executor); ok { + hosts[n.Name] = exec + } + continue + } + c, err := ssh.Dial(n.SSH()) + if err != nil { + failures = append(failures, NodeFailure{Name: n.Name, Reason: err.Error()}) + continue + } + open = append(open, c) + hosts[n.Name] = c + } + return hosts, failures, closeAll +} + +// Forward is a live port-forward, reachable on this machine. +type Forward interface { + // LocalPort is the port bound here. + LocalPort() int + Close() error +} + +// StartForward opens a port-forward to a pod or service and returns once it is +// listening locally. +// +// seq only matters for the SSH transport, which has to pick a distinct port on +// the cluster node for each forward so that several can coexist. +func StartForward(c Cluster, ns, target string, remotePort, seq int) (Forward, error) { + switch t := c.(type) { + case kubectl.Local: + return t.StartForward(ns, target, remotePort) + case *ssh.Client: + return startSSHForward(t, ns, target, remotePort, seq) + default: + return nil, fmt.Errorf("port-forward is not supported by this connection") + } +} + +// RequireHosts refuses an operation that has to run on the machines +// themselves. `vm setup` installs a distribution and `registry` writes +// containerd configuration; neither has any meaning without a host, and doing +// half of one is worse than doing none. +func RequireHosts(t *config.Targets, what string) error { + if t.Mode() != config.ModeKubeconfig { + return nil + } + return fmt.Errorf("%s needs SSH access to the cluster's machines, and %q is reached through a kubeconfig\n\n"+ + "A kubeconfig reaches the API server, not the nodes behind it. To use this command, describe the "+ + "nodes in the targets file instead. On a managed cluster (EKS/GKE/AKS) this is the provider's job, "+ + "not k3helper's.", what, t.Cluster) +} diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go new file mode 100644 index 0000000..5e6c6a2 --- /dev/null +++ b/internal/transport/transport_test.go @@ -0,0 +1,101 @@ +package transport + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/solutionforest/k3helper/internal/config" + "github.com/solutionforest/k3helper/internal/kubectl" +) + +func kubeconfigCluster(t *testing.T) *config.Targets { + t.Helper() + path := filepath.Join(t.TempDir(), "config") + if err := os.WriteFile(path, []byte("apiVersion: v1\nkind: Config\n"), 0o600); err != nil { + t.Fatalf("write kubeconfig: %v", err) + } + return &config.Targets{Cluster: "prod", Kubeconfig: path, KubeContext: "admin"} +} + +func sshCluster() *config.Targets { + return &config.Targets{Cluster: "lab", Nodes: []config.Node{ + {Name: "server", Role: "server", Host: "10.0.0.1", Port: 22, User: "root"}, + {Name: "agent1", Role: "agent", Host: "10.0.0.2", Port: 22, User: "root"}, + }} +} + +// A kubeconfig cluster has no hosts, and that is the absence of a layer rather +// than a fault. Reporting the nodes as unreachable would make every managed +// cluster look degraded. +func TestHostsIsEmptyForKubeconfigCluster(t *testing.T) { + tg := kubeconfigCluster(t) + server, err := Local(tg) + if err != nil { + t.Skipf("kubectl not available: %v", err) + } + hosts, failures, closeAll := Hosts(tg, server, ServerName(tg)) + defer closeAll() + if len(hosts) != 0 { + t.Errorf("got %d hosts for a kubeconfig cluster, want none", len(hosts)) + } + if len(failures) != 0 { + t.Errorf("got %d node failures, want none — there are no nodes to fail", len(failures)) + } +} + +func TestRequireHostsRefusesKubeconfigCluster(t *testing.T) { + tg := kubeconfigCluster(t) + err := RequireHosts(tg, "vm setup") + if err == nil { + t.Fatal("vm setup was allowed against a cluster with no machines") + } + for _, want := range []string{"vm setup", "prod", "kubeconfig"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} + +func TestRequireHostsAllowsSSHCluster(t *testing.T) { + if err := RequireHosts(sshCluster(), "vm setup"); err != nil { + t.Errorf("vm setup refused for an SSH cluster: %v", err) + } +} + +func TestServerNameNamesTheContext(t *testing.T) { + if got := ServerName(kubeconfigCluster(t)); !strings.Contains(got, "admin") { + t.Errorf("ServerName() = %q, want the kube context in it", got) + } + if got := ServerName(sshCluster()); got != "server" { + t.Errorf("ServerName() = %q, want the server node's name", got) + } +} + +// Server must not dial anything for a kubeconfig cluster, and must hand back a +// transport pointed at the file the targets named. +func TestServerForKubeconfigClusterIsLocalKubectl(t *testing.T) { + tg := kubeconfigCluster(t) + c, err := Server(tg) + if err != nil { + t.Skipf("kubectl not available: %v", err) + } + defer c.Close() + l, ok := c.(kubectl.Local) + if !ok { + t.Fatalf("Server returned %T, want kubectl.Local", c) + } + if l.Kubeconfig != tg.KubeconfigPath() { + t.Errorf("kubeconfig = %q, want %q", l.Kubeconfig, tg.KubeconfigPath()) + } + if l.Context != "admin" { + t.Errorf("context = %q, want admin", l.Context) + } +} + +// kubectl.Local must satisfy everything deploy and verify --live need, or the +// mismatch only shows up the first time somebody deploys to a managed cluster. +func TestLocalSatisfiesCluster(t *testing.T) { + var _ Cluster = kubectl.Local{} +} diff --git a/internal/troubleshoot/gather.go b/internal/troubleshoot/gather.go index db07394..d080a4b 100644 --- a/internal/troubleshoot/gather.go +++ b/internal/troubleshoot/gather.go @@ -23,6 +23,9 @@ type Gatherer struct { // to assess etcd quorum from host evidence when the API server — the thing // quorum loss takes down — cannot be reached to ask. ServerNodes []string + // NoHostLayer says there is no host layer to gather, as opposed to one + // that was not reachable. Set for kubeconfig clusters. + NoHostLayer bool } // Collect assembles an Evidence bundle. Never fails: collection problems @@ -50,6 +53,8 @@ func (g Gatherer) Collect() Evidence { NodeAlias: map[string]string{}, ServerNodes: g.ServerNodes, Unreachable: g.Unreachable, + + HostLayerUnavailable: g.NoHostLayer, } // --- cluster layer via kubectl --- diff --git a/internal/troubleshoot/hostlayer_test.go b/internal/troubleshoot/hostlayer_test.go new file mode 100644 index 0000000..54b486e --- /dev/null +++ b/internal/troubleshoot/hostlayer_test.go @@ -0,0 +1,173 @@ +package troubleshoot + +import ( + "strings" + "testing" + + "github.com/solutionforest/k3helper/internal/ssh" +) + +// fakeExec answers a fixed script of commands, so a gather can be driven with +// no cluster and no SSH. +type fakeExec struct{ responses map[string]string } + +func (f fakeExec) Run(cmd string) (string, int, error) { + for prefix, out := range f.responses { + if strings.Contains(cmd, prefix) { + return out, 0, nil + } + } + return "", 1, nil +} + +// A kubeconfig cluster reports the missing host layer as a note on the scope +// of the diagnosis, never as a fault. +func TestHostLayerUnavailableIsInformational(t *testing.T) { + ds := Diagnose(Evidence{HostLayerUnavailable: true}) + var found bool + for _, d := range ds { + if d.SignatureID == "cluster.host-layer-unavailable" { + found = true + if !d.Informational() { + t.Error("the host-layer note is treated as a fault; it would fail every managed cluster's CI run") + } + if !strings.Contains(d.Remediation, "kubeconfig") { + t.Errorf("remediation %q does not explain the limit", d.Remediation) + } + } + } + if !found { + t.Fatal("no host-layer finding for a cluster with no host layer — silence reads as a clean bill of health") + } + if !OnlyInformational(ds) { + t.Error("a healthy kubeconfig cluster produced a non-informational finding") + } +} + +func TestHostLayerSignatureSilentForSSHCluster(t *testing.T) { + for _, d := range Diagnose(Evidence{}) { + if d.SignatureID == "cluster.host-layer-unavailable" { + t.Error("the host-layer note fired for a cluster that has one") + } + } +} + +// The two ways of seeing less must stay distinct. "We tried and failed" is a +// fault; "there was nothing to try" is how managed clusters work. +func TestHostLayerIsNotAProbeError(t *testing.T) { + e := Evidence{HostLayerUnavailable: true} + if len(e.ProbeErrors) != 0 { + t.Error("the missing host layer was recorded as a failed probe") + } + for _, d := range Diagnose(e) { + if d.SignatureID == "cluster.partial-evidence" { + t.Error("a kubeconfig cluster reported failed probes; it would send the operator to check RBAC") + } + if d.SignatureID == "node.unreachable" { + t.Error("a kubeconfig cluster reported unreachable nodes; it has none") + } + } +} + +// Real faults still have to be reported, and still have to set the exit code. +func TestClusterFaultsStillFireWithoutHostLayer(t *testing.T) { + e := Evidence{ + HostLayerUnavailable: true, + PodStatuses: map[string]string{"default/web": "CrashLoopBackOff"}, + } + ds := Diagnose(e) + if OnlyInformational(ds) { + t.Fatal("a crash-looping pod produced only scope notes") + } + var found bool + for _, d := range ds { + if d.SignatureID == "pod.crashloop" { + found = true + } + } + if !found { + t.Error("pod.crashloop did not fire on a kubeconfig cluster") + } +} + +// Gatherer must carry the flag through: setting it on the gatherer and losing +// it in the evidence is how the note would go missing. +func TestGathererCarriesNoHostLayer(t *testing.T) { + g := Gatherer{ + Server: fakeExec{responses: map[string]string{}}, + Hosts: map[string]ssh.Executor{}, + NoHostLayer: true, + } + e := g.Collect() + if !e.HostLayerUnavailable { + t.Error("Gatherer.NoHostLayer did not reach the evidence") + } +} + +// A crashlooping container is only reported as CrashLoopBackOff while it is +// waiting between attempts. Sampled the moment it restarts, the pod is Running +// with no reason attached — which used to read as nothing worse than an +// unready pod, and made the fault matrix catch the fault or miss it depending +// on when doctor happened to look. +func TestCrashLoopFoundFromRestartsWhenSampledMidRestart(t *testing.T) { + e := Evidence{ + NotReadyPods: []string{"default/fault-crashloop"}, + PodRestarts: map[string]int{"default/fault-crashloop": 5}, + // No PodStatuses entry: kubectl said Running, because it was. + } + var found bool + for _, d := range Diagnose(e) { + if d.SignatureID == "pod.crashloop" { + found = true + } + } + if !found { + t.Error("a pod that is not ready after 5 restarts was not called a crash loop") + } +} + +// The waiting half of the cycle must still work, and the same pod seen both +// ways must not be counted twice — a doubled count inflates the confidence. +func TestCrashLoopCountsEachPodOnce(t *testing.T) { + both := Evidence{ + PodStatuses: map[string]string{"default/web": "CrashLoopBackOff"}, + NotReadyPods: []string{"default/web"}, + PodRestarts: map[string]int{"default/web": 9}, + } + waitingOnly := Evidence{ + PodStatuses: map[string]string{"default/web": "CrashLoopBackOff"}, + } + if got, want := confidenceOf(Diagnose(both), "pod.crashloop"), + confidenceOf(Diagnose(waitingOnly), "pod.crashloop"); got != want { + t.Errorf("one pod seen two ways scored %d, want %d", got, want) + } +} + +// A pod that fell over once and came back is not a crash loop, and a pod that +// is simply slow to pass readiness is not one either. Reporting them would +// send people to `kubectl logs --previous` for a healthy rollout. +func TestCrashLoopIgnoresFewRestartsAndReadyPods(t *testing.T) { + tests := []struct { + name string + e Evidence + }{ + {"one restart, still starting up", Evidence{ + NotReadyPods: []string{"default/web"}, + PodRestarts: map[string]int{"default/web": 1}, + }}, + {"never restarted, failing readiness", Evidence{ + NotReadyPods: []string{"default/web"}, + PodRestarts: map[string]int{"default/web": 0}, + }}, + {"restarted often but now ready", Evidence{ + PodRestarts: map[string]int{"default/web": 12}, + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if c := confidenceOf(Diagnose(tc.e), "pod.crashloop"); c != 0 { + t.Errorf("pod.crashloop fired at %d%%", c) + } + }) + } +} diff --git a/internal/troubleshoot/parse.go b/internal/troubleshoot/parse.go index 49dbad2..bc78f52 100644 --- a/internal/troubleshoot/parse.go +++ b/internal/troubleshoot/parse.go @@ -114,6 +114,14 @@ func parsePods(data string, e *Evidence) bool { // separately or it goes entirely unreported. if total > 0 && ready < total { e.NotReadyPods = append(e.NotReadyPods, key) + // Keep the restart count with it. A crashlooping container + // spends part of its cycle Running rather than waiting, and a + // diagnosis taken in that window sees an unready pod with no + // reason attached — the restarts are what tell the two apart. + if e.PodRestarts == nil { + e.PodRestarts = map[string]int{} + } + e.PodRestarts[key] = restarts } case "Succeeded": e.settledPods[key] = true diff --git a/internal/troubleshoot/signatures.go b/internal/troubleshoot/signatures.go index 711e825..3bda9c7 100644 --- a/internal/troubleshoot/signatures.go +++ b/internal/troubleshoot/signatures.go @@ -33,6 +33,14 @@ type Evidence struct { // NotReadyPods: namespace/pod of pods that are Running but have containers // that never became ready. NotReadyPods []string + // PodRestarts: namespace/pod → total container restarts, for pods that are + // Running and not ready. + // + // A crashlooping container only reads as "CrashLoopBackOff" while it is + // waiting between attempts; the moment it starts again the status is + // Running, and a diagnosis taken then sees an unready pod and no reason. + // The restart count is the durable evidence, so it is kept. + PodRestarts map[string]int // K3sService: node → systemd state ("active", "failed", "inactive", ""=not found) K3sService map[string]string // HostMetrics: node → {disk used %, availMemMB} @@ -72,6 +80,16 @@ type Evidence struct { // only good news if we actually looked, so these are reported rather than // silently narrowing the diagnosis. ProbeErrors map[string]string + // HostLayerUnavailable: this cluster is reached through a kubeconfig, so + // there is no host layer to gather — no disks, no systemd units, no + // container runtime state. + // + // Deliberately not a ProbeError and deliberately not an Unreachable node. + // Both of those mean "we tried and failed", which is a fault. This means + // "there was never anything there to try", which is how managed clusters + // work and is not a fault. Conflating them would report every healthy EKS + // cluster as degraded. + HostLayerUnavailable bool // livePVCs: namespace/pvc keys of PVCs that currently exist. livePVCs map[string]bool @@ -85,6 +103,12 @@ type Evidence struct { livePods map[string]bool } +// restartsMeaningCrashLoop is how many restarts an unready pod needs before it +// is called a crash loop rather than a pod that had a bad start. kubelet's +// backoff reaches 40s by the third restart, so a pod at this count has been +// failing for the best part of a minute. +const restartsMeaningCrashLoop = 3 + // HostMetric is a node's host-level vitals. type HostMetric struct { DiskUsedPercent int @@ -136,6 +160,30 @@ type Diagnosis struct { Evidence string `json:"evidence"` } +// Informational reports whether this finding describes the scope of the +// diagnosis rather than a fault in the cluster. +// +// These must not set doctor's exit code. An RBAC-scoped kubeconfig or a +// managed cluster with no host layer would otherwise fail every CI run while +// being perfectly healthy. +func (d Diagnosis) Informational() bool { + switch d.SignatureID { + case "cluster.partial-evidence", "cluster.host-layer-unavailable": + return true + } + return false +} + +// OnlyInformational reports whether nothing but scope notes were found. +func OnlyInformational(ds []Diagnosis) bool { + for _, d := range ds { + if !d.Informational() { + return false + } + } + return true +} + // pullFailures counts image-pull failures by what the registry actually said. type pullFailures struct { auth int @@ -227,7 +275,20 @@ var registry = []Signature{ // exists so "nothing found" is never mistaken for "nothing wrong". return 30 }, - Remediation: "One or more probes failed, so faults they would have caught cannot be ruled out. Check that the kubeconfig the server node uses has list access to pods, events, PVCs, endpoints and services, and that the API server is responsive.", + Remediation: "One or more probes failed, so faults they would have caught cannot be ruled out. Check that the credentials in use have list access to pods, events, PVCs, endpoints and services, and that the API server is responsive.", + }, + { + ID: "cluster.host-layer-unavailable", + Title: "Host layer not visible — findings cover the cluster only", + Match: func(e Evidence) int { + if !e.HostLayerUnavailable { + return 0 + } + // Low, like cluster.partial-evidence: this is a note on the scope + // of the diagnosis, not a fault competing with real findings. + return 20 + }, + Remediation: "This cluster is reached through a kubeconfig, so k3helper can see the API server but not the machines behind it. Disk pressure, swap, cgroups, systemd units, the container runtime and certificate expiry are not checked. For a self-managed cluster, describe its nodes in the targets file to get the host layer as well; for a managed cluster, that layer belongs to the provider.", }, { ID: "node.unreachable", @@ -560,13 +621,27 @@ var registry = []Signature{ ID: "pod.crashloop", Title: "Container crashing on start (CrashLoopBackOff)", Match: func(e Evidence) int { - n := 0 - for _, st := range e.PodStatuses { + crashing := map[string]bool{} + for pod, st := range e.PodStatuses { if st == "CrashLoopBackOff" { - n++ + crashing[pod] = true } } - return min(90, n*40) + // A container in backoff is only reported as CrashLoopBackOff while + // it is waiting; between attempts it is Running with no reason + // attached, and a diagnosis taken in that window used to see + // nothing but an unready pod. Restarts are what does not go away: + // a pod that is not ready and has restarted repeatedly is + // crashlooping whichever half of the cycle we happened to catch. + // + // The threshold is deliberately not 1. A single restart is a pod + // that fell over once and came back, which is not this finding. + for _, pod := range e.NotReadyPods { + if e.PodRestarts[pod] >= restartsMeaningCrashLoop { + crashing[pod] = true + } + } + return min(90, len(crashing)*40) }, Remediation: "Read the container logs: `kubectl logs --previous`. Typical causes: bad command/args, missing config/env, app failing at startup. Fix the cause — do NOT use restart policies to mask it.", }, diff --git a/internal/tui/browser.go b/internal/tui/browser.go index 7cb0d32..f8b58b4 100644 --- a/internal/tui/browser.go +++ b/internal/tui/browser.go @@ -2,7 +2,6 @@ package tui import ( "fmt" - "net" "regexp" "sort" "strconv" @@ -14,7 +13,7 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/solutionforest/k3helper/internal/kube" - "github.com/solutionforest/k3helper/internal/ssh" + "github.com/solutionforest/k3helper/internal/transport" ) // view identifies what the browser is currently showing. @@ -737,8 +736,7 @@ type forward struct { Target string RemotePort int LocalPort int - pf *kube.PortForward - tunnel *ssh.Tunnel + fwd transport.Forward } func (f forward) String() string { @@ -751,52 +749,36 @@ type forwardsMsg struct { err error } -// startForward opens a port-forward on the node and tunnels it here. +// startForward opens a port-forward and reports the local port it landed on. // -// Both halves are needed: kubectl binds on the cluster node, so without the -// tunnel the port is open there and not on the operator's machine. -func startForward(client *ssh.Client, ns, target string, remotePort int, existing []forward) tea.Cmd { +// How that happens depends on the transport: over SSH kubectl binds on the +// cluster node and a tunnel carries the port here, while through a kubeconfig +// kubectl binds here directly. The TUI does not need to know which. +func startForward(client transport.Cluster, ns, target string, remotePort int, existing []forward) tea.Cmd { return func() tea.Msg { if client == nil { return forwardsMsg{forwards: existing, err: fmt.Errorf("no server connection")} } - // A node-side port distinct from the local one, so several forwards - // can coexist and neither side collides with something already bound. - nodePort := 39000 + len(existing) - pf, err := kube.StartPortForward(client, ns, target, remotePort, nodePort) + fwd, err := transport.StartForward(client, ns, target, remotePort, len(existing)) if err != nil { return forwardsMsg{forwards: existing, err: err} } - // :0 lets the OS choose a free local port and report which. - tunnel, err := client.Forward("127.0.0.1:0", pf.NodeAddr()) - if err != nil { - pf.Stop() - return forwardsMsg{forwards: existing, err: err} - } - local := 0 - if _, portStr, e := net.SplitHostPort(tunnel.LocalAddr); e == nil { - local, _ = strconv.Atoi(portStr) - } f := forward{ - Namespace: ns, Target: target, RemotePort: remotePort, LocalPort: local, - pf: pf, tunnel: tunnel, + Namespace: ns, Target: target, RemotePort: remotePort, + LocalPort: fwd.LocalPort(), fwd: fwd, } return forwardsMsg{forwards: append(existing, f)} } } -// stopForward tears one down, remote process and tunnel both. +// stopForward tears one down. func stopForward(fs []forward, idx int) tea.Cmd { return func() tea.Msg { if idx < 0 || idx >= len(fs) { return forwardsMsg{forwards: fs} } - f := fs[idx] - if f.tunnel != nil { - f.tunnel.Close() - } - if f.pf != nil { - f.pf.Stop() + if f := fs[idx]; f.fwd != nil { + f.fwd.Close() } return forwardsMsg{forwards: append(append([]forward{}, fs[:idx]...), fs[idx+1:]...)} } diff --git a/internal/tui/doctor.go b/internal/tui/doctor.go index eee9fe8..0293aa9 100644 --- a/internal/tui/doctor.go +++ b/internal/tui/doctor.go @@ -9,7 +9,7 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/solutionforest/k3helper/internal/config" - "github.com/solutionforest/k3helper/internal/ssh" + "github.com/solutionforest/k3helper/internal/transport" "github.com/solutionforest/k3helper/internal/troubleshoot" ) @@ -29,30 +29,17 @@ type doctorMsg struct { // it was made. func runDoctor(targets *config.Targets) tea.Cmd { return func() tea.Msg { - srvNode, err := targets.Server() + server, err := transport.Server(targets) if err != nil { return doctorMsg{err: err} } - server, err := ssh.Dial(srvNode.SSH()) - if err != nil { - return doctorMsg{err: fmt.Errorf("connect to server: %w", err)} - } defer server.Close() - hosts := map[string]ssh.Executor{} + hosts, failures, closeHosts := transport.Hosts(targets, server, transport.ServerName(targets)) + defer closeHosts() var unreachable []troubleshoot.UnreachableNode - for _, n := range targets.Nodes { - if n.Name == srvNode.Name { - hosts[n.Name] = server - continue - } - c, err := ssh.Dial(n.SSH()) - if err != nil { - unreachable = append(unreachable, troubleshoot.UnreachableNode{Name: n.Name, Reason: err.Error()}) - continue - } - defer c.Close() - hosts[n.Name] = c + for _, f := range failures { + unreachable = append(unreachable, troubleshoot.UnreachableNode{Name: f.Name, Reason: f.Reason}) } var serverNames []string for _, n := range targets.Servers() { @@ -60,6 +47,7 @@ func runDoctor(targets *config.Targets) tea.Cmd { } evidence := troubleshoot.Gatherer{ Server: server, Hosts: hosts, Unreachable: unreachable, ServerNodes: serverNames, + NoHostLayer: targets.Mode() == config.ModeKubeconfig, }.Collect() return doctorMsg{ diagnoses: troubleshoot.Diagnose(evidence), diff --git a/internal/tui/studio.go b/internal/tui/studio.go index 608bb23..22e9059 100644 --- a/internal/tui/studio.go +++ b/internal/tui/studio.go @@ -12,7 +12,7 @@ import ( "github.com/solutionforest/k3helper/internal/deploy" "github.com/solutionforest/k3helper/internal/kyaml" - "github.com/solutionforest/k3helper/internal/ssh" + "github.com/solutionforest/k3helper/internal/transport" ) // --- YAML studio (:gen) ------------------------------------------------------ @@ -126,7 +126,7 @@ type deployMsg struct { // dryRunDeploy shows what applying a manifest would change, without changing // anything. The operator applies it with a second keystroke. -func dryRunDeploy(client *ssh.Client, path, namespace string) tea.Cmd { +func dryRunDeploy(client transport.Cluster, path, namespace string) tea.Cmd { return func() tea.Msg { if client == nil { return deployMsg{file: path, err: fmt.Errorf("no server connection")} @@ -145,7 +145,7 @@ func dryRunDeploy(client *ssh.Client, path, namespace string) tea.Cmd { } // applyDeploy applies for real and waits for rollout. -func applyDeploy(client *ssh.Client, path, namespace string) tea.Cmd { +func applyDeploy(client transport.Cluster, path, namespace string) tea.Cmd { return func() tea.Msg { if client == nil { return deployMsg{file: path, applied: true, err: fmt.Errorf("no server connection")} diff --git a/internal/tui/tui.go b/internal/tui/tui.go index 5040f96..03beac6 100644 --- a/internal/tui/tui.go +++ b/internal/tui/tui.go @@ -20,6 +20,7 @@ import ( "github.com/solutionforest/k3helper/internal/kube" "github.com/solutionforest/k3helper/internal/kyaml" "github.com/solutionforest/k3helper/internal/ssh" + "github.com/solutionforest/k3helper/internal/transport" "github.com/solutionforest/k3helper/internal/troubleshoot" "github.com/solutionforest/k3helper/internal/vm" ) @@ -40,7 +41,7 @@ type checkDoneMsg struct { // serverReadyMsg carries the connection used for cluster queries. type serverReadyMsg struct { - client *ssh.Client + client transport.Cluster err error } @@ -67,9 +68,10 @@ type Model struct { // status is a transient one-line notice (a file saved, a context switched). status string - // server is the connection cluster queries run through. Nil until the - // dial completes, or when it failed. - server *ssh.Client + // server is the connection cluster queries run through. Nil until it is + // opened, or when opening failed. It is an interface because the cluster + // may be reached over SSH or through a local kubectl and a kubeconfig. + server transport.Cluster serverErr error // resource browser state @@ -173,12 +175,11 @@ func tickEvery() tea.Cmd { // dialServer opens the connection cluster queries run through. func dialServer(targets *config.Targets) tea.Cmd { return func() tea.Msg { - srv, err := targets.Server() + c, err := transport.Server(targets) if err != nil { return serverReadyMsg{err: err} } - c, err := ssh.Dial(srv.SSH()) - return serverReadyMsg{client: c, err: err} + return serverReadyMsg{client: c} } } @@ -186,6 +187,12 @@ func doChecks(targets *config.Targets) tea.Cmd { return func() tea.Msg { results := map[string][]check.Result{} samples := map[string]hostSample{} + if targets.Mode() == config.ModeKubeconfig { + // No machines to probe. Report the host checks as skipped rather + // than as an empty dashboard, which would read as "all clear". + results[transport.ServerName(targets)] = check.SkippedHostResults("server") + return checkDoneMsg{nodeResults: results, samples: samples} + } for _, node := range targets.Nodes { client, err := ssh.Dial(node.SSH()) if err != nil { @@ -195,14 +202,7 @@ func doChecks(targets *config.Targets) tea.Cmd { }} continue } - runner := check.NewRunner( - check.DiskUsageCheck{}, - check.MemoryCheck{}, - check.SwapCheck{}, - check.CgroupCheck{}, - check.ServiceCheck{Role: node.Role}, - check.RegistryCheck{}, - ) + runner := check.NewRunner(check.HostChecks(node.Role)...) results[node.Name] = runner.RunAll(check.Context{Exec: execAdapter{client}, Node: node.Name}) samples[node.Name] = sampleHost(client) client.Close() @@ -800,11 +800,8 @@ func (m Model) switchContext() (tea.Model, tea.Cmd) { m.server.Close() } for _, f := range m.forwards { - if f.tunnel != nil { - f.tunnel.Close() - } - if f.pf != nil { - f.pf.Stop() + if f.fwd != nil { + f.fwd.Close() } } m.targets = t diff --git a/scripts/portable-check.sh b/scripts/portable-check.sh new file mode 100755 index 0000000..34935a7 --- /dev/null +++ b/scripts/portable-check.sh @@ -0,0 +1,95 @@ +#!/bin/sh +# Assert that a built k3helper is actually portable: it runs from a directory +# it has never seen, with nothing installed, and writes nothing outside that +# directory. +# +# scripts/portable-check.sh ./bin/k3helper +# +# "Single portable binary" is the first line of the README. It is a claim about +# the runtime, not the build, so it has to be checked by running the thing — +# a cross-compile that succeeds says nothing about whether the binary needs a +# config directory, a writable home, or a shell that is not there. +# +# POSIX sh: this runs on Linux and macOS runners, and by hand on a developer's +# machine. The Windows equivalent lives in the workflow, in PowerShell, so that +# nothing POSIX is involved in proving the .exe stands alone. +set -eu + +BIN="${1:-}" +[ -n "$BIN" ] || { echo "usage: $0 " >&2; exit 1; } + +# Resolve before leaving the current directory. +case "$BIN" in + /*) ;; + *) BIN="$(pwd)/$BIN" ;; +esac +[ -x "$BIN" ] || { echo "error: $BIN is not executable" >&2; exit 1; } + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT INT TERM +cd "$WORK" + +pass() { printf ' ✓ %s\n' "$*"; } +fail() { printf ' ✗ %s\n' "$*" >&2; exit 1; } + +echo "portable check: $BIN" +echo " in: $WORK" + +# --- it runs at all ------------------------------------------------------- +"$BIN" version >/dev/null || fail "version failed" +pass "version" + +"$BIN" --help >/dev/null || fail "--help failed" +pass "--help" + +# --- a missing targets file is explained, not stack-traced ---------------- +if "$BIN" ctx >/dev/null 2>&1; then + fail "ctx succeeded in an empty directory with no targets file" +fi +out=$("$BIN" ctx 2>&1 || true) +case "$out" in + *"no targets file"*) pass "missing targets file explains itself" ;; + *) fail "unhelpful error for a missing targets file: $out" ;; +esac + +# --- a kubeconfig that is not there is refused, and nothing is written ---- +# Paths are relative on purpose: we are already in $WORK, and Git Bash on +# Windows rewrites POSIX-looking absolute arguments on their way to a native +# binary. A relative path takes that translation out of the test, which is +# about k3helper rather than about MSYS. +if "$BIN" init --kubeconfig missing.yaml --cluster demo >/dev/null 2>&1; then + fail "init accepted a kubeconfig that does not exist" +fi +[ -f targets.yaml ] && fail "init wrote a targets file after failing" +pass "missing kubeconfig refused, nothing written" + +# --- the kubeconfig flow works end to end -------------------------------- +printf 'apiVersion: v1\nkind: Config\n' > kube.yaml +"$BIN" init --kubeconfig kube.yaml --kube-context admin --cluster demo >/dev/null \ + || fail "init failed on a valid kubeconfig" +[ -f targets.yaml ] || fail "init reported success but wrote no file" +pass "init --kubeconfig" + +"$BIN" ctx | grep -q kubeconfig || fail "ctx does not report the cluster as kubeconfig-reached" +pass "ctx reports how the cluster is reached" + +# --- SSH mode still works from a bare directory too ---------------------- +"$BIN" init --out ssh-targets.yaml --server 10.0.0.10 --agent 10.0.0.11 \ + --user ubuntu --cluster lab >/dev/null || fail "init failed for an SSH cluster" +"$BIN" ctx -t ssh-targets.yaml | grep -q ssh || fail "ctx does not report an SSH cluster" +pass "init/ctx for an SSH cluster" + +# --- offline YAML work needs no cluster at all --------------------------- +"$BIN" gen deployment web --image nginx:1.27 > app.yaml || fail "gen failed" +"$BIN" verify app.yaml >/dev/null || fail "verify rejected its own generated manifest" +pass "gen + verify with no cluster and no network" + +# --- nothing was written outside the working directory ------------------- +# The claim is "nothing installed". A config or state directory appearing in +# $HOME is the usual way that stops being true. +for d in "${HOME:-/nonexistent}/.k3helper" "${HOME:-/nonexistent}/.config/k3helper"; do + [ -e "$d" ] && fail "wrote outside the working directory: $d" +done +pass "nothing written outside the working directory" + +echo "portable check passed" diff --git a/test/e2e.sh b/test/e2e.sh index 6f3f04e..ef5b2ed 100755 --- a/test/e2e.sh +++ b/test/e2e.sh @@ -80,7 +80,8 @@ OUT=$($K init --server "$SERVER_IP" --agent "$AGENT1_IP" --user sandbox \ -o "$WORK/targets.yaml" 2>&1) assert_contains "init writes a targets file" "$OUT" "wrote $WORK/targets.yaml" OUT=$($K ctx -t "$WORK/targets.yaml" 2>&1) -assert_matches "the generated file loads back" "$OUT" "my-cluster +2" +# ctx prints CLUSTER, REACHED, NODES: an SSH cluster with two nodes. +assert_matches "the generated file loads back" "$OUT" "my-cluster +ssh +2" OUT=$($K init -o "$WORK/targets.yaml" 2>&1); RC=$? assert_exit "init refuses to clobber an existing file" 1 $RC OUT=$($K init --local -o "$WORK/local.yaml" 2>&1) diff --git a/test/faults/check-all.sh b/test/faults/check-all.sh index 61b11e6..1ead2d9 100755 --- a/test/faults/check-all.sh +++ b/test/faults/check-all.sh @@ -40,10 +40,16 @@ settle_for() { # interval, so DiskPressure appears well after the disk is actually full disk-full) echo 60 ;; k3s-down|oom) echo 45 ;; + # A crashlooping pod is not reported as CrashLoopBackOff the moment it + # fails. It has to pull its image, run, exit, and be restarted enough times + # for kubelet to give up and back off — and until then the pod is Running + # and unready, which doctor correctly reports as pod.not-ready instead. + # 30s caught it on a fast machine and missed it on a loaded CI runner. The + # oom fault is the same shape and already has 45. + crashloop) echo 45 ;; # the scheduler retries FailedScheduling with backoff, so the event can # take longer to appear than the pod takes to go Pending cordon|pending) echo 40 ;; - crashloop) echo 30 ;; imagepull) echo 25 ;; registry) echo 25 ;; coredns) echo 10 ;;