fix(doctor): don't treat an assigned-but-Pending training pod as a running job (backend#3247)

internal/cli/doctor.go

@@ -519,6 +519,31 @@ func summarizeDoctor(results []doctor.Result, tok tokenState) (connected, ready
 		ready = healthLine{doctor.StatusWarn,
 			"Ready to run training — a training is already running, and the next one is waiting for it to finish.",
 			fmt.Sprintf("A pod is waiting to start because a running job holds this machine's free compute. Let the job finish, or stop it if it is not needed; asking for less per run or resizing will not help — the room comes back when the job ends. If the pod is still waiting after that, something else is holding it: `%s doctor --verbose`.", launcher())}
+	case by["Node capacity"].Status == doctor.StatusFail &&
+		strings.HasPrefix(by["Node capacity"].Detail, doctor.StuckJobPod):
+		// A training pod is scheduled to a node but stuck Pending -- an image pull
+		// backing off, or a container stuck creating -- NOT running (backend#3247).
+		// checkNodeFit used to count it as a running job holding the room, so with
+		// a pod also stuck Pending this rolled up through `stuckPending && heldByJob`
+		// to the transient "a training is already running, wait for it" Warn at
+		// exit 0 -- on a pod that is wedged and never will.
+		//
+		// IT SITS ABOVE THE STUCK-PENDING ARM by the same measured-beats-inferred
+		// rule that puts `stuckPending && heldByJob` there. checkNodeFit escalates
+		// to this Fail ONLY for a genuinely-wedged reason -- an image pull backing
+		// off or a create error -- so the cause is measured, not inferred: "waiting
+		// will not clear it" is exact. A pod merely still pulling/creating past the
+		// grace window is left to the stuck-Pending arm below, whose "usually not
+		// enough free compute, or an image that can't be pulled" wording is an
+		// honest age-based inference (a large first pull on a cold node CAN exceed
+		// the grace). Only the measured capacity Fails (OverCommitted) and a hard
+		// Pod-health crash-loop Fail outrank it. Its remedy is the OPPOSITE of the
+		// transient Warn's (inspect the pod, do NOT wait); the pod it names is one
+		// `--verbose` away -- and PLAIN TERMS, no Kubernetes vocabulary, like its
+		// neighbours (the granular checkNodeFit remedy carries the `kubectl` form).
+		ready = healthLine{doctor.StatusFail,
+			"Not ready — a training pod is stuck starting and isn't running yet.",
+			fmt.Sprintf("A scheduled training pod is stuck (usually a training image that can't be pulled). Waiting will not clear it — `%s doctor --verbose` names the pod.", launcher())}
 	case stuckPending:
 		// Pods stuck Pending past the grace window (unschedulable / image can't
 		// pull) mean training can't actually schedule — so this is NOT ready, even

internal/cli/doctor_test.go

@@ -583,6 +583,62 @@ func TestSummarizeDoctor(t *testing.T) {
 		}
 	})
 
+	// backend#3247: the defect combination. A training pod is scheduled but
+	// wedged on an image pull (Node capacity Fail, StuckJobPod) and Pod health
+	// also sees it Pending past grace (stuckPending). This used to roll up through
+	// `stuckPending && heldByJob` to "a training is already running, wait for it
+	// to finish" at exit 0 -- because checkNodeFit mislabelled the wedged pod a
+	// running job. Now the Node-capacity Fail is a MEASURED cause and outranks the
+	// inferred stuck-Pending arm: a Fail that says the pod is stuck, whose remedy
+	// is to inspect the pod, not to wait or resize.
+	t.Run("a scheduled-but-stuck training pod is a Fail, not the wait-for-a-running-job Warn", func(t *testing.T) {
+		results := withDetail(allOK, "Node capacity", doctor.StatusFail,
+			doctor.StuckJobPod+": tracebloc/train-stuck on n1 (ImagePullBackOff). The next run does not wait on a pod that is not running")
+		results = withDetail(results, "Pod health", doctor.StatusWarn,
+			"Pending > 5m0s: [train-stuck]")
+		c, r := summarizeDoctor(results, tokenOK)
+		if r.status != doctor.StatusFail {
+			t.Fatalf("a wedged training pod must fail the rollup, got %v (%q)", r.status, r.text)
+		}
+		if strings.Contains(r.text, "already running") || strings.Contains(r.text, "waiting for it") {
+			t.Errorf("must not tell the operator to wait on a job that is not running: %q", r.text)
+		}
+		if !strings.Contains(r.text, "stuck starting") {
+			t.Errorf("the top line should say the pod is stuck starting, got %q", r.text)
+		}
+		if strings.Contains(r.remedy, "resources set max") || strings.Contains(r.remedy, "Ask for less") {
+			t.Errorf("resizing does not clear an image-pull stall; the remedy must not send them there: %q", r.remedy)
+		}
+		// PLAIN TERMS: this rolled-up line must carry no Kubernetes vocabulary --
+		// the `kubectl` form lives in the granular checkNodeFit remedy, one
+		// `--verbose` away (the invariant summarizeDoctor documents three times).
+		if strings.Contains(r.remedy, "kubectl") {
+			t.Errorf("the rolled-up remedy must stay plain-terms, no `kubectl`: %q", r.remedy)
+		}
+		if !strings.Contains(r.remedy, "--verbose") {
+			t.Errorf("the remedy should point at --verbose to name the pod, got %q", r.remedy)
+		}
+		if v := doctorVerdict(c.status, r.status); v != doctor.StatusFail {
+			t.Errorf("verdict must be a Fail (exit 2), not a clean pass, got %v", v)
+		}
+	})
+
+	// The same measured Fail with Pod health NOT flagging it (checkPods could be
+	// scoped to a namespace that missed it, or unable to list). The dedicated arm
+	// must still fire on the Node-capacity signal alone -- never falling through
+	// to a green.
+	t.Run("a scheduled-but-stuck training pod fails even when Pod health is silent", func(t *testing.T) {
+		results := withDetail(allOK, "Node capacity", doctor.StatusFail,
+			doctor.StuckJobPod+": tracebloc/train-stuck on n1 (ErrImagePull). The next run does not wait on a pod that is not running")
+		c, r := summarizeDoctor(results, tokenOK)
+		if r.status != doctor.StatusFail || !strings.Contains(r.text, "stuck starting") {
+			t.Fatalf("want the stuck-pod Fail on the Node-capacity signal alone, got %v (%q)", r.status, r.text)
+		}
+		if v := doctorVerdict(c.status, r.status); v != doctor.StatusFail {
+			t.Errorf("verdict must be a Fail (exit 2), got %v", v)
+		}
+	})
+
 	t.Run("a crash-looping pod still outranks the running-job explanation", func(t *testing.T) {
 		// The exception is scoped to the stuck-Pending WARN; a Pod-health FAIL is
 		// a measured failure with a different fix, and must keep winning.

internal/cli/testdata/golden/zz-all-strings.golden

@@ -81,6 +81,8 @@ screen. %s/%d are runtime placeholders.
 "%s, so free compute could not be verified — checked against allocatable only; an over-committed control plane would be invisible here. Also, no single Ready node satisfies cpu+memory AND %s, so GPU jobs would rely on the CPU fallback (needs %s)"
 "%s, … and %d more"
 "%s/%s"
+"%s/%s on %s (%s)"
+"%s: %s. The next run does not wait on a pod that is not running"
 "%s: %v"
 "%s: %w"
 "%s: a Ready node fits a training job (%s) beside the platform's own pods, but running job(s) on %s hold %s right now, so the next run waits Pending until they finish"
@@ -127,6 +129,7 @@ screen. %s/%d are runtime placeholders.
 "A newer tracebloc is available: %s (you have %s). Update: tracebloc upgrade"
 "A pod is waiting to start because a running job holds this machine's free compute. Let the job finish, or stop it if it is not needed; asking for less per run or resizing will not help — the room comes back when the job ends. If the pod is still waiting after that, something else is holding it: `%s doctor --verbose`."
 "A real run continues with step 2 (copy into your secure environment) and step 3 (validate and load)."
+"A scheduled training pod is stuck (usually a training image that can't be pulled). Waiting will not clear it — `%s doctor --verbose` names the pod."
 "A tracebloc client is already running on this cluster — adopting it. Couldn't read the cluster identity, so its idempotency anchor was left unchanged; point --kubeconfig/--context at a cluster where kube-system is readable to stamp it."
 "A training run is allocated up to:"
 "Add --help to any command for the flags."
@@ -251,6 +254,7 @@ screen. %s/%d are runtime placeholders.
 "Ingestion summary"
 "Ingestor SA token"
 "Ingests a local dataset into your secure environment's storage,\nsubmits the ingestion run, and follows it to completion (streaming\nprogress + the final summary). Your data never leaves your own\ninfrastructure. Supports %[1]d tasks across the image, text, and\ntabular / time-series families; pick one with --task.\n\n<dataset> is the data itself. What it looks like depends on the task:\n\n  tabular / time-series — the dataset is a single CSV. Pass the .csv\n  file directly, or a folder holding exactly one .csv:\n\n      churn.csv                (the .csv file itself)\n    or\n      churn/\n        data.csv               (the one .csv in the folder)\n\n  image classification / keypoint detection — a folder with\n  labels.csv + an images/ subfolder:\n\n      cats_dogs/\n        labels.csv             (required)\n        images/                (required)\n          001.jpg\n          ...\n\n  object detection — a folder with images/ + annotations/ and NO\n  labels.csv: records are enumerated from the Pascal-VOC XML, one per\n  image, so there is no manifest and no label column to declare.\n\n      visdrone/\n        images/                (required)\n          001.jpg\n        annotations/           (required — 001.xml pairs with 001.jpg)\n          001.xml\n\n  text (classification, masked language modeling) — a folder with\n  labels.csv + a %[2]s/ subfolder (masked language modeling uses %[3]s/):\n\n      reviews/\n        labels.csv             (required)\n        %[2]s/                 (required — %[3]s/ for masked language modeling)\n          001.txt\n          ...\n\nA bare .csv file is accepted only for the tabular / time-series family;\nimage and text datasets must be a folder.\n\nAccepted image extensions: .jpg, .jpeg, or .png (case-insensitive).\nAll images in one dataset must share a single type — the cluster\nvalidates the type it was told to expect.\n\nv0.1 caps the dataset at 1 GiB total + 500 MiB per file. Larger\ndatasets need the v0.2 cloud-source story (S3/GCS/HTTPS sources) —\nsee tracebloc/client#147 non-goals.\n\nExit codes:\n  0   files staged + ingested successfully (or --detach: just staged + submitted)\n  2   schema validation failed (synthesized spec rejected) or\n      v0.1-unsupported task passed\n  3   local-layout or kubeconfig error\n  4   cluster reachable but no tracebloc client / shared storage missing\n  5   ingestor SA token couldn't be obtained, or jobs-manager\n      rejected the token (401/403)\n  6   destination table already exists (re-run with --overwrite to\n      replace it, or pick a different --name)\n  7   pre-flight succeeded but staging the files failed\n      (Pod creation, image pull, exec stream, or remote tar error) —\n      or, with --overwrite, removing the old table failed\n  8   jobs-manager rejected the submit (4xx/5xx other than auth)\n  9   ingestion Job exited non-zero, or completed with row-level\n      failures the summary panel reports"
+"Inspect the stuck pod: kubectl describe pod -n %s %s — usually a training image that can't be pulled, or a container that can't be created. This is not a capacity shortage; lowering RESOURCE_REQUESTS or resizing will not clear it."
 "Interrupted before the change could be confirmed."
 "It may already have applied — re-run `%s resources set` to check the current per-run ceiling."
 "It reports more memory than the machine really has, so two trainings that each look like they fit can together run it out of memory and take the environment down. Run one training at a time; to fix it for good, recreate the environment as a single-node one. `%s doctor --verbose` shows the numbers and the exact flags."
@@ -281,6 +285,7 @@ screen. %s/%d are runtime placeholders.
 "Not connected — couldn't read your secure environment."
 "Not connected — tracebloc didn't confirm your session (server error)."
 "Not connected — your secure environment isn't answering."
+"Not ready — a training pod is stuck starting and isn't running yet."
 "Not ready — dataset storage isn't available."
 "Not ready — not enough free compute to start a training."
 "Not ready — part of your secure environment can't start yet."