tracebloc · saqlainsyed007 · Sep 9, 2026 · Sep 9, 2026 · Sep 9, 2026 · Sep 9, 2026
diff --git a/client/Chart.yaml b/client/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.110
-appVersion: "1.9.110"
+version: 1.9.111
+appVersion: "1.9.111"
 keywords:
   - tracebloc
   - kubernetes

@@ -480,6 +480,20 @@ true
   freezing — a frozen control plane with no signal is worse than a restart that
   needs the network.
 
+  #569 OFFLINE GUARANTEE — NARROWED as of 1.9.111 (#1013). IfNotPresent only
+  helps when the REFERENCE is cache-satisfiable. Until 1.9.111 a `helm upgrade`
+  re-rendered the bare `:tag`, which a node that had pulled that tag could always
+  satisfy from cache, so an offline restart ran the (possibly stale) cached
+  image. From 1.9.111 tracebloc.controlPlaneDigest can render `repo@<digest>`
+  seeded from the last-refreshed annotation (see its header), and because the
+  first-observation path RECORDS a digest without re-imaging, the rendered digest
+  may be one the node has never pulled. Online that is the fix working; OFFLINE
+  it turns "restarts on a stale image" into "cannot start" (ImagePullBackOff) for
+  that one digest. Two corollaries: a `global.imageRegistry` mirror is exempt
+  (branch 2 stays inert there → `:tag`), and a side-loaded k3d image is affected
+  because `k3d image import` stores a tag alias with no resolvable digest and a
+  local k3d install IS docker.io, so the mirror gate does not help it.
+
   Usage: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" $d "root" $) }}
 */}}
 {{- define "tracebloc.controlPlanePullPolicy" -}}
@@ -493,6 +507,150 @@ Always
 {{- end -}}
 {{- end }}
 
+{{/*
+  tracebloc.controlPlaneDigest — the EFFECTIVE digest for a control-plane image,
+  resolved in priority order:
+
+    1. an operator's explicit values pin (`images.<name>.digest`) — wins,
+       behaviour unchanged;
+    2. else, when image-refresh is the update path (enabled AND the docker.io
+       mirror), the digest image-refresh last OBSERVED, read from the
+       jobs-manager Deployment's `tracebloc.io/last-refreshed-<annotationImage>-digest`
+       annotation via `lookup`. ("Observed", not "applied": on the FIRST tick
+       image-refresh records the current digest WITHOUT `set image`, deliberately
+       — see the image-refresh CronJob header — so the annotation is what refresh
+       last saw, which is only the same as what it last applied once a real
+       refresh has run.) This is what makes a `helm upgrade` RENDER `repo@digest`
+       instead of reverting to the bare `:tag` and dropping image-refresh's
+       out-of-band `kubectl set image` pin — the revert that let a stale-`:tag`
+       node cache silently run an OLD control-plane image (client-runtime#199).
+       `lookup` returns empty during `helm template` / `helm diff` / the FIRST
+       install (no Deployment yet), so it degrades to `""` → `:tag` there, which
+       is correct: nothing is pinned yet and the node's fresh tag pull is the
+       right image.
+    3. else `""` (bare `:tag`; `Always` via controlPlanePullPolicy is then the
+       update path — the non-refresh / mirror edges #569 protects).
+
+  On `imageRefresh.suspend: true` the pin is KEPT, not dropped. suspend stops the
+  CronJob from POLLING; it does not un-pin (@LukasWodka on #1013). The knob is
+  used to FREEZE an edge during an incident, and `values.schema.json` promises it
+  "pause[s] without removing the resources" — un-pinning here would re-render
+  jobs-manager (Recreate), requests-proxy and the resource-monitor DaemonSet onto
+  the floating `:tag` + `Always`, three unplanned rollouts plus a downgrade to
+  whatever `:tag` points at now, which is the opposite of freezing. Keeping the
+  last-observed `@digest` freezes the edge on a real, previously-resolved image;
+  a newly joined node then pulls that frozen digest (a valid ref), not a missing
+  one, which still answers the newly-joined-node concern that first put a suspend
+  gate here (@shujaatTracebloc's earlier BLOCKING-2 on #1013, reversed here).
+
+  ENV-SCOPING CAVEAT (@shujaatTracebloc #1013, Medium; fix lives in A, not here).
+  The `last-refreshed-<image>-digest` annotation key is NOT tag/env-scoped, and
+  `tracebloc.image` drops the tag once a digest is present, so a `CLIENT_ENV` /
+  tag change made AT upgrade time (e.g. `--set env.CLIENT_ENV=prod`, or the fleet
+  moving an edge between environments) is silently inert on the image: the render
+  pins the OLD env's `@digest` while every label reads the new env. It self-heals
+  on image-refresh's next tick (it resolves the new `:tag`, sees `recorded !=
+  latest`, and `set image`s) — a window of up to `imageRefresh.schedule`, and
+  NEVER while `imageRefresh.suspend: true`, since suspend now keeps the pin. The
+  containment is an A-side change: image-refresh records the resolved tag
+  alongside the digest (`tracebloc.io/last-refreshed-<image>-tag`) so this helper
+  can honour the annotation only when that tag equals the currently resolved
+  `tracebloc.clientEnv`, falling back to `:tag` otherwise. Called out here because
+  suspend makes the mismatch permanent.
+
+  The lookup targets ONLY the jobs-manager Deployment (where image-refresh writes
+  every last-refreshed annotation) in the release namespace — a read the
+  auto-upgrade SA already holds (its release-ns Role grants all verbs on all
+  resources in that namespace), so this adds no RBAC and does not hit the
+  backend#2469 bootstrap lockout.
+
+  Args: (dict "root" $ "operatorDigest" <values digest for this image>
+              "annotationImage" <"jobs-manager"|"pods-monitor"|"resource-monitor">)
+*/}}
+{{- define "tracebloc.controlPlaneDigest" -}}
+{{- $mirror := (dig "imageRegistry" "docker.io" (.root.Values.global | default dict)) | default "docker.io" -}}
+{{- if .operatorDigest -}}
+{{- .operatorDigest -}}
+{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") -}}
+{{- $dep := lookup "apps/v1" "Deployment" .root.Release.Namespace (printf "%s-jobs-manager" (include "tracebloc.fullname" .root)) -}}
+{{- if $dep -}}
+{{- $ann := index (($dep.metadata).annotations | default dict) (printf "tracebloc.io/last-refreshed-%s-digest" .annotationImage) | default "" -}}
+{{/*
+  VALIDATE before rendering: the 11 values digest keys are schema-guarded by
+  `^(sha256:[a-f0-9]{64})?$`, but this annotation is written out-of-band by
+  image-refresh (kubectl) and reaches an `image:` field unchecked. A malformed
+  value (`tracebloc.image` drops the tag when a digest is present) renders an
+  unstartable ref helm cannot detect — the kubelet reports InvalidImageName
+  while the apiserver accepts the spec. Require a full sha256 digest; anything
+  else degrades to `""` → `:tag`, the safe fallback (@shujaatTracebloc on #1013).
+  `suspend` is deliberately NOT gated here: a suspended edge keeps rendering the
+  last-observed digest so `helm upgrade` freezes it in place rather than rolling
+  it onto the floating tag (see the header note).
+*/}}
+{{- if regexMatch "^sha256:[a-f0-9]{64}$" $ann -}}
+{{- $ann -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end }}
+
+{{/*
+  tracebloc.controlPlaneDigestSource — the PROVENANCE of what controlPlaneDigest
+  resolved, rendered onto the jobs-manager Deployment as
+  `tracebloc.io/controlplane-digest-source` so a monitored edge can SEE which
+  branch produced the image ref. It mirrors controlPlaneDigest's resolution
+  exactly and must stay in lockstep with it. Values:
+
+    values             — an operator `images.<name>.digest` pin (priority 1).
+    annotation         — image-refresh's last-refreshed `@digest` (the pin is
+                         preserved on this upgrade — the healthy steady state).
+    tag                — the bare `:tag`. Legitimate here: first install, a
+                         `global.imageRegistry` mirror, `imageRefresh.enabled:
+                         false`, or the first-observation window (Deployment
+                         present but not yet annotated).
+    tag-lookup-failed  — the ANOMALY (@shujaatTracebloc #1013, Medium): `lookup`
+                         fails OPEN — Helm returns an empty map on EVERY failure
+                         (RBAC denial, apiserver 5xx, client timeout, a
+                         kubectl-less renderer), not just NotFound — so a read
+                         failure is otherwise indistinguishable from "first
+                         install" and would silently drop the pin back to `:tag`
+                         (client-runtime#199, now non-deterministic and unsignalled).
+                         On an UPGRADE the jobs-manager Deployment MUST exist, so
+                         `.Release.IsUpgrade` AND an empty lookup means the READ
+                         failed, not that the release is new. We cannot recover
+                         the digest from a read we could not do, and a hard `fail`
+                         would break `helm diff upgrade` (lookup is empty there
+                         too), so we SURFACE it instead: a monitored edge alerts
+                         on this value. (It also shows in `helm diff upgrade` for
+                         the same empty-lookup reason the digest churn does — the
+                         diff caveat the header already documents; it is accurate
+                         on a real apply.)
+
+  Args: same as controlPlaneDigest.
+*/}}
+{{- define "tracebloc.controlPlaneDigestSource" -}}
+{{- $mirror := (dig "imageRegistry" "docker.io" (.root.Values.global | default dict)) | default "docker.io" -}}
+{{- if .operatorDigest -}}
+values
+{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") -}}
+{{- $dep := lookup "apps/v1" "Deployment" .root.Release.Namespace (printf "%s-jobs-manager" (include "tracebloc.fullname" .root)) -}}
+{{- if $dep -}}
+{{- $ann := index (($dep.metadata).annotations | default dict) (printf "tracebloc.io/last-refreshed-%s-digest" .annotationImage) | default "" -}}
+{{- if regexMatch "^sha256:[a-f0-9]{64}$" $ann -}}
+annotation
+{{- else -}}
+tag
+{{- end -}}
+{{- else if .root.Release.IsUpgrade -}}
+tag-lookup-failed
+{{- else -}}
+tag
+{{- end -}}
+{{- else -}}
+tag
+{{- end -}}
+{{- end }}
+
 {{/*
   StorageClass name: when storageClass.create is true, use a release-unique name
   so each release gets its own StorageClass (avoids Helm ownership conflicts).

@@ -54,17 +54,23 @@ data:
     #   in the same tick); it does not detect and repair skew that predates
     #   it, because it compares registry-vs-annotation, never pod-vs-pod.
     #
-    # HELM RE-RENDER, by contrast, is HANDLED rather than tolerated (client-runtime#199).
-    # `helm upgrade --reset-then-reuse-values` (the fleet auto-upgrade path)
-    # re-renders the templates, which write `repo:tag` and so revert an earlier
-    # `set image` pin. `recorded == latest` no longer means no-op: the loop reads
-    # each workload's LIVE image and re-pins the digest whenever the workload is
-    # off it. The revert therefore lasts ONE tick — the next tick puts the digest
-    # back — instead of floating on the tag until the next upstream release. It
-    # is compared on the @sha256 digest, so a registry-prefix rewrite (a mutating
-    # webhook) is not mistaken for a revert. This only happens on a chart VERSION
-    # bump anyway — auto-upgrade compares versions and skips otherwise — so it is
-    # not an hourly revert.
+    # HELM RE-RENDER, by contrast, is HANDLED rather than tolerated (client-runtime#199),
+    # from BOTH sides now. Chart-side (#1013, from 1.9.111): the control-plane
+    # image sites render through `tracebloc.controlPlaneDigest`, which reads THIS
+    # annotation via `lookup`, so `helm upgrade --reset-then-reuse-values` (the
+    # fleet auto-upgrade path) re-renders `repo@<last-refreshed digest>` and no
+    # longer writes the bare `repo:tag` that used to revert an earlier `set image`
+    # pin. Script-side (#1008): `recorded == latest` no longer means no-op — the
+    # loop reads each workload's LIVE image and re-pins the digest whenever the
+    # workload is off it, so any residual revert (an annotation that lags the live
+    # spec — a timed-out rollout, `helm rollback`) lasts ONE tick instead of
+    # floating on the tag until the next upstream release. Compared on the @sha256
+    # digest, so a registry-prefix rewrite (a mutating webhook) is not mistaken
+    # for a revert. This only bites on a chart VERSION bump anyway — auto-upgrade
+    # compares versions and skips otherwise — so it is not an hourly revert. (One
+    # residue the script-side re-pin does not yet cover is the durable flap-lockout
+    # state, where the annotation stays stuck while the live spec has moved on —
+    # tracked as a follow-up; see the first-tick contract below.)
     #
     # Reconciling against the live spec is what makes that possible — the
     # declarative reconcile `set image` enables (`rollout restart` was a blind
@@ -118,6 +124,19 @@ data:
     # price, and a follow-up may gate the re-pin on "have we ever applied a digest
     # here?" so a genuine fresh install skips the flap path entirely.
     #
+    # First-observation roll (chart ≥ 1.9.111, #1013). Once this tick has
+    # RECORDED D0 without re-imaging, the live spec still says `:tag` but the
+    # annotation now says `@D0`, so the NEXT chart bump renders `repo@D0`
+    # (tracebloc.controlPlaneDigest reads the annotation) and rolls jobs-manager
+    # (Recreate), requests-proxy and the resource-monitor DaemonSet ONCE, for
+    # byte-identical content. auto-upgrade runs that upgrade `--atomic --wait`
+    # (auto-upgrade-cronjob.yaml), so on a slow single-node edge the one-time
+    # roll can hit the timeout and roll back, then re-attempt identically on the
+    # next bump. This is the deliberate tradeoff for NOT `set image`-ing on the
+    # first tick; pinning on the first tick instead would move the same roll to
+    # install time for every edge. Also narrows #569's offline guarantee — see
+    # the note at controlPlanePullPolicy in _helpers.tpl.
+    #
     # Parsing: awk/sed/grep + jq. jq used only where JSON-with-dotted-keys
     # or container/env-array filtering motivates it; the rest stays in pure
     # shell so the script survives image swaps to leaner bases — same

@@ -10,6 +10,16 @@ metadata:
   namespace: {{ .Release.Namespace }}
   labels:
     {{- include "tracebloc.labels" . | nindent 4 }}
+  annotations:
+    # PROVENANCE of the control-plane image ref this render resolved
+    # (client-runtime#199, @shujaatTracebloc on #1013). `lookup` fails OPEN, so a
+    # transient read failure would silently drop image-refresh's pin back to
+    # `:tag` with a green upgrade and no signal. This records which branch won —
+    # `values` | `annotation` | `tag` | `tag-lookup-failed` — so a monitored edge
+    # can alert on the fallback instead of discovering the #199 revert after the
+    # fact. Metadata-only: changing it does not roll the pods. See
+    # tracebloc.controlPlaneDigestSource.
+    tracebloc.io/controlplane-digest-source: {{ include "tracebloc.controlPlaneDigestSource" (dict "root" $ "operatorDigest" .Values.images.jobsManager.digest "annotationImage" "jobs-manager") | quote }}
 spec:
   selector:
     matchLabels:
@@ -100,6 +110,19 @@ spec:
         #
         # It also removes an image: busybox was a second pull on every edge, for a
         # TCP connect the app image can already do.
+        #
+        # NOT seeded from the last-refreshed annotation (client-runtime#199,
+        # @shujaatTracebloc on #1013): this is an INIT container, so a digest a
+        # node has never pulled fails as `Init:ImagePullBackOff` and the pod never
+        # reaches any container at all — on exactly the offline/side-loaded k3d /
+        # Docker-Desktop edge #569 was written to protect (a `k3d image import`
+        # stores a tag alias with no resolvable digest; see the #569 note at
+        # controlPlanePullPolicy). And `image-refresh` never reconciles this site:
+        # `kubectl set image` only names `api` and `pods-monitor-container`, so the
+        # chart render would be its ONLY driver, with no `set image` to move it
+        # forward. The pin buys nothing here anyway — it is a MySQL TCP wait on the
+        # same image. So keep the OPERATOR pin (`images.jobsManager.digest`) for an
+        # explicit, reproducible pin, but do NOT seed the floating annotation.
         image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
         securityContext:
           # NO runAsUser, DELIBERATELY (Bugbot High, #942). OpenShift's
@@ -265,7 +288,7 @@ spec:
       {{- end }}
       containers:
       - name: api
-        image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
+        image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.jobsManager.digest "annotationImage" "jobs-manager")) "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
         # #569: IfNotPresent wherever an update path exists without `Always` --
         # a pinned digest, or the image-refresh reconcile. `Always` forced a
         # registry round-trip on every (re)start, so an offline Docker Desktop /
@@ -274,7 +297,7 @@ spec:
         # is NOT unconditional: on an edge the reconcile cannot reach (private
         # mirror, or imageRefresh disabled) a floating tag plus a restart is the
         # only update path there is, so it stays Always.
-        imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" .Values.images.jobsManager.digest "root" $) }}
+        imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.jobsManager.digest "annotationImage" "jobs-manager")) "root" $) }}
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
@@ -906,10 +929,10 @@ spec:
         {{- end }}
         {{- end }}
       - name: pods-monitor-container
-        image: {{ include "tracebloc.image" (dict "repository" "tracebloc/pods-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.podsMonitor.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
+        image: {{ include "tracebloc.image" (dict "repository" "tracebloc/pods-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.podsMonitor.digest "annotationImage" "pods-monitor")) "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
         # #569: same policy as the api container above -- see
         # tracebloc.controlPlanePullPolicy.
-        imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" .Values.images.podsMonitor.digest "root" $) }}
+        imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.podsMonitor.digest "annotationImage" "pods-monitor")) "root" $) }}
         securityContext:
           allowPrivilegeEscalation: false
           capabilities: