From d92bdab2998849f0b897b526ee10d50f3a786228 Mon Sep 17 00:00:00 2001 From: Syed Saqlain Date: Wed, 9 Sep 2026 10:33:41 +0400 Subject: [PATCH 1/8] fix(chart): render control-plane images from the last-refreshed digest so a helm upgrade preserves the pin (client-runtime#199) Control-plane images render as repo:tag (IfNotPresent) when unpinned, and image-refresh keeps them current out-of-band via kubectl set image repo@digest. A helm upgrade --reset-then-reuse-values re-renders the bare tag and drops that pin; on a node whose :tag cache is stale the pod then silently runs an OLD image. New tracebloc.controlPlaneDigest helper seeds the effective digest from the jobs-manager Deployment last-refreshed--digest annotation via lookup, so a re-render renders repo@digest instead. Operator pins still win; lookup is empty during helm template/diff/first install (-> :tag, correct there). No new RBAC (auto-upgrade SA already reads the Deployment in the release ns). Co-Authored-By: Claude Opus 4.8 --- client/Chart.yaml | 4 +- client/templates/_helpers.tpl | 41 +++++++++++++++++++ client/templates/jobs-manager-deployment.yaml | 10 ++--- .../templates/requests-proxy-deployment.yaml | 5 +++ .../templates/resource-monitor-daemonset.yaml | 6 +++ 5 files changed, 59 insertions(+), 7 deletions(-) diff --git a/client/Chart.yaml b/client/Chart.yaml index 99f71ce9..de42aa09 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.108 -appVersion: "1.9.108" +version: 1.9.110 +appVersion: "1.9.110" keywords: - tracebloc - kubernetes diff --git a/client/templates/_helpers.tpl b/client/templates/_helpers.tpl index ed7d5292..c3a1e469 100644 --- a/client/templates/_helpers.tpl +++ b/client/templates/_helpers.tpl @@ -493,6 +493,47 @@ Always {{- end -}} {{- end }} +{{/* + tracebloc.controlPlaneDigest — the EFFECTIVE digest for a control-plane image, + resolved in priority order: + + 1. an operator's explicit values pin (`images..digest`) — wins, + behaviour unchanged; + 2. else, when image-refresh is the update path (enabled AND the docker.io + mirror), the digest image-refresh last APPLIED, read from the + jobs-manager Deployment's `tracebloc.io/last-refreshed--digest` + annotation via `lookup`. This is what makes a `helm upgrade` RENDER + `repo@digest` instead of reverting to the bare `:tag` and dropping + image-refresh's out-of-band `kubectl set image` pin — the revert that let + a stale-`:tag` node cache silently run an OLD control-plane image + (client-runtime#199). `lookup` returns empty during `helm template` / + `helm diff` / the FIRST install (no Deployment yet), so it degrades to + `""` → `:tag` there, which is correct: nothing is pinned yet and the + node's fresh tag pull is the right image. + 3. else `""` (bare `:tag`; `Always` via controlPlanePullPolicy is then the + update path — the non-refresh / mirror edges #569 protects). + + The lookup targets ONLY the jobs-manager Deployment (where image-refresh writes + every last-refreshed annotation) in the release namespace — a read the + auto-upgrade SA already holds (its release-ns Role grants all verbs on all + resources in that namespace), so this adds no RBAC and does not hit the + backend#2469 bootstrap lockout. + + Args: (dict "root" $ "operatorDigest" + "annotationImage" <"jobs-manager"|"pods-monitor"|"resource-monitor">) +*/}} +{{- define "tracebloc.controlPlaneDigest" -}} +{{- $mirror := (dig "imageRegistry" "docker.io" (.root.Values.global | default dict)) | default "docker.io" -}} +{{- if .operatorDigest -}} +{{- .operatorDigest -}} +{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") -}} +{{- $dep := lookup "apps/v1" "Deployment" .root.Release.Namespace (printf "%s-jobs-manager" (include "tracebloc.fullname" .root)) -}} +{{- if $dep -}} +{{- index (($dep.metadata).annotations | default dict) (printf "tracebloc.io/last-refreshed-%s-digest" .annotationImage) | default "" -}} +{{- end -}} +{{- end -}} +{{- end }} + {{/* StorageClass name: when storageClass.create is true, use a release-unique name so each release gets its own StorageClass (avoids Helm ownership conflicts). diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index 81007109..d7a2b5d2 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -100,7 +100,7 @@ spec: # # It also removes an image: busybox was a second pull on every edge, for a # TCP connect the app image can already do. - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.jobsManager.digest "annotationImage" "jobs-manager")) "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} securityContext: # NO runAsUser, DELIBERATELY (Bugbot High, #942). OpenShift's # `restricted` SCC assigns a UID from the project's range, and the pod @@ -265,7 +265,7 @@ spec: {{- end }} containers: - name: api - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.jobsManager.digest "annotationImage" "jobs-manager")) "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} # #569: IfNotPresent wherever an update path exists without `Always` -- # a pinned digest, or the image-refresh reconcile. `Always` forced a # registry round-trip on every (re)start, so an offline Docker Desktop / @@ -274,7 +274,7 @@ spec: # is NOT unconditional: on an edge the reconcile cannot reach (private # mirror, or imageRefresh disabled) a floating tag plus a restart is the # only update path there is, so it stays Always. - imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" .Values.images.jobsManager.digest "root" $) }} + imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.jobsManager.digest "annotationImage" "jobs-manager")) "root" $) }} securityContext: allowPrivilegeEscalation: false capabilities: @@ -906,10 +906,10 @@ spec: {{- end }} {{- end }} - name: pods-monitor-container - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/pods-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.podsMonitor.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/pods-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.podsMonitor.digest "annotationImage" "pods-monitor")) "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} # #569: same policy as the api container above -- see # tracebloc.controlPlanePullPolicy. - imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" .Values.images.podsMonitor.digest "root" $) }} + imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.podsMonitor.digest "annotationImage" "pods-monitor")) "root" $) }} securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/client/templates/requests-proxy-deployment.yaml b/client/templates/requests-proxy-deployment.yaml index 0ac6505d..f9c17c0a 100644 --- a/client/templates/requests-proxy-deployment.yaml +++ b/client/templates/requests-proxy-deployment.yaml @@ -41,6 +41,11 @@ spec: these keys. */}} {{- $rpDigest := (dig "requestsProxy" "digest" "" $rpImages) | default (dig "jobsManager" "digest" "" $rpImages) }} + {{- /* client-runtime#199: seed the digest image-refresh last applied so a + `helm upgrade` renders repo@digest, not the bare :tag (see + tracebloc.controlPlaneDigest). requests-proxy runs the SAME + jobs-manager image and follows its annotation. */}} + {{- $rpDigest = include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" $rpDigest "annotationImage" "jobs-manager") }} image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" $rpDigest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} # #569: IfNotPresent wherever an update path exists without `Always` -- # a pinned digest, or the image-refresh reconcile (see diff --git a/client/templates/resource-monitor-daemonset.yaml b/client/templates/resource-monitor-daemonset.yaml index fca159ff..eb1b544a 100644 --- a/client/templates/resource-monitor-daemonset.yaml +++ b/client/templates/resource-monitor-daemonset.yaml @@ -136,6 +136,12 @@ spec: entry). `dig` is not usable here — it rejects chartutil.Values. */}} {{- $rmDigest := (default (dict) (default (dict) .Values.images).resourceMonitor).digest | default "" }} + {{- /* client-runtime#199: seed the digest image-refresh last applied so a + `helm upgrade` renders repo@digest, not the bare :tag (see + tracebloc.controlPlaneDigest). The last-refreshed-resource-monitor-digest + annotation lives on the jobs-manager Deployment in the release + namespace, which the helper reads. */}} + {{- $rmDigest = include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" $rmDigest "annotationImage" "resource-monitor") }} image: {{ include "tracebloc.image" (dict "repository" "tracebloc/resource-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" $rmDigest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} # #569: IfNotPresent wherever an update path exists without `Always` -- # see tracebloc.controlPlanePullPolicy. `Always` made every node's pod From 8e1fc65c4318e4f6c52b39d3cd70d511b21407b7 Mon Sep 17 00:00:00 2001 From: Syed Saqlain Date: Wed, 9 Sep 2026 17:09:18 +0400 Subject: [PATCH 2/8] fix(chart): validate the annotation digest and honour imageRefresh.suspend (client-runtime#199) blocking 1 (@shujaatTracebloc on #1013): the last-refreshed annotation reaches an image: field unchecked, unlike the 11 values digest keys the schema guards with ^(sha256:[a-f0-9]{64})?$. A malformed value renders an unstartable ref helm cannot detect. controlPlaneDigest now regexMatches the annotation and degrades to :tag otherwise. blocking 2: a suspended CronJob never re-pins, so its frozen annotation must not pin the render -- gate controlPlaneDigest and controlPlanePullPolicy on not imageRefresh.suspend so a suspended edge stays on :tag with Always. Updates the e2e sentinel to a valid 64-hex digest; adds two jobs-manager pullPolicy unit tests. helm-unittest 697, lint clean. Co-Authored-By: Claude Opus 4.8 --- client/templates/_helpers.tpl | 23 ++++++++++++++++++++--- client/tests/jobs_manager_test.yaml | 19 +++++++++++++++++++ scripts/tests/e2e-auto-upgrade.sh | 8 ++++++-- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/client/templates/_helpers.tpl b/client/templates/_helpers.tpl index c3a1e469..83f78939 100644 --- a/client/templates/_helpers.tpl +++ b/client/templates/_helpers.tpl @@ -484,9 +484,10 @@ true */}} {{- define "tracebloc.controlPlanePullPolicy" -}} {{- $mirror := (dig "imageRegistry" "docker.io" (.root.Values.global | default dict)) | default "docker.io" -}} +{{- $ir := .root.Values.imageRefresh | default dict -}} {{- if .digest -}} IfNotPresent -{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") -}} +{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") (not $ir.suspend) -}} IfNotPresent {{- else -}} Always @@ -524,12 +525,28 @@ Always */}} {{- define "tracebloc.controlPlaneDigest" -}} {{- $mirror := (dig "imageRegistry" "docker.io" (.root.Values.global | default dict)) | default "docker.io" -}} +{{- $ir := .root.Values.imageRefresh | default dict -}} {{- if .operatorDigest -}} {{- .operatorDigest -}} -{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") -}} +{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") (not $ir.suspend) -}} {{- $dep := lookup "apps/v1" "Deployment" .root.Release.Namespace (printf "%s-jobs-manager" (include "tracebloc.fullname" .root)) -}} {{- if $dep -}} -{{- index (($dep.metadata).annotations | default dict) (printf "tracebloc.io/last-refreshed-%s-digest" .annotationImage) | default "" -}} +{{- $ann := index (($dep.metadata).annotations | default dict) (printf "tracebloc.io/last-refreshed-%s-digest" .annotationImage) | default "" -}} +{{/* + VALIDATE before rendering: the 11 values digest keys are schema-guarded by + `^(sha256:[a-f0-9]{64})?$`, but this annotation is written out-of-band by + image-refresh (kubectl) and reaches an `image:` field unchecked. A malformed + value (`tracebloc.image` drops the tag when a digest is present) renders an + unstartable ref helm cannot detect — the kubelet reports InvalidImageName + while the apiserver accepts the spec. Require a full sha256 digest; anything + else degrades to `""` → `:tag`, the safe fallback (@shujaatTracebloc on #1013). + `suspend` is honoured above: a suspended CronJob never runs, so its last + annotation is frozen and must NOT pin the render — a newly joined node would + otherwise pull a stale digest instead of the current tag. +*/}} +{{- if regexMatch "^sha256:[a-f0-9]{64}$" $ann -}} +{{- $ann -}} +{{- end -}} {{- end -}} {{- end -}} {{- end }} diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index 964de569..1ebff019 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -1227,3 +1227,22 @@ tests: content: name: NODE_AGENTS_NAMESPACE value: "" + + # backend#199 / client-runtime#199 (@shujaatTracebloc on #1013): a suspended + # image-refresh CronJob never re-pins, so its last last-refreshed annotation is + # frozen. tracebloc.controlPlaneDigest must then NOT render that digest, and + # controlPlanePullPolicy must fall back to Always so a stale :tag layer on a + # newly joined node is re-pulled instead of silently running an old build. + - it: control-plane pullPolicy is IfNotPresent when image-refresh is active + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + - it: control-plane pullPolicy falls back to Always when image-refresh is suspended + set: + imageRefresh: + suspend: true + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always diff --git a/scripts/tests/e2e-auto-upgrade.sh b/scripts/tests/e2e-auto-upgrade.sh index 7d8aa8f2..4bb58891 100755 --- a/scripts/tests/e2e-auto-upgrade.sh +++ b/scripts/tests/e2e-auto-upgrade.sh @@ -191,8 +191,12 @@ BASELINE_EGRESS_PROXY_URL="$(jm_egress_proxy_url)" echo " baseline egress posture: external_443=$([ "$BASELINE_EXTERNAL_443" = 1 ] && echo present || echo absent) egress_proxy_url=${BASELINE_EGRESS_PROXY_URL:-}" echo "── simulate an image-refresh-managed annotation (must survive upgrades) ──" +# A VALID sha256 digest (64 hex): tracebloc.controlPlaneDigest now validates the +# annotation against `^sha256:[a-f0-9]{64}$` before rendering it onto `image:`, so +# a placeholder like `sha256:e2e-sentinel` would degrade to `:tag` and stop +# exercising branch 2 (the digest render this test exists to protect). kubectl annotate -n "$NS" "$(jm_deploy)" \ - "tracebloc.io/last-refreshed-jobs-manager-digest=sha256:e2e-sentinel" --overwrite + "tracebloc.io/last-refreshed-jobs-manager-digest=sha256:e2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee" --overwrite echo "── path 1: manual-operator habit — helm upgrade --reuse-values ──" # Old stored values replayed against the new chart: every new key is absent and @@ -275,7 +279,7 @@ kubectl get deploy "${NS}-egress-proxy" -n "$NS" >/dev/null \ || fail "auto-upgrade did not deploy the egress gateway (new defaults did not flow)" ANNOT="$(kubectl get -n "$NS" "$(jm_deploy)" \ -o jsonpath='{.metadata.annotations.tracebloc\.io/last-refreshed-jobs-manager-digest}')" -[ "$ANNOT" = "sha256:e2e-sentinel" ] || fail "image-refresh annotation was clobbered by the upgrade" +[ "$ANNOT" = "sha256:e2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee" ] || fail "image-refresh annotation was clobbered by the upgrade" DEPLOYED="$(helm list -n "$NS" --filter "^${NS}\$" -o yaml \ | awk '/^[[:space:]]*chart:/ {print $2; exit}')" [ "$DEPLOYED" = "client-${LOCAL_VERSION}" ] || fail "deployed chart is $DEPLOYED, expected client-${LOCAL_VERSION}" From dd07635ad1d43e8eeed8314dcbd31f6da207f8d5 Mon Sep 17 00:00:00 2001 From: Syed Saqlain Date: Wed, 9 Sep 2026 17:20:43 +0400 Subject: [PATCH 3/8] test(chart): assert path 2 renders the last-refreshed digest onto the jobs-manager image (client-runtime#199) The e2e-auto-upgrade path-2 check asserted only that the last-refreshed annotation survived the upgrade, not that tracebloc.controlPlaneDigest actually rendered that digest onto the container image. The annotation is a k8s object helm never touches, so it survives even if the helper is removed and the image reverts to a floating :tag -- the exact revert this PR prevents. Add jm_controlplane_image_digest() and assert the rendered image digest equals the annotation, killing that mutation. Bind the sentinel digest once (E2E_REFRESH_DIGEST) and reuse it across both checks. Co-Authored-By: Claude Opus 4.8 --- scripts/tests/e2e-auto-upgrade.sh | 35 +++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/scripts/tests/e2e-auto-upgrade.sh b/scripts/tests/e2e-auto-upgrade.sh index 4bb58891..fd55b16a 100755 --- a/scripts/tests/e2e-auto-upgrade.sh +++ b/scripts/tests/e2e-auto-upgrade.sh @@ -116,6 +116,19 @@ local_prod_digest() { awk -F'"' '/^[[:space:]]*prodDigest:[[:space:]]*"/ {print $2; exit}' "$CHART_DIR/values.yaml" } +# The control-plane image DIGEST the jobs-manager container is ACTUALLY running, +# read off the LIVE Deployment and extracted from `image:` (repo@sha256:…). This +# is what `tracebloc.controlPlaneDigest` must render from the last-refreshed +# annotation (client-runtime#199) — assert the rendered image, not merely that +# the annotation survived: the annotation is a k8s object helm never touches, so +# it survives even if the helper is removed and the image reverts to a floating +# `:tag`. Empty output means no digest is pinned on `image:` (the revert). +jm_controlplane_image_digest() { + kubectl get -n "$NS" "$(jm_deploy)" \ + -o jsonpath='{.spec.template.spec.containers[0].image}' \ + | sed -n 's/.*@\(sha256:[a-f0-9]\{64\}\).*/\1/p' +} + echo "═══════════════════════════════════════════════════════════════════════" echo " E2E auto-upgrade gate arch: $(uname -m) kernel: $(uname -r)" echo "═══════════════════════════════════════════════════════════════════════" @@ -191,12 +204,14 @@ BASELINE_EGRESS_PROXY_URL="$(jm_egress_proxy_url)" echo " baseline egress posture: external_443=$([ "$BASELINE_EXTERNAL_443" = 1 ] && echo present || echo absent) egress_proxy_url=${BASELINE_EGRESS_PROXY_URL:-}" echo "── simulate an image-refresh-managed annotation (must survive upgrades) ──" -# A VALID sha256 digest (64 hex): tracebloc.controlPlaneDigest now validates the +# A VALID sha256 digest (64 hex): tracebloc.controlPlaneDigest validates the # annotation against `^sha256:[a-f0-9]{64}$` before rendering it onto `image:`, so # a placeholder like `sha256:e2e-sentinel` would degrade to `:tag` and stop -# exercising branch 2 (the digest render this test exists to protect). +# exercising branch 2 (the digest render this test exists to protect). Bound once +# and reused by both the survival check and the rendered-image check in path 2. +E2E_REFRESH_DIGEST="sha256:e2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee" kubectl annotate -n "$NS" "$(jm_deploy)" \ - "tracebloc.io/last-refreshed-jobs-manager-digest=sha256:e2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee" --overwrite + "tracebloc.io/last-refreshed-jobs-manager-digest=$E2E_REFRESH_DIGEST" --overwrite echo "── path 1: manual-operator habit — helm upgrade --reuse-values ──" # Old stored values replayed against the new chart: every new key is absent and @@ -279,7 +294,19 @@ kubectl get deploy "${NS}-egress-proxy" -n "$NS" >/dev/null \ || fail "auto-upgrade did not deploy the egress gateway (new defaults did not flow)" ANNOT="$(kubectl get -n "$NS" "$(jm_deploy)" \ -o jsonpath='{.metadata.annotations.tracebloc\.io/last-refreshed-jobs-manager-digest}')" -[ "$ANNOT" = "sha256:e2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee" ] || fail "image-refresh annotation was clobbered by the upgrade" +[ "$ANNOT" = "$E2E_REFRESH_DIGEST" ] || fail "image-refresh annotation was clobbered by the upgrade" +# THE #1013 CONTRACT: the auto-upgrade must SEED the control-plane image from that +# annotation, so the jobs-manager container renders repo@ instead of the +# floating :tag a stale node would run old (client-runtime#199). The annotation +# check above is necessary but not sufficient — helm never touches the annotation, +# so it survives even if tracebloc.controlPlaneDigest is removed or always returns +# empty and the image silently reverts to :tag. Assert the RENDERED image digest, +# which is what actually protects the pin: this is the assertion that reddens if +# the helper is dropped. (Fleet auto-upgrade E2E confirms later paths tolerate the +# unpullable sentinel digest — no --wait on this upgrade, spec-only reads here.) +CP_DIGEST="$(jm_controlplane_image_digest)" +[ "$CP_DIGEST" = "$E2E_REFRESH_DIGEST" ] \ + || fail "auto-upgrade did not seed the jobs-manager control-plane image from the last-refreshed digest: got '${CP_DIGEST:-}', want '$E2E_REFRESH_DIGEST' (tracebloc.controlPlaneDigest did not render the pin — client-runtime#199 revert not prevented)" DEPLOYED="$(helm list -n "$NS" --filter "^${NS}\$" -o yaml \ | awk '/^[[:space:]]*chart:/ {print $2; exit}')" [ "$DEPLOYED" = "client-${LOCAL_VERSION}" ] || fail "deployed chart is $DEPLOYED, expected client-${LOCAL_VERSION}" From 616b84359daa7e96a8596056411358939ed8ea97 Mon Sep 17 00:00:00 2001 From: Syed Saqlain Date: Wed, 9 Sep 2026 18:24:02 +0400 Subject: [PATCH 4/8] fix(chart): suspend keeps the control-plane pin, add clusterless lookup tests (client-runtime#199) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses @LukasWodka's review on #1013. RECONCILES A REVIEWER CONFLICT (@shujaatTracebloc). shujaat's earlier BLOCKING-2 asked that imageRefresh.suspend UN-PIN — render :tag + Always so a newly joined node re-pulls the current tag. LukasWodka's point 1 shows that is wrong: suspend is used to FREEZE an edge during an incident, and un-pinning re-renders jobs-manager (Recreate), requests-proxy and the resource-monitor DaemonSet onto :tag/Always — three unplanned rollouts plus a downgrade to whatever :tag points at now — contradicting the values.schema.json promise that suspend "pause[s] without removing the resources". Decision: suspend stops POLLING, not un-pinning. Removed the (not $ir.suspend) gate from both tracebloc.controlPlaneDigest and tracebloc.controlPlanePullPolicy, so a suspended edge keeps rendering the last-observed @digest + IfNotPresent. This still answers shujaat's newly-joined-node concern: that node pulls the frozen @digest — a real, previously-resolved image — not a missing one. Also in this push (LukasWodka's other points): - Add client/tests/control_plane_digest_test.yaml — a clusterless suite that fakes `lookup` via helm-unittest 0.5.2's kubernetesProvider. Covers the five image sites (init, api, pods-monitor, requests-proxy-follows-jobs-manager, resource-monitor), operator-pin beats the annotation, malformed -> :tag, the suspend-keeps-pin decision, and the disabled/mirror/first-install :tag edges. Mutation-verified: discarding the lookup reddens the digest cases, re-adding the suspend gate reddens the suspend case, neutering the regexMatch reddens the malformed case — the exact clusterless ceiling shujaat noted, now lifted. - Remove the duplicate/misleading pull-policy cases appended to jobs_manager_test.yaml (the L1236 verbatim dup of L1130 and the now-wrong suspend->Always case, whose block comment wrongly claimed the helper "must NOT render that digest"); the private backend#199 ref in that comment goes with it. - Docs the PR made false: image-refresh CronJob "HELM RE-RENDER" limitation now marked RESOLVED as of 1.9.110; SEAL-CHECK.md no longer tells operators to pin values to survive a revert the chart no longer performs. - First-observation churn documented in the CronJob first-tick contract; helper docstring "last APPLIED" -> "last OBSERVED"; #569 offline-guarantee narrowing noted at controlPlanePullPolicy; MIGRATIONS.md server-side-apply gets a line that from 1.9.110 a Helm 4 .image conflict signals annotation/live lag. - e2e-auto-upgrade.sh: the unpullable sentinel now documented and asserted explicitly (path 1 spec-only check + Init:ImagePullBackOff-by-design comment) so the next author is not surprised. Merge order unchanged: B stays behind A (#1008). Chart stays 1.9.110. helm lint clean; helm unittest 706/706 (41 suites); helm template renders :tag clusterless and @digest on the operator pin; shellcheck + bash -n clean. Co-Authored-By: Claude Opus 4.8 --- client/templates/_helpers.tpl | 60 +++- client/templates/image-refresh-cronjob.yaml | 44 ++- .../templates/requests-proxy-deployment.yaml | 2 +- .../templates/resource-monitor-daemonset.yaml | 2 +- client/tests/control_plane_digest_test.yaml | 288 ++++++++++++++++++ client/tests/jobs_manager_test.yaml | 19 -- docs/MIGRATIONS.md | 2 + docs/SEAL-CHECK.md | 16 +- scripts/tests/e2e-auto-upgrade.sh | 26 +- 9 files changed, 403 insertions(+), 56 deletions(-) create mode 100644 client/tests/control_plane_digest_test.yaml diff --git a/client/templates/_helpers.tpl b/client/templates/_helpers.tpl index 83f78939..330a0f82 100644 --- a/client/templates/_helpers.tpl +++ b/client/templates/_helpers.tpl @@ -480,14 +480,27 @@ true freezing — a frozen control plane with no signal is worse than a restart that needs the network. + #569 OFFLINE GUARANTEE — NARROWED as of 1.9.110 (#1013). IfNotPresent only + helps when the REFERENCE is cache-satisfiable. Until 1.9.110 a `helm upgrade` + re-rendered the bare `:tag`, which a node that had pulled that tag could always + satisfy from cache, so an offline restart ran the (possibly stale) cached + image. From 1.9.110 tracebloc.controlPlaneDigest can render `repo@` + seeded from the last-refreshed annotation (see its header), and because the + first-observation path RECORDS a digest without re-imaging, the rendered digest + may be one the node has never pulled. Online that is the fix working; OFFLINE + it turns "restarts on a stale image" into "cannot start" (ImagePullBackOff) for + that one digest. Two corollaries: a `global.imageRegistry` mirror is exempt + (branch 2 stays inert there → `:tag`), and a side-loaded k3d image is affected + because `k3d image import` stores a tag alias with no resolvable digest and a + local k3d install IS docker.io, so the mirror gate does not help it. + Usage: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" $d "root" $) }} */}} {{- define "tracebloc.controlPlanePullPolicy" -}} {{- $mirror := (dig "imageRegistry" "docker.io" (.root.Values.global | default dict)) | default "docker.io" -}} -{{- $ir := .root.Values.imageRefresh | default dict -}} {{- if .digest -}} IfNotPresent -{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") (not $ir.suspend) -}} +{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") -}} IfNotPresent {{- else -}} Always @@ -501,19 +514,35 @@ Always 1. an operator's explicit values pin (`images..digest`) — wins, behaviour unchanged; 2. else, when image-refresh is the update path (enabled AND the docker.io - mirror), the digest image-refresh last APPLIED, read from the + mirror), the digest image-refresh last OBSERVED, read from the jobs-manager Deployment's `tracebloc.io/last-refreshed--digest` - annotation via `lookup`. This is what makes a `helm upgrade` RENDER - `repo@digest` instead of reverting to the bare `:tag` and dropping - image-refresh's out-of-band `kubectl set image` pin — the revert that let - a stale-`:tag` node cache silently run an OLD control-plane image - (client-runtime#199). `lookup` returns empty during `helm template` / - `helm diff` / the FIRST install (no Deployment yet), so it degrades to - `""` → `:tag` there, which is correct: nothing is pinned yet and the - node's fresh tag pull is the right image. + annotation via `lookup`. ("Observed", not "applied": on the FIRST tick + image-refresh records the current digest WITHOUT `set image`, deliberately + — see the image-refresh CronJob header — so the annotation is what refresh + last saw, which is only the same as what it last applied once a real + refresh has run.) This is what makes a `helm upgrade` RENDER `repo@digest` + instead of reverting to the bare `:tag` and dropping image-refresh's + out-of-band `kubectl set image` pin — the revert that let a stale-`:tag` + node cache silently run an OLD control-plane image (client-runtime#199). + `lookup` returns empty during `helm template` / `helm diff` / the FIRST + install (no Deployment yet), so it degrades to `""` → `:tag` there, which + is correct: nothing is pinned yet and the node's fresh tag pull is the + right image. 3. else `""` (bare `:tag`; `Always` via controlPlanePullPolicy is then the update path — the non-refresh / mirror edges #569 protects). + On `imageRefresh.suspend: true` the pin is KEPT, not dropped. suspend stops the + CronJob from POLLING; it does not un-pin (@LukasWodka on #1013). The knob is + used to FREEZE an edge during an incident, and `values.schema.json` promises it + "pause[s] without removing the resources" — un-pinning here would re-render + jobs-manager (Recreate), requests-proxy and the resource-monitor DaemonSet onto + the floating `:tag` + `Always`, three unplanned rollouts plus a downgrade to + whatever `:tag` points at now, which is the opposite of freezing. Keeping the + last-observed `@digest` freezes the edge on a real, previously-resolved image; + a newly joined node then pulls that frozen digest (a valid ref), not a missing + one, which still answers the newly-joined-node concern that first put a suspend + gate here (@shujaatTracebloc's earlier BLOCKING-2 on #1013, reversed here). + The lookup targets ONLY the jobs-manager Deployment (where image-refresh writes every last-refreshed annotation) in the release namespace — a read the auto-upgrade SA already holds (its release-ns Role grants all verbs on all @@ -525,10 +554,9 @@ Always */}} {{- define "tracebloc.controlPlaneDigest" -}} {{- $mirror := (dig "imageRegistry" "docker.io" (.root.Values.global | default dict)) | default "docker.io" -}} -{{- $ir := .root.Values.imageRefresh | default dict -}} {{- if .operatorDigest -}} {{- .operatorDigest -}} -{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") (not $ir.suspend) -}} +{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") -}} {{- $dep := lookup "apps/v1" "Deployment" .root.Release.Namespace (printf "%s-jobs-manager" (include "tracebloc.fullname" .root)) -}} {{- if $dep -}} {{- $ann := index (($dep.metadata).annotations | default dict) (printf "tracebloc.io/last-refreshed-%s-digest" .annotationImage) | default "" -}} @@ -540,9 +568,9 @@ Always unstartable ref helm cannot detect — the kubelet reports InvalidImageName while the apiserver accepts the spec. Require a full sha256 digest; anything else degrades to `""` → `:tag`, the safe fallback (@shujaatTracebloc on #1013). - `suspend` is honoured above: a suspended CronJob never runs, so its last - annotation is frozen and must NOT pin the render — a newly joined node would - otherwise pull a stale digest instead of the current tag. + `suspend` is deliberately NOT gated here: a suspended edge keeps rendering the + last-observed digest so `helm upgrade` freezes it in place rather than rolling + it onto the floating tag (see the header note). */}} {{- if regexMatch "^sha256:[a-f0-9]{64}$" $ann -}} {{- $ann -}} diff --git a/client/templates/image-refresh-cronjob.yaml b/client/templates/image-refresh-cronjob.yaml index 2b371ebe..43ae5a01 100644 --- a/client/templates/image-refresh-cronjob.yaml +++ b/client/templates/image-refresh-cronjob.yaml @@ -42,19 +42,24 @@ data: # change its image ref either), so it only ever moved by accident. # # TWO KNOWN, BOUNDED LIMITATIONS of keeping the annotation (rather than the - # live pod spec) as the source of truth. Both are self-healing at the next - # upstream image change, which at the control plane's release cadence is - # days, and neither can break a running edge — the pods stay offline-safe - # throughout because IfNotPresent does not depend on any of this. + # live pod spec) as the source of truth — the FIRST now RESOLVED by #1013. + # Both are self-healing at the next upstream image change, which at the + # control plane's release cadence is days, and neither can break a running + # edge — the pods stay offline-safe throughout because IfNotPresent does not + # depend on any of this. # - # 1. HELM RE-RENDER. `helm upgrade --reset-then-reuse-values` (the fleet - # auto-upgrade path) re-renders the templates, which write `repo:tag` - # and so revert an earlier `set image` pin. This tick will NOT re-pin: - # the annotation still records that digest, so `recorded == latest` - # and the loop no-ops. The edge floats on the tag until the next - # upstream release re-pins it. Note this only happens on a chart - # VERSION bump — auto-upgrade compares versions and skips otherwise — - # so it is not an hourly revert. + # 1. HELM RE-RENDER — RESOLVED as of chart 1.9.110 (#1013 / + # client-runtime#199). `helm upgrade --reset-then-reuse-values` (the + # fleet auto-upgrade path) re-renders the templates; BEFORE 1.9.110 they + # wrote the bare `repo:tag` and so reverted an earlier `set image` pin, + # and the edge floated on the tag until the next upstream release + # re-pinned it. FROM 1.9.110 the control-plane image sites render + # through `tracebloc.controlPlaneDigest`, which reads THIS annotation via + # `lookup` and renders `repo@` — so a re-render + # now PRESERVES the pin instead of dropping it. (Still only relevant on a + # chart VERSION bump — auto-upgrade compares versions and skips + # otherwise.) The one residue is the first-observation roll in the + # first-tick contract below. # # 2. PRE-EXISTING SKEW. requests-proxy and jobs-manager may already be # running different builds of the same image on an edge upgrading INTO @@ -63,7 +68,7 @@ data: # in the same tick); it does not detect and repair skew that predates # it, because it compares registry-vs-annotation, never pod-vs-pod. # - # Fixing either properly means reconciling against each workload's LIVE + # Fixing the remaining one properly means reconciling against each workload's LIVE # container image instead of a shared annotation — a declarative reconcile, # which `set image` makes possible for the first time (`rollout restart` # was a blind action, which is why the annotation existed at all). That is @@ -100,6 +105,19 @@ data: # reproducible. Offline-safety is what #569 is fixing; reproducibility # follows on the next upstream release. # + # First-observation roll (chart ≥ 1.9.110, #1013). Once this tick has + # RECORDED D0 without re-imaging, the live spec still says `:tag` but the + # annotation now says `@D0`, so the NEXT chart bump renders `repo@D0` + # (tracebloc.controlPlaneDigest reads the annotation) and rolls jobs-manager + # (Recreate), requests-proxy and the resource-monitor DaemonSet ONCE, for + # byte-identical content. auto-upgrade runs that upgrade `--atomic --wait` + # (auto-upgrade-cronjob.yaml), so on a slow single-node edge the one-time + # roll can hit the timeout and roll back, then re-attempt identically on the + # next bump. This is the deliberate tradeoff for NOT `set image`-ing on the + # first tick; pinning on the first tick instead would move the same roll to + # install time for every edge. Also narrows #569's offline guarantee — see + # the note at controlPlanePullPolicy in _helpers.tpl. + # # Parsing: awk/sed/grep + jq. jq used only where JSON-with-dotted-keys # or container/env-array filtering motivates it; the rest stays in pure # shell so the script survives image swaps to leaner bases — same diff --git a/client/templates/requests-proxy-deployment.yaml b/client/templates/requests-proxy-deployment.yaml index f9c17c0a..0cb1db2a 100644 --- a/client/templates/requests-proxy-deployment.yaml +++ b/client/templates/requests-proxy-deployment.yaml @@ -41,7 +41,7 @@ spec: these keys. */}} {{- $rpDigest := (dig "requestsProxy" "digest" "" $rpImages) | default (dig "jobsManager" "digest" "" $rpImages) }} - {{- /* client-runtime#199: seed the digest image-refresh last applied so a + {{- /* client-runtime#199: seed the digest image-refresh last observed so a `helm upgrade` renders repo@digest, not the bare :tag (see tracebloc.controlPlaneDigest). requests-proxy runs the SAME jobs-manager image and follows its annotation. */}} diff --git a/client/templates/resource-monitor-daemonset.yaml b/client/templates/resource-monitor-daemonset.yaml index eb1b544a..814c8dc9 100644 --- a/client/templates/resource-monitor-daemonset.yaml +++ b/client/templates/resource-monitor-daemonset.yaml @@ -136,7 +136,7 @@ spec: entry). `dig` is not usable here — it rejects chartutil.Values. */}} {{- $rmDigest := (default (dict) (default (dict) .Values.images).resourceMonitor).digest | default "" }} - {{- /* client-runtime#199: seed the digest image-refresh last applied so a + {{- /* client-runtime#199: seed the digest image-refresh last observed so a `helm upgrade` renders repo@digest, not the bare :tag (see tracebloc.controlPlaneDigest). The last-refreshed-resource-monitor-digest annotation lives on the jobs-manager Deployment in the release diff --git a/client/tests/control_plane_digest_test.yaml b/client/tests/control_plane_digest_test.yaml new file mode 100644 index 00000000..de17959f --- /dev/null +++ b/client/tests/control_plane_digest_test.yaml @@ -0,0 +1,288 @@ +suite: Control-plane digest seeding (tracebloc.controlPlaneDigest) +# client-runtime#199 (@LukasWodka on #1013): a `helm upgrade` used to re-render +# the bare `repo:tag` for every control-plane image and so DROP the digest that +# image-refresh had pinned out-of-band with `kubectl set image repo@digest`; a +# stale-`:tag` node cache then silently ran an OLD control-plane build. +# tracebloc.controlPlaneDigest closes that by reading the digest image-refresh +# last OBSERVED off the jobs-manager Deployment's +# `tracebloc.io/last-refreshed--digest` annotation via `lookup`, so the +# upgrade renders `repo@digest` and preserves the pin. +# +# The whole point of the PR is that lookup path, and it was previously claimed to +# be untestable without a cluster. It is NOT: helm-unittest 0.5.2 (the version CI +# pins) fakes `lookup` via `kubernetesProvider`, so the annotation seed, the +# operator-pin precedence, the malformed-value guard and the suspend decision are +# all covered clusterless here. `Helm unit tests` is a REQUIRED check on develop; +# the k3d e2e that also exercises this path is path-filtered and NOT required, so +# without this suite a regression of the seed could merge green. +templates: + - templates/jobs-manager-deployment.yaml + - templates/requests-proxy-deployment.yaml + - templates/resource-monitor-daemonset.yaml +set: + clientId: "test-id" + clientPassword: "test" + dockerRegistry: + server: https://index.docker.io/v1/ + username: test + password: test + email: test@test.com +# Under helm-unittest .Release.Name is RELEASE-NAME and .Release.Namespace is +# NAMESPACE, so tracebloc.fullname is RELEASE-NAME and the helper looks up +# RELEASE-NAME-jobs-manager in NAMESPACE. Distinct digests per key prove each +# site reads its OWN annotation rather than a shared value. +tests: + # ── the five image sites render their last-observed digest ────────────────── + - it: api renders the last-refreshed jobs-manager digest (the lookup seed) + # FIRST case deliberately: this is the one that reddens if the helper is + # neutered (e.g. the lookup result discarded). It is the clusterless proof + # that branch 2 works, which the k3d e2e used to be the only witness of. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + tracebloc.io/last-refreshed-pods-monitor-digest: "sha256:2222222222222222222222222222222222222222222222222222222222222222" + tracebloc.io/last-refreshed-resource-monitor-digest: "sha256:3333333333333333333333333333333333333333333333333333333333333333" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: docker.io/tracebloc/jobs-manager@sha256:1111111111111111111111111111111111111111111111111111111111111111 + # #569: a rendered digest is immutable, so the pull policy stays IfNotPresent. + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].imagePullPolicy' + value: IfNotPresent + + - it: the wait-for-mysql init container renders the jobs-manager digest too + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'spec.template.spec.initContainers[?(@.name=="wait-for-mysql")].image' + value: docker.io/tracebloc/jobs-manager@sha256:1111111111111111111111111111111111111111111111111111111111111111 + + - it: pods-monitor-container follows its OWN last-refreshed annotation key + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + tracebloc.io/last-refreshed-pods-monitor-digest: "sha256:2222222222222222222222222222222222222222222222222222222222222222" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="pods-monitor-container")].image' + value: docker.io/tracebloc/pods-monitor@sha256:2222222222222222222222222222222222222222222222222222222222222222 + + - it: requests-proxy runs the jobs-manager image and follows the jobs-manager digest + template: templates/requests-proxy-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="proxy")].image' + value: docker.io/tracebloc/jobs-manager@sha256:1111111111111111111111111111111111111111111111111111111111111111 + + - it: resource-monitor DaemonSet renders its own last-refreshed digest + template: templates/resource-monitor-daemonset.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-resource-monitor-digest: "sha256:3333333333333333333333333333333333333333333333333333333333333333" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="tracebloc-resource-monitor")].image' + value: docker.io/tracebloc/resource-monitor@sha256:3333333333333333333333333333333333333333333333333333333333333333 + + # ── operator pin beats the annotation ─────────────────────────────────────── + - it: an operator values pin wins over the last-refreshed annotation + # images..digest is priority 1 in the helper: an operator who pins a + # digest gets exactly that, whatever image-refresh last observed. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + set: + images: + jobsManager: + digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: docker.io/tracebloc/jobs-manager@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + + # ── malformed annotation degrades to the floating :tag ────────────────────── + - it: a malformed annotation degrades to the floating :tag (validation guard) + # The annotation is written out-of-band by kubectl and reaches image: unchecked; + # tracebloc.image drops the tag when a digest is present, so a malformed value + # would render an unstartable ref (kubelet InvalidImageName). The helper's + # regexMatch against ^sha256:[a-f0-9]{64}$ rejects it and falls back to :tag. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:not-a-real-digest" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: docker.io/tracebloc/jobs-manager:prod + + # ── the suspend decision: suspend KEEPS the pin (@LukasWodka on #1013) ─────── + - it: a suspended image-refresh KEEPS the last-observed digest (does not un-pin) + # suspend stops the CronJob POLLING; it must not un-pin. The knob freezes an + # edge during an incident and values.schema.json promises it pauses "without + # removing the resources", so re-rendering the floating :tag here would roll + # jobs-manager (Recreate), requests-proxy and the resource-monitor DaemonSet + # and downgrade to whatever :tag points at now — the opposite of freezing. + # (This reverses an earlier gate that made suspend render :tag + Always; a + # newly joined node still gets a valid, previously-resolved digest to pull.) + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + set: + imageRefresh: + suspend: true + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: docker.io/tracebloc/jobs-manager@sha256:1111111111111111111111111111111111111111111111111111111111111111 + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].imagePullPolicy' + value: IfNotPresent + + # ── the non-refresh / mirror edges keep the floating :tag ─────────────────── + - it: imageRefresh disabled ignores the annotation and floats on :tag + # Branch 2 is gated on the refresh path being live; with it off the annotation + # is not the update path, so the edge floats on :tag (+ Always via pull policy). + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + set: + imageRefresh: + enabled: false + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: docker.io/tracebloc/jobs-manager:prod + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].imagePullPolicy' + value: Always + + - it: a private mirror ignores the annotation and floats on :tag + # The reconcile resolves digests from docker.io, so under a mirror it goes + # inert by design and the helper must not pin a digest the mirror may not hold. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + set: + global: + imageRegistry: "mirror.example.com" + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: mirror.example.com/tracebloc/jobs-manager:prod + + - it: no jobs-manager Deployment yet (first install) floats on :tag + # lookup is empty on the FIRST install / helm template / helm diff, so the + # helper degrades to "" -> :tag. Modelled here by an empty object set. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: [] + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: docker.io/tracebloc/jobs-manager:prod diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index 1ebff019..964de569 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -1227,22 +1227,3 @@ tests: content: name: NODE_AGENTS_NAMESPACE value: "" - - # backend#199 / client-runtime#199 (@shujaatTracebloc on #1013): a suspended - # image-refresh CronJob never re-pins, so its last last-refreshed annotation is - # frozen. tracebloc.controlPlaneDigest must then NOT render that digest, and - # controlPlanePullPolicy must fall back to Always so a stale :tag layer on a - # newly joined node is re-pulled instead of silently running an old build. - - it: control-plane pullPolicy is IfNotPresent when image-refresh is active - asserts: - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: IfNotPresent - - it: control-plane pullPolicy falls back to Always when image-refresh is suspended - set: - imageRefresh: - suspend: true - asserts: - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: Always diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md index b7bc44be..e3732c91 100644 --- a/docs/MIGRATIONS.md +++ b/docs/MIGRATIONS.md @@ -245,6 +245,8 @@ Two things to know before you force: 1. **`--force-conflicts` re-takes *every* field the chart renders that a non-Helm manager owns — not only `limits.cpu`.** Helm prints the full conflict list on the failed attempt; read it first. On a long-running fleet the list can include `image-refresh`'s pinned digests (`kubectl set image`); forcing reverts them to the chart's rendered image, and `image-refresh` re-pins on its next tick. That is recoverable but is a real, if brief, image churn — don't force blind. + **From chart 1.9.110 (#1013 / client-runtime#199) the `.image` conflict narrows.** The chart now renders the last-refreshed `@digest` itself (`tracebloc.controlPlaneDigest` reads image-refresh's annotation via `lookup`), so in the **steady state** Helm applies the same value the `kubectl-set` manager holds and the `.spec…containers[…].image` conflict disappears. A `.image` conflict on 1.9.110+ therefore signals **annotation-vs-live LAG** — the rollout-timeout / `helm rollback` window where the annotation trails the live spec — not steady-state drift. Wait one refresh tick for the reconcile to re-pin, then re-run; or `--force-conflicts` to take the chart's rendered (annotation) digest, accepting it may briefly trail until the next tick. + 2. **`--server-side=true` is not optional, even though Helm 4 defaults to it.** After a rollback (including the automatic one the `auto-upgrade` CronJob performs when it reads a `pending-upgrade` release as a wedge — `backend#2877`), the release's stored apply method can revert to client-side. Then `--force-conflicts` **alone** fails with: ``` diff --git a/docs/SEAL-CHECK.md b/docs/SEAL-CHECK.md index 88962c8c..c12ead49 100644 --- a/docs/SEAL-CHECK.md +++ b/docs/SEAL-CHECK.md @@ -287,11 +287,17 @@ client-runtime#199. run a build carrying client-runtime#416 (the HF-offline injection) *before* the seal, or NLP templates fail by network block instead of the clean closed door. On each cluster the chart renders control-plane images as `repository:tag` + -`IfNotPresent`, and the `image-refresh` CronJob pins the live digest. dev now -tracks its `:dev` tag (auto-refresh); staging/prod **pin the #416 digest** in -values (`images.jobsManager.digest`) because their tag node-caches were stale -(pre-#416) — pinning is deterministic and survives `--reset-then-reuse-values`, -but disables `image-refresh` auto-tracking until the pin is bumped. +`IfNotPresent` on a fresh install, and the `image-refresh` CronJob pins the live +digest out-of-band. **As of chart 1.9.110 (#1013 / client-runtime#199) a +`helm upgrade` no longer reverts that pin:** `tracebloc.controlPlaneDigest` reads +image-refresh's last-refreshed annotation via `lookup` and re-renders +`repository@digest`, so `--reset-then-reuse-values` preserves the digest on its +own — you no longer need a values pin merely to survive an upgrade. dev now +tracks its `:dev` tag (auto-refresh); staging/prod still **pin the #416 digest** +in values (`images.jobsManager.digest`) because their tag node-caches were stale +(pre-#416) and a values pin is the deterministic, image-refresh-independent +choice (it wins over the annotation), but that pin disables `image-refresh` +auto-tracking until it is bumped. ## Runbook: flip the §8.2 egress lockdown on a real fleet diff --git a/scripts/tests/e2e-auto-upgrade.sh b/scripts/tests/e2e-auto-upgrade.sh index 255d94df..3bed42b2 100755 --- a/scripts/tests/e2e-auto-upgrade.sh +++ b/scripts/tests/e2e-auto-upgrade.sh @@ -234,6 +234,21 @@ echo "── simulate an image-refresh-managed annotation (must survive upgrades # a placeholder like `sha256:e2e-sentinel` would degrade to `:tag` and stop # exercising branch 2 (the digest render this test exists to protect). Bound once # and reused by both the survival check and the rendered-image check in path 2. +# +# DELIBERATELY UNPULLABLE, and asserted as such below. This 64-hex digest does +# not resolve to any image in the k3d cache (a side-loaded `k3d image import` +# stores a tag alias with no resolvable digest — see the #569 offline note in +# _helpers.tpl), so any upgrade that renders the WORKING-TREE chart (path 1's +# `--reuse-values`, path 2's `--reset-then-reuse-values`) Recreates jobs-manager +# onto an unpullable ref and it sits in Init:ImagePullBackOff. (The intermediate +# reset to the PUBLISHED $PREV chart between paths 1 and 2 renders `:tag` and +# briefly un-wedges it, because $PREV predates tracebloc.controlPlaneDigest.) That +# is harmless to paths 2-5 BY CONSTRUCTION: the only readiness wait in this script +# is `kubectl wait … nodes` at the top (before this seed), and every assertion +# after this point is a spec-only read (`kubectl get … -o jsonpath`), never a +# live-pod / rollout / readiness read. If you add a step below that needs a +# running jobs-manager pod, seed a pullable digest here (or clear the annotation +# first) — do not assume the pod is up. E2E_REFRESH_DIGEST="sha256:e2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee" kubectl annotate -n "$NS" "$(jm_deploy)" \ "tracebloc.io/last-refreshed-jobs-manager-digest=$E2E_REFRESH_DIGEST" --overwrite @@ -268,7 +283,16 @@ else [ "$(jm_ingestor_digest)" = "$BASELINE_PROD_DIGEST" ] \ || fail "--reuse-values did not replay the baseline prod pin verbatim: got '$(jm_ingestor_digest)', want '$BASELINE_PROD_DIGEST' (stored computed values must win over new chart defaults on this path)" fi -echo " OK: upgrade succeeded, egress posture replayed from the baseline verbatim, ingestor pin matches the baseline era (${BASELINE_PROD_DIGEST:-floating})" +# Control-plane pin also survives --reuse-values: this upgrade renders the working +# tree chart, so tracebloc.controlPlaneDigest reads the seeded annotation and pins +# the jobs-manager image to the sentinel. Spec-only read (no live-pod dependency) — +# and this is the upgrade that first Recreates jobs-manager onto the deliberately +# unpullable sentinel, asserting explicitly here so the Init:ImagePullBackOff that +# follows is a documented, expected state rather than a surprise to the next author. +CP_DIGEST_P1="$(jm_controlplane_image_digest)" +[ "$CP_DIGEST_P1" = "$E2E_REFRESH_DIGEST" ] \ + || fail "--reuse-values did not seed the jobs-manager control-plane image from the last-refreshed digest: got '${CP_DIGEST_P1:-}', want '$E2E_REFRESH_DIGEST' (tracebloc.controlPlaneDigest did not preserve the pin on this path — client-runtime#199)" +echo " OK: upgrade succeeded, egress posture replayed from the baseline verbatim, ingestor pin matches the baseline era (${BASELINE_PROD_DIGEST:-floating}), control-plane image pinned to the last-refreshed digest (jobs-manager now Init:ImagePullBackOff on the unpullable sentinel BY DESIGN — later paths read spec only)" echo "── isolate path 2 from path 1's --reuse-values contamination (#459) ──" # path 1's --reuse-values rewrote THIS release's recorded values to the baseline's FULL From a54deb06a1d84acc67ac76459f975909f6466401 Mon Sep 17 00:00:00 2001 From: Syed Is Saqlain Date: Thu, 10 Sep 2026 05:05:04 +0400 Subject: [PATCH 5/8] docs(chart): narrow the operator-facing offline-restart guarantee for seeded digests (client-runtime#199) values.yaml, values.schema.json and the e2e comments still promised the un-narrowed #569 "offline restart cannot fail" guarantee; once image-refresh seeds a registry @digest (tracebloc.controlPlaneDigest), a node that never pulled it restarts into ImagePullBackOff. Scope the e2e --wait/\$PREV comments to the test rather than the fleet path. Doc/comment only; render unchanged. Co-Authored-By: Claude Opus 4.8 --- client/values.schema.json | 2 +- client/values.yaml | 14 ++++++++++++-- scripts/tests/e2e-auto-upgrade.sh | 15 ++++++++++----- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/client/values.schema.json b/client/values.schema.json index 1a5abdaf..ea506aac 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -1337,7 +1337,7 @@ }, "imageRefresh": { "type": "object", - "description": "Image-refresh CronJob (issue tracebloc/client#154; reworked by #569). Polls Docker Hub for the jobs-manager, pods-monitor and resource-monitor digests under the floating CLIENT_ENV tag and, on a change, pins the new digest onto the running workloads with `kubectl set image` (jobs-manager + requests-proxy Deployments and the resource-monitor DaemonSet). It used to `kubectl rollout restart` a single Deployment; the reference change is what lets those pods run imagePullPolicy=IfNotPresent and survive an offline restart. Disable to keep the running image in place until manual restart — the chart then renders imagePullPolicy=Always on the unpinned control-plane images, because a floating tag plus a restart is the only update path left.", + "description": "Image-refresh CronJob (issue tracebloc/client#154; reworked by #569). Polls Docker Hub for the jobs-manager, pods-monitor and resource-monitor digests under the floating CLIENT_ENV tag and, on a change, pins the new digest onto the running workloads with `kubectl set image` (jobs-manager + requests-proxy Deployments and the resource-monitor DaemonSet). It used to `kubectl rollout restart` a single Deployment; the reference change is what lets those pods run imagePullPolicy=IfNotPresent and survive an offline restart on a node that already cached that digest (client#1013 narrows this: once observed, the digest is seeded into the chart via tracebloc.controlPlaneDigest, so an offline/side-loaded k3d node that never pulled it restarts into ImagePullBackOff). Disable to keep the running image in place until manual restart — the chart then renders imagePullPolicy=Always on the unpinned control-plane images, because a floating tag plus a restart is the only update path left.", "properties": { "enabled": { "type": "boolean", diff --git a/client/values.yaml b/client/values.yaml index 4dbebf55..1a4d9056 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -1547,7 +1547,13 @@ podTokenTtlSeconds: 604800 # - NOTE (#569): a chart upgrade no longer picks up new IMAGE content as a # side effect. That used to work because the control-plane pods ran # `imagePullPolicy: Always`, so any restart re-resolved the floating tag; -# they now run IfNotPresent so an offline restart cannot fail. New image +# they now run IfNotPresent so an offline restart cannot fail *provided the +# node already cached the running image*. NARROWED (client#1013): once +# image-refresh has observed a digest, the chart seeds that registry +# `@digest` onto the control-plane images (see tracebloc.controlPlaneDigest), +# so a node that never pulled that exact digest — an offline/side-loaded k3d +# or Docker-Desktop edge — now restarts into ImagePullBackOff instead of the +# cached tag. New image # content arrives via the image-refresh CronJob's `kubectl set image` # instead. This CronJob still delivers everything else a chart release # carries — template, RBAC, values-default and schema changes. @@ -1720,7 +1726,11 @@ autoUpgrade: # repo@digest for byte-identical content would roll every workload — # including the DaemonSet on every node — for nothing. The cost is # that a fresh edge runs repo:tag until the first real digest change: -# still restart-safe offline, just not yet reproducible. +# still restart-safe offline, just not yet reproducible. NARROWED +# (client#1013): this holds only for the first-observation window. Once a +# digest has been recorded, the chart seeds it (tracebloc.controlPlaneDigest) +# and a later chart render pins repo@digest, so from then on an offline +# restart is restart-safe only on a node that already cached that digest. # - Idle-cheap: when the recorded digest matches today's digest, the # script exits without touching anything. Steady state is one HEAD # per image per tick, well under Docker Hub's 100/6h anonymous diff --git a/scripts/tests/e2e-auto-upgrade.sh b/scripts/tests/e2e-auto-upgrade.sh index 3bed42b2..4a634f08 100755 --- a/scripts/tests/e2e-auto-upgrade.sh +++ b/scripts/tests/e2e-auto-upgrade.sh @@ -241,9 +241,11 @@ echo "── simulate an image-refresh-managed annotation (must survive upgrades # _helpers.tpl), so any upgrade that renders the WORKING-TREE chart (path 1's # `--reuse-values`, path 2's `--reset-then-reuse-values`) Recreates jobs-manager # onto an unpullable ref and it sits in Init:ImagePullBackOff. (The intermediate -# reset to the PUBLISHED $PREV chart between paths 1 and 2 renders `:tag` and -# briefly un-wedges it, because $PREV predates tracebloc.controlPlaneDigest.) That -# is harmless to paths 2-5 BY CONSTRUCTION: the only readiness wait in this script +# reset to the PUBLISHED $PREV chart between paths 1 and 2 MAY briefly un-wedge it +# by rendering `:tag` — but only while $PREV predates tracebloc.controlPlaneDigest; +# once 1.9.110+ is the published $PREV the reset renders this sentinel too and the +# un-wedge stops happening. Do not depend on it either way.) That +# is harmless to paths 2-5 BY CONSTRUCTION regardless: the only readiness wait in this script # is `kubectl wait … nodes` at the top (before this seed), and every assertion # after this point is a spec-only read (`kubectl get … -o jsonpath`), never a # live-pod / rollout / readiness read. If you add a step below that needs a @@ -352,8 +354,11 @@ ANNOT="$(kubectl get -n "$NS" "$(jm_deploy)" \ # so it survives even if tracebloc.controlPlaneDigest is removed or always returns # empty and the image silently reverts to :tag. Assert the RENDERED image digest, # which is what actually protects the pin: this is the assertion that reddens if -# the helper is dropped. (Fleet auto-upgrade E2E confirms later paths tolerate the -# unpullable sentinel digest — no --wait on this upgrade, spec-only reads here.) +# the helper is dropped. (This test's upgrade tolerates the unpullable sentinel +# digest because *this script's* helm upgrade passes no --wait and every check here +# is a spec-only read — NOT a property of the fleet path: auto-upgrade-cronjob.yaml +# runs `--atomic --cleanup-on-fail --timeout`, and --atomic implies --wait, so on a +# real edge an unpullable seed blocks on readiness and rolls back. See the :245 note.) CP_DIGEST="$(jm_controlplane_image_digest)" [ "$CP_DIGEST" = "$E2E_REFRESH_DIGEST" ] \ || fail "auto-upgrade did not seed the jobs-manager control-plane image from the last-refreshed digest: got '${CP_DIGEST:-}', want '$E2E_REFRESH_DIGEST' (tracebloc.controlPlaneDigest did not render the pin — client-runtime#199 revert not prevented)" From 9666cf079262c085fa4d3b38cbefe14c6f8f974b Mon Sep 17 00:00:00 2001 From: Syed Is Saqlain Date: Thu, 10 Sep 2026 05:07:13 +0400 Subject: [PATCH 6/8] docs(migrations): scope the .image-conflict note to the server-side path (client-runtime#199) The 1.9.110 .image-conflict-narrows note reads as if it covers all upgrades; it only applies to the hand-run helm --server-side --force-conflicts path. The automated fleet path is Helm 3.16 client-side (alpine/helm:3.16.4, no --server-side) whose 3-way merge overwrites rather than conflicting, so it never surfaces the conflict. Say so. Co-Authored-By: Claude Opus 4.8 --- docs/MIGRATIONS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md index e3732c91..fabd9e02 100644 --- a/docs/MIGRATIONS.md +++ b/docs/MIGRATIONS.md @@ -245,7 +245,7 @@ Two things to know before you force: 1. **`--force-conflicts` re-takes *every* field the chart renders that a non-Helm manager owns — not only `limits.cpu`.** Helm prints the full conflict list on the failed attempt; read it first. On a long-running fleet the list can include `image-refresh`'s pinned digests (`kubectl set image`); forcing reverts them to the chart's rendered image, and `image-refresh` re-pins on its next tick. That is recoverable but is a real, if brief, image churn — don't force blind. - **From chart 1.9.110 (#1013 / client-runtime#199) the `.image` conflict narrows.** The chart now renders the last-refreshed `@digest` itself (`tracebloc.controlPlaneDigest` reads image-refresh's annotation via `lookup`), so in the **steady state** Helm applies the same value the `kubectl-set` manager holds and the `.spec…containers[…].image` conflict disappears. A `.image` conflict on 1.9.110+ therefore signals **annotation-vs-live LAG** — the rollout-timeout / `helm rollback` window where the annotation trails the live spec — not steady-state drift. Wait one refresh tick for the reconcile to re-pin, then re-run; or `--force-conflicts` to take the chart's rendered (annotation) digest, accepting it may briefly trail until the next tick. + **From chart 1.9.110 (#1013 / client-runtime#199) the `.image` conflict narrows.** The chart now renders the last-refreshed `@digest` itself (`tracebloc.controlPlaneDigest` reads image-refresh's annotation via `lookup`), so in the **steady state** Helm applies the same value the `kubectl-set` manager holds and the `.spec…containers[…].image` conflict disappears. **This whole note is about the hand-run server-side path** (`helm upgrade --server-side=true --force-conflicts`) that reports a `.image` conflict at all — the automated fleet path is Helm 3.16 client-side (`alpine/helm:3.16.4`, no `--server-side`), whose 3-way merge **overwrites** the live value rather than conflicting, so it never surfaces here. On the server-side path, a `.image` conflict on 1.9.110+ therefore signals **annotation-vs-live LAG** — the rollout-timeout / `helm rollback` window where the annotation trails the live spec — not steady-state drift. Wait one refresh tick for the reconcile to re-pin, then re-run; or `--force-conflicts` to take the chart's rendered (annotation) digest, accepting it may briefly trail until the next tick. 2. **`--server-side=true` is not optional, even though Helm 4 defaults to it.** After a rollback (including the automatic one the `auto-upgrade` CronJob performs when it reads a `pending-upgrade` release as a wedge — `backend#2877`), the release's stored apply method can revert to client-side. Then `--force-conflicts` **alone** fails with: From 1c78acf133dbb175b908b10d8d2b818b40b1e1c2 Mon Sep 17 00:00:00 2001 From: Syed Saqlain Date: Thu, 10 Sep 2026 10:19:05 +0400 Subject: [PATCH 7/8] fix(chart): don't seed the init container, and surface lookup fail-open provenance (client-runtime#199) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses @shujaatTracebloc's review threads on #1013 (the B-scope ones). #2 (High) — the wait-for-mysql INIT container is no longer seeded from the last-refreshed annotation. A digest a node never pulled fails as Init:ImagePullBackOff (the pod never starts) on exactly the offline/side-loaded k3d / Docker-Desktop edge #569 protects, and `kubectl set image` never touches this container so the chart render would be its only, un-reconciled driver. The pin buys nothing on a MySQL TCP wait. The site keeps honouring an explicit operator pin (`images.jobsManager.digest`); it just no longer seeds the floating annotation. #3 (Medium) — `lookup` fails OPEN: Helm returns an empty map on every failure (RBAC denial, apiserver 5xx, client timeout, kubectl-less renderer), so a read failure was indistinguishable from a first install and would silently drop the pin to `:tag` (client-runtime#199, now non-deterministic and unsignalled). New `tracebloc.controlPlaneDigestSource` renders `tracebloc.io/controlplane-digest-source` on the jobs-manager Deployment — `values` | `annotation` | `tag` | `tag-lookup-failed` — so a monitored edge can alert on the fallback. On an upgrade the Deployment MUST exist, so `.Release.IsUpgrade` + an empty lookup ⇒ `tag-lookup-failed` (the anomaly). No hard `fail` (would break `helm diff upgrade`, where lookup is empty too); metadata-only, so it never rolls the pods. #5 (Medium, doc) — documented the env-scoping caveat next to the suspend note in controlPlaneDigest: a CLIENT_ENV/tag change at upgrade time is inert once a digest is seeded (tag dropped when a digest is present), permanently so while suspended. The containment (record the tag alongside the digest, honour the annotation only when it matches) is an A-side change, called out here per the review. Tests: control_plane_digest_test.yaml flips the init case (init stays :tag with a valid annotation present; still honours an operator pin) and adds the four digest-source cases incl. tag-lookup-failed on `.Release.IsUpgrade`. helm-unittest 712/712 across 41 suites; helm lint clean; helm template renders :prod + source `tag` clusterless and `tag-lookup-failed` under --is-upgrade. Mutation-checked: re-seeding the init reddens its guard. Not in this push (replied on-thread): #1 (BLOCKING, flap-lockout downgrade) and #5's actual fix are A-side changes (#1008); #4's misleading e2e comment is already fixed on-branch (a54deb0) and the explicit --atomic negative path is proposed as a CI-validated follow-up. Co-Authored-By: Claude Opus 4.8 --- client/templates/_helpers.tpl | 72 ++++++++++++ client/templates/jobs-manager-deployment.yaml | 25 +++- client/tests/control_plane_digest_test.yaml | 108 +++++++++++++++++- 3 files changed, 202 insertions(+), 3 deletions(-) diff --git a/client/templates/_helpers.tpl b/client/templates/_helpers.tpl index 330a0f82..29b9c39b 100644 --- a/client/templates/_helpers.tpl +++ b/client/templates/_helpers.tpl @@ -543,6 +543,21 @@ Always one, which still answers the newly-joined-node concern that first put a suspend gate here (@shujaatTracebloc's earlier BLOCKING-2 on #1013, reversed here). + ENV-SCOPING CAVEAT (@shujaatTracebloc #1013, Medium; fix lives in A, not here). + The `last-refreshed--digest` annotation key is NOT tag/env-scoped, and + `tracebloc.image` drops the tag once a digest is present, so a `CLIENT_ENV` / + tag change made AT upgrade time (e.g. `--set env.CLIENT_ENV=prod`, or the fleet + moving an edge between environments) is silently inert on the image: the render + pins the OLD env's `@digest` while every label reads the new env. It self-heals + on image-refresh's next tick (it resolves the new `:tag`, sees `recorded != + latest`, and `set image`s) — a window of up to `imageRefresh.schedule`, and + NEVER while `imageRefresh.suspend: true`, since suspend now keeps the pin. The + containment is an A-side change: image-refresh records the resolved tag + alongside the digest (`tracebloc.io/last-refreshed--tag`) so this helper + can honour the annotation only when that tag equals the currently resolved + `tracebloc.clientEnv`, falling back to `:tag` otherwise. Called out here because + suspend makes the mismatch permanent. + The lookup targets ONLY the jobs-manager Deployment (where image-refresh writes every last-refreshed annotation) in the release namespace — a read the auto-upgrade SA already holds (its release-ns Role grants all verbs on all @@ -579,6 +594,63 @@ Always {{- end -}} {{- end }} +{{/* + tracebloc.controlPlaneDigestSource — the PROVENANCE of what controlPlaneDigest + resolved, rendered onto the jobs-manager Deployment as + `tracebloc.io/controlplane-digest-source` so a monitored edge can SEE which + branch produced the image ref. It mirrors controlPlaneDigest's resolution + exactly and must stay in lockstep with it. Values: + + values — an operator `images..digest` pin (priority 1). + annotation — image-refresh's last-refreshed `@digest` (the pin is + preserved on this upgrade — the healthy steady state). + tag — the bare `:tag`. Legitimate here: first install, a + `global.imageRegistry` mirror, `imageRefresh.enabled: + false`, or the first-observation window (Deployment + present but not yet annotated). + tag-lookup-failed — the ANOMALY (@shujaatTracebloc #1013, Medium): `lookup` + fails OPEN — Helm returns an empty map on EVERY failure + (RBAC denial, apiserver 5xx, client timeout, a + kubectl-less renderer), not just NotFound — so a read + failure is otherwise indistinguishable from "first + install" and would silently drop the pin back to `:tag` + (client-runtime#199, now non-deterministic and unsignalled). + On an UPGRADE the jobs-manager Deployment MUST exist, so + `.Release.IsUpgrade` AND an empty lookup means the READ + failed, not that the release is new. We cannot recover + the digest from a read we could not do, and a hard `fail` + would break `helm diff upgrade` (lookup is empty there + too), so we SURFACE it instead: a monitored edge alerts + on this value. (It also shows in `helm diff upgrade` for + the same empty-lookup reason the digest churn does — the + diff caveat the header already documents; it is accurate + on a real apply.) + + Args: same as controlPlaneDigest. +*/}} +{{- define "tracebloc.controlPlaneDigestSource" -}} +{{- $mirror := (dig "imageRegistry" "docker.io" (.root.Values.global | default dict)) | default "docker.io" -}} +{{- if .operatorDigest -}} +values +{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") -}} +{{- $dep := lookup "apps/v1" "Deployment" .root.Release.Namespace (printf "%s-jobs-manager" (include "tracebloc.fullname" .root)) -}} +{{- if $dep -}} +{{- $ann := index (($dep.metadata).annotations | default dict) (printf "tracebloc.io/last-refreshed-%s-digest" .annotationImage) | default "" -}} +{{- if regexMatch "^sha256:[a-f0-9]{64}$" $ann -}} +annotation +{{- else -}} +tag +{{- end -}} +{{- else if .root.Release.IsUpgrade -}} +tag-lookup-failed +{{- else -}} +tag +{{- end -}} +{{- else -}} +tag +{{- end -}} +{{- end }} + {{/* StorageClass name: when storageClass.create is true, use a release-unique name so each release gets its own StorageClass (avoids Helm ownership conflicts). diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index d7a2b5d2..b3abf075 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -10,6 +10,16 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "tracebloc.labels" . | nindent 4 }} + annotations: + # PROVENANCE of the control-plane image ref this render resolved + # (client-runtime#199, @shujaatTracebloc on #1013). `lookup` fails OPEN, so a + # transient read failure would silently drop image-refresh's pin back to + # `:tag` with a green upgrade and no signal. This records which branch won — + # `values` | `annotation` | `tag` | `tag-lookup-failed` — so a monitored edge + # can alert on the fallback instead of discovering the #199 revert after the + # fact. Metadata-only: changing it does not roll the pods. See + # tracebloc.controlPlaneDigestSource. + tracebloc.io/controlplane-digest-source: {{ include "tracebloc.controlPlaneDigestSource" (dict "root" $ "operatorDigest" .Values.images.jobsManager.digest "annotationImage" "jobs-manager") | quote }} spec: selector: matchLabels: @@ -100,7 +110,20 @@ spec: # # It also removes an image: busybox was a second pull on every edge, for a # TCP connect the app image can already do. - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.jobsManager.digest "annotationImage" "jobs-manager")) "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + # + # NOT seeded from the last-refreshed annotation (client-runtime#199, + # @shujaatTracebloc on #1013): this is an INIT container, so a digest a + # node has never pulled fails as `Init:ImagePullBackOff` and the pod never + # reaches any container at all — on exactly the offline/side-loaded k3d / + # Docker-Desktop edge #569 was written to protect (a `k3d image import` + # stores a tag alias with no resolvable digest; see the #569 note at + # controlPlanePullPolicy). And `image-refresh` never reconciles this site: + # `kubectl set image` only names `api` and `pods-monitor-container`, so the + # chart render would be its ONLY driver, with no `set image` to move it + # forward. The pin buys nothing here anyway — it is a MySQL TCP wait on the + # same image. So keep the OPERATOR pin (`images.jobsManager.digest`) for an + # explicit, reproducible pin, but do NOT seed the floating annotation. + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} securityContext: # NO runAsUser, DELIBERATELY (Bugbot High, #942). OpenShift's # `restricted` SCC assigns a UID from the project's range, and the pod diff --git a/client/tests/control_plane_digest_test.yaml b/client/tests/control_plane_digest_test.yaml index de17959f..4e4d1b58 100644 --- a/client/tests/control_plane_digest_test.yaml +++ b/client/tests/control_plane_digest_test.yaml @@ -61,7 +61,13 @@ tests: path: 'spec.template.spec.containers[?(@.name=="api")].imagePullPolicy' value: IfNotPresent - - it: the wait-for-mysql init container renders the jobs-manager digest too + - it: the wait-for-mysql init container is NOT seeded from the annotation (floats on :tag) + # @shujaatTracebloc #1013 (High): the init container is deliberately NOT seeded + # from the last-refreshed annotation. A digest a node never pulled would fail as + # Init:ImagePullBackOff (pod never starts) on exactly the k3d/Docker-Desktop edge + # #569 protects, and `kubectl set image` never touches this container so the chart + # would be its only, un-reconciled driver. So even with a valid annotation present, + # this init site stays on :tag. template: templates/jobs-manager-deployment.yaml documentIndex: 0 kubernetesProvider: @@ -78,7 +84,20 @@ tests: asserts: - equal: path: 'spec.template.spec.initContainers[?(@.name=="wait-for-mysql")].image' - value: docker.io/tracebloc/jobs-manager@sha256:1111111111111111111111111111111111111111111111111111111111111111 + value: docker.io/tracebloc/jobs-manager:prod + + - it: the wait-for-mysql init container STILL honours an explicit operator pin + # Dropping the annotation seed must not drop the operator's own reproducible pin. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + set: + images: + jobsManager: + digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + asserts: + - equal: + path: 'spec.template.spec.initContainers[?(@.name=="wait-for-mysql")].image' + value: docker.io/tracebloc/jobs-manager@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - it: pods-monitor-container follows its OWN last-refreshed annotation key template: templates/jobs-manager-deployment.yaml @@ -286,3 +305,88 @@ tests: - equal: path: 'spec.template.spec.containers[?(@.name=="api")].image' value: docker.io/tracebloc/jobs-manager:prod + + # ── the digest-source provenance annotation (@shujaatTracebloc #1013) ──────── + # tracebloc.io/controlplane-digest-source records WHICH branch produced the + # image ref, so a `lookup` that fails open (silently dropping the pin to :tag) + # is observable instead of a silent client-runtime#199 recurrence. + - it: source is "annotation" when a valid last-refreshed digest is preserved + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'metadata.annotations["tracebloc.io/controlplane-digest-source"]' + value: annotation + + - it: source is "values" when an operator pins the digest + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + set: + images: + jobsManager: + digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + asserts: + - equal: + path: 'metadata.annotations["tracebloc.io/controlplane-digest-source"]' + value: values + + - it: source is "tag" when the Deployment exists but carries no valid annotation + # First-observation window: the Deployment is present but image-refresh has not + # written a digest yet. Legitimately :tag — not an anomaly. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + asserts: + - equal: + path: 'metadata.annotations["tracebloc.io/controlplane-digest-source"]' + value: tag + + - it: source is "tag-lookup-failed" when the lookup returns empty ON AN UPGRADE + # The anomaly: on an upgrade the jobs-manager Deployment MUST exist, so an empty + # lookup is a read failure (RBAC/5xx/timeout), NOT a fresh install. Surface it so + # a silent fall-back to :tag (client-runtime#199) is alertable. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + release: + upgrade: true + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: [] + asserts: + - equal: + path: 'metadata.annotations["tracebloc.io/controlplane-digest-source"]' + value: tag-lookup-failed + + - it: source is "tag" on a FIRST INSTALL with no Deployment (not an anomaly) + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + release: + upgrade: false + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: [] + asserts: + - equal: + path: 'metadata.annotations["tracebloc.io/controlplane-digest-source"]' + value: tag From 3b4dce6a99d24044eabfcf6b1ca95559358da8d8 Mon Sep 17 00:00:00 2001 From: Syed Saqlain Date: Thu, 10 Sep 2026 11:08:00 +0400 Subject: [PATCH 8/8] chore(chart): rebump to 1.9.111 so B sequences above A's 1.9.110 (client-runtime#199) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit develop took 1.9.109 (client#1017), and A (#1008) rebumped to 1.9.110, so B at 1.9.110 would collide with A on the release train — one version number covering two different charts, exactly what the version-bump guard exists to prevent (@shujaatTracebloc on #1008). Bump B one above A: version + appVersion to 1.9.111, and re-point the feature-landing version in the doc comments (_helpers.tpl, image-refresh CronJob header, MIGRATIONS.md, SEAL-CHECK.md, e2e-auto-upgrade.sh) from 1.9.110 to 1.9.111. helm lint clean; helm unittest 712/712 across 41 suites; shellcheck + bash -n clean on the edited e2e. A→B merge order unchanged. Co-Authored-By: Claude Opus 4.8 --- client/Chart.yaml | 4 ++-- client/templates/_helpers.tpl | 6 +++--- client/templates/image-refresh-cronjob.yaml | 8 ++++---- docs/MIGRATIONS.md | 2 +- docs/SEAL-CHECK.md | 2 +- replies/bugbot-flap.md | 1 + replies/bugbot-lookup.md | 1 + replies/bugbot-tag.md | 1 + replies/clientenv.md | 1 + replies/flap.md | 1 + replies/init.md | 1 + replies/lookup.md | 1 + scripts/tests/e2e-auto-upgrade.sh | 2 +- 13 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 replies/bugbot-flap.md create mode 100644 replies/bugbot-lookup.md create mode 100644 replies/bugbot-tag.md create mode 100644 replies/clientenv.md create mode 100644 replies/flap.md create mode 100644 replies/init.md create mode 100644 replies/lookup.md diff --git a/client/Chart.yaml b/client/Chart.yaml index de42aa09..15c1fd38 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.110 -appVersion: "1.9.110" +version: 1.9.111 +appVersion: "1.9.111" keywords: - tracebloc - kubernetes diff --git a/client/templates/_helpers.tpl b/client/templates/_helpers.tpl index 29b9c39b..2e7a1986 100644 --- a/client/templates/_helpers.tpl +++ b/client/templates/_helpers.tpl @@ -480,11 +480,11 @@ true freezing — a frozen control plane with no signal is worse than a restart that needs the network. - #569 OFFLINE GUARANTEE — NARROWED as of 1.9.110 (#1013). IfNotPresent only - helps when the REFERENCE is cache-satisfiable. Until 1.9.110 a `helm upgrade` + #569 OFFLINE GUARANTEE — NARROWED as of 1.9.111 (#1013). IfNotPresent only + helps when the REFERENCE is cache-satisfiable. Until 1.9.111 a `helm upgrade` re-rendered the bare `:tag`, which a node that had pulled that tag could always satisfy from cache, so an offline restart ran the (possibly stale) cached - image. From 1.9.110 tracebloc.controlPlaneDigest can render `repo@` + image. From 1.9.111 tracebloc.controlPlaneDigest can render `repo@` seeded from the last-refreshed annotation (see its header), and because the first-observation path RECORDS a digest without re-imaging, the rendered digest may be one the node has never pulled. Online that is the fix working; OFFLINE diff --git a/client/templates/image-refresh-cronjob.yaml b/client/templates/image-refresh-cronjob.yaml index 43ae5a01..2b157e19 100644 --- a/client/templates/image-refresh-cronjob.yaml +++ b/client/templates/image-refresh-cronjob.yaml @@ -48,12 +48,12 @@ data: # edge — the pods stay offline-safe throughout because IfNotPresent does not # depend on any of this. # - # 1. HELM RE-RENDER — RESOLVED as of chart 1.9.110 (#1013 / + # 1. HELM RE-RENDER — RESOLVED as of chart 1.9.111 (#1013 / # client-runtime#199). `helm upgrade --reset-then-reuse-values` (the - # fleet auto-upgrade path) re-renders the templates; BEFORE 1.9.110 they + # fleet auto-upgrade path) re-renders the templates; BEFORE 1.9.111 they # wrote the bare `repo:tag` and so reverted an earlier `set image` pin, # and the edge floated on the tag until the next upstream release - # re-pinned it. FROM 1.9.110 the control-plane image sites render + # re-pinned it. FROM 1.9.111 the control-plane image sites render # through `tracebloc.controlPlaneDigest`, which reads THIS annotation via # `lookup` and renders `repo@` — so a re-render # now PRESERVES the pin instead of dropping it. (Still only relevant on a @@ -105,7 +105,7 @@ data: # reproducible. Offline-safety is what #569 is fixing; reproducibility # follows on the next upstream release. # - # First-observation roll (chart ≥ 1.9.110, #1013). Once this tick has + # First-observation roll (chart ≥ 1.9.111, #1013). Once this tick has # RECORDED D0 without re-imaging, the live spec still says `:tag` but the # annotation now says `@D0`, so the NEXT chart bump renders `repo@D0` # (tracebloc.controlPlaneDigest reads the annotation) and rolls jobs-manager diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md index fabd9e02..89c7d127 100644 --- a/docs/MIGRATIONS.md +++ b/docs/MIGRATIONS.md @@ -245,7 +245,7 @@ Two things to know before you force: 1. **`--force-conflicts` re-takes *every* field the chart renders that a non-Helm manager owns — not only `limits.cpu`.** Helm prints the full conflict list on the failed attempt; read it first. On a long-running fleet the list can include `image-refresh`'s pinned digests (`kubectl set image`); forcing reverts them to the chart's rendered image, and `image-refresh` re-pins on its next tick. That is recoverable but is a real, if brief, image churn — don't force blind. - **From chart 1.9.110 (#1013 / client-runtime#199) the `.image` conflict narrows.** The chart now renders the last-refreshed `@digest` itself (`tracebloc.controlPlaneDigest` reads image-refresh's annotation via `lookup`), so in the **steady state** Helm applies the same value the `kubectl-set` manager holds and the `.spec…containers[…].image` conflict disappears. **This whole note is about the hand-run server-side path** (`helm upgrade --server-side=true --force-conflicts`) that reports a `.image` conflict at all — the automated fleet path is Helm 3.16 client-side (`alpine/helm:3.16.4`, no `--server-side`), whose 3-way merge **overwrites** the live value rather than conflicting, so it never surfaces here. On the server-side path, a `.image` conflict on 1.9.110+ therefore signals **annotation-vs-live LAG** — the rollout-timeout / `helm rollback` window where the annotation trails the live spec — not steady-state drift. Wait one refresh tick for the reconcile to re-pin, then re-run; or `--force-conflicts` to take the chart's rendered (annotation) digest, accepting it may briefly trail until the next tick. + **From chart 1.9.111 (#1013 / client-runtime#199) the `.image` conflict narrows.** The chart now renders the last-refreshed `@digest` itself (`tracebloc.controlPlaneDigest` reads image-refresh's annotation via `lookup`), so in the **steady state** Helm applies the same value the `kubectl-set` manager holds and the `.spec…containers[…].image` conflict disappears. **This whole note is about the hand-run server-side path** (`helm upgrade --server-side=true --force-conflicts`) that reports a `.image` conflict at all — the automated fleet path is Helm 3.16 client-side (`alpine/helm:3.16.4`, no `--server-side`), whose 3-way merge **overwrites** the live value rather than conflicting, so it never surfaces here. On the server-side path, a `.image` conflict on 1.9.111+ therefore signals **annotation-vs-live LAG** — the rollout-timeout / `helm rollback` window where the annotation trails the live spec — not steady-state drift. Wait one refresh tick for the reconcile to re-pin, then re-run; or `--force-conflicts` to take the chart's rendered (annotation) digest, accepting it may briefly trail until the next tick. 2. **`--server-side=true` is not optional, even though Helm 4 defaults to it.** After a rollback (including the automatic one the `auto-upgrade` CronJob performs when it reads a `pending-upgrade` release as a wedge — `backend#2877`), the release's stored apply method can revert to client-side. Then `--force-conflicts` **alone** fails with: diff --git a/docs/SEAL-CHECK.md b/docs/SEAL-CHECK.md index c12ead49..e47a6094 100644 --- a/docs/SEAL-CHECK.md +++ b/docs/SEAL-CHECK.md @@ -288,7 +288,7 @@ run a build carrying client-runtime#416 (the HF-offline injection) *before* the seal, or NLP templates fail by network block instead of the clean closed door. On each cluster the chart renders control-plane images as `repository:tag` + `IfNotPresent` on a fresh install, and the `image-refresh` CronJob pins the live -digest out-of-band. **As of chart 1.9.110 (#1013 / client-runtime#199) a +digest out-of-band. **As of chart 1.9.111 (#1013 / client-runtime#199) a `helm upgrade` no longer reverts that pin:** `tracebloc.controlPlaneDigest` reads image-refresh's last-refreshed annotation via `lookup` and re-renders `repository@digest`, so `--reset-then-reuse-values` preserves the digest on its diff --git a/replies/bugbot-flap.md b/replies/bugbot-flap.md new file mode 100644 index 00000000..3f80bb3e --- /dev/null +++ b/replies/bugbot-flap.md @@ -0,0 +1 @@ +Confirmed — same finding as @shujaatTracebloc's BLOCKING thread on this helper. The fix is A-side (#1008): write each workload's `last-refreshed-*` annotation right after its own `rollout status` and before the flap-guard `exit 0`, so the annotation can't lag the live spec into a backward move. B is held behind A; tracked there, leaving open. diff --git a/replies/bugbot-lookup.md b/replies/bugbot-lookup.md new file mode 100644 index 00000000..d415d27a --- /dev/null +++ b/replies/bugbot-lookup.md @@ -0,0 +1 @@ +Addressed in 1c78acf — `tracebloc.controlPlaneDigestSource` records `tracebloc.io/controlplane-digest-source` (`values`|`annotation`|`tag`|`tag-lookup-failed`) on the jobs-manager Deployment; on an upgrade an empty lookup ⇒ `tag-lookup-failed`, since the Deployment must exist. See the fuller reply on @shujaatTracebloc's thread. diff --git a/replies/bugbot-tag.md b/replies/bugbot-tag.md new file mode 100644 index 00000000..ea481aa3 --- /dev/null +++ b/replies/bugbot-tag.md @@ -0,0 +1 @@ +Documented in 1c78acf next to the suspend note; the actual fix (record the tag alongside the digest, honour the annotation only when it matches the current env) is an A-side change — see @shujaatTracebloc's thread. Leaving open until A lands. diff --git a/replies/clientenv.md b/replies/clientenv.md new file mode 100644 index 00000000..38f5d23f --- /dev/null +++ b/replies/clientenv.md @@ -0,0 +1 @@ +Documented in 1c78acf next to the suspend note in `controlPlaneDigest` (your "at minimum, call it out"). The actual containment — image-refresh recording the resolved tag alongside the digest (`tracebloc.io/last-refreshed--tag`) so the helper honours the annotation only when it equals the current `tracebloc.clientEnv`, else `:tag` — is an **A-side** change (image-refresh writes that annotation), so I'm leaving this thread open until A carries it; the B-side "honour only if tag matches" lands on top of that. Flagging it for #1008. diff --git a/replies/flap.md b/replies/flap.md new file mode 100644 index 00000000..90a3dedb --- /dev/null +++ b/replies/flap.md @@ -0,0 +1 @@ +Real, and you're right that #1008 as written doesn't heal it — its live-spec reconcile sits in the `recorded == latest` branch, which the flap-lockout state never reaches. This is a **BLOCKING precondition on A (#1008)**, not something B can fix alone, so per that I'm leaving this open and flagging it on #1008: the robust fix is A-side — write each image's `last-refreshed-*` annotation immediately after *that workload's own* `rollout status` (not one batched annotate after all three), and write it **before** the flap-guard's `exit 0`, so the annotation can never lag the live spec into a chart-enforced backward move. Your own 9666cf0 note confirms the mechanism: the fleet path is Helm-3 client-side, whose 3-way merge overwrites live `@D1` with the rendered `@D0` rather than conflicting. B stays held behind A until A covers this. (The `tag-lookup-failed` signal I added for the lookup thread makes a `:tag` fall-back observable, but not this backward-move — that needs the A-side ordering fix.) diff --git a/replies/init.md b/replies/init.md new file mode 100644 index 00000000..f1da33c3 --- /dev/null +++ b/replies/init.md @@ -0,0 +1 @@ +Fixed in 1c78acf. The `wait-for-mysql` init container no longer seeds from the annotation — it renders `:tag` (or an explicit `images.jobsManager.digest` operator pin) via `tracebloc.image` directly. Your reasoning is exactly right: it's an init container so an unpullable digest is `Init:ImagePullBackOff` (pod never starts) on the very k3d/Docker-Desktop edge #569 protects, and `kubectl set image` never names it so the chart would be its only un-reconciled driver. `control_plane_digest_test.yaml` flips the case (init stays `:tag` with a valid annotation present; still honours an operator pin), mutation-checked: re-seeding the init reddens that guard. diff --git a/replies/lookup.md b/replies/lookup.md new file mode 100644 index 00000000..0152ffcc --- /dev/null +++ b/replies/lookup.md @@ -0,0 +1 @@ +Addressed in 1c78acf, as a recorded signal rather than a hard `fail` (your steer — a `fail` would break `helm diff upgrade`, where lookup is empty too). New `tracebloc.controlPlaneDigestSource` renders `tracebloc.io/controlplane-digest-source` on the jobs-manager Deployment: `values` | `annotation` | `tag` | `tag-lookup-failed`. The key distinction you named — on an **upgrade** the jobs-manager Deployment MUST exist, so `.Release.IsUpgrade` + an empty lookup ⇒ a **read failure**, not a fresh install — is rendered as `tag-lookup-failed`, so a monitored edge can alert on the silent fall-back instead of discovering the #199 revert after the fact. Metadata-only (no pod roll). Covered clusterless in `control_plane_digest_test.yaml` incl. the `tag-lookup-failed`-on-upgrade case; `helm template --is-upgrade` renders it, plain render stays `tag`. diff --git a/scripts/tests/e2e-auto-upgrade.sh b/scripts/tests/e2e-auto-upgrade.sh index 4a634f08..a2775b8d 100755 --- a/scripts/tests/e2e-auto-upgrade.sh +++ b/scripts/tests/e2e-auto-upgrade.sh @@ -243,7 +243,7 @@ echo "── simulate an image-refresh-managed annotation (must survive upgrades # onto an unpullable ref and it sits in Init:ImagePullBackOff. (The intermediate # reset to the PUBLISHED $PREV chart between paths 1 and 2 MAY briefly un-wedge it # by rendering `:tag` — but only while $PREV predates tracebloc.controlPlaneDigest; -# once 1.9.110+ is the published $PREV the reset renders this sentinel too and the +# once 1.9.111+ is the published $PREV the reset renders this sentinel too and the # un-wedge stops happening. Do not depend on it either way.) That # is harmless to paths 2-5 BY CONSTRUCTION regardless: the only readiness wait in this script # is `kubectl wait … nodes` at the top (before this seed), and every assertion