diff --git a/client/Chart.yaml b/client/Chart.yaml index de42aa09..15c1fd38 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.110 -appVersion: "1.9.110" +version: 1.9.111 +appVersion: "1.9.111" keywords: - tracebloc - kubernetes diff --git a/client/templates/_helpers.tpl b/client/templates/_helpers.tpl index ed7d5292..2e7a1986 100644 --- a/client/templates/_helpers.tpl +++ b/client/templates/_helpers.tpl @@ -480,6 +480,20 @@ true freezing — a frozen control plane with no signal is worse than a restart that needs the network. + #569 OFFLINE GUARANTEE — NARROWED as of 1.9.111 (#1013). IfNotPresent only + helps when the REFERENCE is cache-satisfiable. Until 1.9.111 a `helm upgrade` + re-rendered the bare `:tag`, which a node that had pulled that tag could always + satisfy from cache, so an offline restart ran the (possibly stale) cached + image. From 1.9.111 tracebloc.controlPlaneDigest can render `repo@` + seeded from the last-refreshed annotation (see its header), and because the + first-observation path RECORDS a digest without re-imaging, the rendered digest + may be one the node has never pulled. Online that is the fix working; OFFLINE + it turns "restarts on a stale image" into "cannot start" (ImagePullBackOff) for + that one digest. Two corollaries: a `global.imageRegistry` mirror is exempt + (branch 2 stays inert there → `:tag`), and a side-loaded k3d image is affected + because `k3d image import` stores a tag alias with no resolvable digest and a + local k3d install IS docker.io, so the mirror gate does not help it. + Usage: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" $d "root" $) }} */}} {{- define "tracebloc.controlPlanePullPolicy" -}} @@ -493,6 +507,150 @@ Always {{- end -}} {{- end }} +{{/* + tracebloc.controlPlaneDigest — the EFFECTIVE digest for a control-plane image, + resolved in priority order: + + 1. an operator's explicit values pin (`images..digest`) — wins, + behaviour unchanged; + 2. else, when image-refresh is the update path (enabled AND the docker.io + mirror), the digest image-refresh last OBSERVED, read from the + jobs-manager Deployment's `tracebloc.io/last-refreshed--digest` + annotation via `lookup`. ("Observed", not "applied": on the FIRST tick + image-refresh records the current digest WITHOUT `set image`, deliberately + — see the image-refresh CronJob header — so the annotation is what refresh + last saw, which is only the same as what it last applied once a real + refresh has run.) This is what makes a `helm upgrade` RENDER `repo@digest` + instead of reverting to the bare `:tag` and dropping image-refresh's + out-of-band `kubectl set image` pin — the revert that let a stale-`:tag` + node cache silently run an OLD control-plane image (client-runtime#199). + `lookup` returns empty during `helm template` / `helm diff` / the FIRST + install (no Deployment yet), so it degrades to `""` → `:tag` there, which + is correct: nothing is pinned yet and the node's fresh tag pull is the + right image. + 3. else `""` (bare `:tag`; `Always` via controlPlanePullPolicy is then the + update path — the non-refresh / mirror edges #569 protects). + + On `imageRefresh.suspend: true` the pin is KEPT, not dropped. suspend stops the + CronJob from POLLING; it does not un-pin (@LukasWodka on #1013). The knob is + used to FREEZE an edge during an incident, and `values.schema.json` promises it + "pause[s] without removing the resources" — un-pinning here would re-render + jobs-manager (Recreate), requests-proxy and the resource-monitor DaemonSet onto + the floating `:tag` + `Always`, three unplanned rollouts plus a downgrade to + whatever `:tag` points at now, which is the opposite of freezing. Keeping the + last-observed `@digest` freezes the edge on a real, previously-resolved image; + a newly joined node then pulls that frozen digest (a valid ref), not a missing + one, which still answers the newly-joined-node concern that first put a suspend + gate here (@shujaatTracebloc's earlier BLOCKING-2 on #1013, reversed here). + + ENV-SCOPING CAVEAT (@shujaatTracebloc #1013, Medium; fix lives in A, not here). + The `last-refreshed--digest` annotation key is NOT tag/env-scoped, and + `tracebloc.image` drops the tag once a digest is present, so a `CLIENT_ENV` / + tag change made AT upgrade time (e.g. `--set env.CLIENT_ENV=prod`, or the fleet + moving an edge between environments) is silently inert on the image: the render + pins the OLD env's `@digest` while every label reads the new env. It self-heals + on image-refresh's next tick (it resolves the new `:tag`, sees `recorded != + latest`, and `set image`s) — a window of up to `imageRefresh.schedule`, and + NEVER while `imageRefresh.suspend: true`, since suspend now keeps the pin. The + containment is an A-side change: image-refresh records the resolved tag + alongside the digest (`tracebloc.io/last-refreshed--tag`) so this helper + can honour the annotation only when that tag equals the currently resolved + `tracebloc.clientEnv`, falling back to `:tag` otherwise. Called out here because + suspend makes the mismatch permanent. + + The lookup targets ONLY the jobs-manager Deployment (where image-refresh writes + every last-refreshed annotation) in the release namespace — a read the + auto-upgrade SA already holds (its release-ns Role grants all verbs on all + resources in that namespace), so this adds no RBAC and does not hit the + backend#2469 bootstrap lockout. + + Args: (dict "root" $ "operatorDigest" + "annotationImage" <"jobs-manager"|"pods-monitor"|"resource-monitor">) +*/}} +{{- define "tracebloc.controlPlaneDigest" -}} +{{- $mirror := (dig "imageRegistry" "docker.io" (.root.Values.global | default dict)) | default "docker.io" -}} +{{- if .operatorDigest -}} +{{- .operatorDigest -}} +{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") -}} +{{- $dep := lookup "apps/v1" "Deployment" .root.Release.Namespace (printf "%s-jobs-manager" (include "tracebloc.fullname" .root)) -}} +{{- if $dep -}} +{{- $ann := index (($dep.metadata).annotations | default dict) (printf "tracebloc.io/last-refreshed-%s-digest" .annotationImage) | default "" -}} +{{/* + VALIDATE before rendering: the 11 values digest keys are schema-guarded by + `^(sha256:[a-f0-9]{64})?$`, but this annotation is written out-of-band by + image-refresh (kubectl) and reaches an `image:` field unchecked. A malformed + value (`tracebloc.image` drops the tag when a digest is present) renders an + unstartable ref helm cannot detect — the kubelet reports InvalidImageName + while the apiserver accepts the spec. Require a full sha256 digest; anything + else degrades to `""` → `:tag`, the safe fallback (@shujaatTracebloc on #1013). + `suspend` is deliberately NOT gated here: a suspended edge keeps rendering the + last-observed digest so `helm upgrade` freezes it in place rather than rolling + it onto the floating tag (see the header note). +*/}} +{{- if regexMatch "^sha256:[a-f0-9]{64}$" $ann -}} +{{- $ann -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end }} + +{{/* + tracebloc.controlPlaneDigestSource — the PROVENANCE of what controlPlaneDigest + resolved, rendered onto the jobs-manager Deployment as + `tracebloc.io/controlplane-digest-source` so a monitored edge can SEE which + branch produced the image ref. It mirrors controlPlaneDigest's resolution + exactly and must stay in lockstep with it. Values: + + values — an operator `images..digest` pin (priority 1). + annotation — image-refresh's last-refreshed `@digest` (the pin is + preserved on this upgrade — the healthy steady state). + tag — the bare `:tag`. Legitimate here: first install, a + `global.imageRegistry` mirror, `imageRefresh.enabled: + false`, or the first-observation window (Deployment + present but not yet annotated). + tag-lookup-failed — the ANOMALY (@shujaatTracebloc #1013, Medium): `lookup` + fails OPEN — Helm returns an empty map on EVERY failure + (RBAC denial, apiserver 5xx, client timeout, a + kubectl-less renderer), not just NotFound — so a read + failure is otherwise indistinguishable from "first + install" and would silently drop the pin back to `:tag` + (client-runtime#199, now non-deterministic and unsignalled). + On an UPGRADE the jobs-manager Deployment MUST exist, so + `.Release.IsUpgrade` AND an empty lookup means the READ + failed, not that the release is new. We cannot recover + the digest from a read we could not do, and a hard `fail` + would break `helm diff upgrade` (lookup is empty there + too), so we SURFACE it instead: a monitored edge alerts + on this value. (It also shows in `helm diff upgrade` for + the same empty-lookup reason the digest churn does — the + diff caveat the header already documents; it is accurate + on a real apply.) + + Args: same as controlPlaneDigest. +*/}} +{{- define "tracebloc.controlPlaneDigestSource" -}} +{{- $mirror := (dig "imageRegistry" "docker.io" (.root.Values.global | default dict)) | default "docker.io" -}} +{{- if .operatorDigest -}} +values +{{- else if and (include "tracebloc.imageRefreshEnabled" .root) (eq $mirror "docker.io") -}} +{{- $dep := lookup "apps/v1" "Deployment" .root.Release.Namespace (printf "%s-jobs-manager" (include "tracebloc.fullname" .root)) -}} +{{- if $dep -}} +{{- $ann := index (($dep.metadata).annotations | default dict) (printf "tracebloc.io/last-refreshed-%s-digest" .annotationImage) | default "" -}} +{{- if regexMatch "^sha256:[a-f0-9]{64}$" $ann -}} +annotation +{{- else -}} +tag +{{- end -}} +{{- else if .root.Release.IsUpgrade -}} +tag-lookup-failed +{{- else -}} +tag +{{- end -}} +{{- else -}} +tag +{{- end -}} +{{- end }} + {{/* StorageClass name: when storageClass.create is true, use a release-unique name so each release gets its own StorageClass (avoids Helm ownership conflicts). diff --git a/client/templates/image-refresh-cronjob.yaml b/client/templates/image-refresh-cronjob.yaml index d664eab1..fe4313ba 100644 --- a/client/templates/image-refresh-cronjob.yaml +++ b/client/templates/image-refresh-cronjob.yaml @@ -54,17 +54,23 @@ data: # in the same tick); it does not detect and repair skew that predates # it, because it compares registry-vs-annotation, never pod-vs-pod. # - # HELM RE-RENDER, by contrast, is HANDLED rather than tolerated (client-runtime#199). - # `helm upgrade --reset-then-reuse-values` (the fleet auto-upgrade path) - # re-renders the templates, which write `repo:tag` and so revert an earlier - # `set image` pin. `recorded == latest` no longer means no-op: the loop reads - # each workload's LIVE image and re-pins the digest whenever the workload is - # off it. The revert therefore lasts ONE tick — the next tick puts the digest - # back — instead of floating on the tag until the next upstream release. It - # is compared on the @sha256 digest, so a registry-prefix rewrite (a mutating - # webhook) is not mistaken for a revert. This only happens on a chart VERSION - # bump anyway — auto-upgrade compares versions and skips otherwise — so it is - # not an hourly revert. + # HELM RE-RENDER, by contrast, is HANDLED rather than tolerated (client-runtime#199), + # from BOTH sides now. Chart-side (#1013, from 1.9.111): the control-plane + # image sites render through `tracebloc.controlPlaneDigest`, which reads THIS + # annotation via `lookup`, so `helm upgrade --reset-then-reuse-values` (the + # fleet auto-upgrade path) re-renders `repo@` and no + # longer writes the bare `repo:tag` that used to revert an earlier `set image` + # pin. Script-side (#1008): `recorded == latest` no longer means no-op — the + # loop reads each workload's LIVE image and re-pins the digest whenever the + # workload is off it, so any residual revert (an annotation that lags the live + # spec — a timed-out rollout, `helm rollback`) lasts ONE tick instead of + # floating on the tag until the next upstream release. Compared on the @sha256 + # digest, so a registry-prefix rewrite (a mutating webhook) is not mistaken + # for a revert. This only bites on a chart VERSION bump anyway — auto-upgrade + # compares versions and skips otherwise — so it is not an hourly revert. (One + # residue the script-side re-pin does not yet cover is the durable flap-lockout + # state, where the annotation stays stuck while the live spec has moved on — + # tracked as a follow-up; see the first-tick contract below.) # # Reconciling against the live spec is what makes that possible — the # declarative reconcile `set image` enables (`rollout restart` was a blind @@ -118,6 +124,19 @@ data: # price, and a follow-up may gate the re-pin on "have we ever applied a digest # here?" so a genuine fresh install skips the flap path entirely. # + # First-observation roll (chart ≥ 1.9.111, #1013). Once this tick has + # RECORDED D0 without re-imaging, the live spec still says `:tag` but the + # annotation now says `@D0`, so the NEXT chart bump renders `repo@D0` + # (tracebloc.controlPlaneDigest reads the annotation) and rolls jobs-manager + # (Recreate), requests-proxy and the resource-monitor DaemonSet ONCE, for + # byte-identical content. auto-upgrade runs that upgrade `--atomic --wait` + # (auto-upgrade-cronjob.yaml), so on a slow single-node edge the one-time + # roll can hit the timeout and roll back, then re-attempt identically on the + # next bump. This is the deliberate tradeoff for NOT `set image`-ing on the + # first tick; pinning on the first tick instead would move the same roll to + # install time for every edge. Also narrows #569's offline guarantee — see + # the note at controlPlanePullPolicy in _helpers.tpl. + # # Parsing: awk/sed/grep + jq. jq used only where JSON-with-dotted-keys # or container/env-array filtering motivates it; the rest stays in pure # shell so the script survives image swaps to leaner bases — same diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index 81007109..b3abf075 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -10,6 +10,16 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "tracebloc.labels" . | nindent 4 }} + annotations: + # PROVENANCE of the control-plane image ref this render resolved + # (client-runtime#199, @shujaatTracebloc on #1013). `lookup` fails OPEN, so a + # transient read failure would silently drop image-refresh's pin back to + # `:tag` with a green upgrade and no signal. This records which branch won — + # `values` | `annotation` | `tag` | `tag-lookup-failed` — so a monitored edge + # can alert on the fallback instead of discovering the #199 revert after the + # fact. Metadata-only: changing it does not roll the pods. See + # tracebloc.controlPlaneDigestSource. + tracebloc.io/controlplane-digest-source: {{ include "tracebloc.controlPlaneDigestSource" (dict "root" $ "operatorDigest" .Values.images.jobsManager.digest "annotationImage" "jobs-manager") | quote }} spec: selector: matchLabels: @@ -100,6 +110,19 @@ spec: # # It also removes an image: busybox was a second pull on every edge, for a # TCP connect the app image can already do. + # + # NOT seeded from the last-refreshed annotation (client-runtime#199, + # @shujaatTracebloc on #1013): this is an INIT container, so a digest a + # node has never pulled fails as `Init:ImagePullBackOff` and the pod never + # reaches any container at all — on exactly the offline/side-loaded k3d / + # Docker-Desktop edge #569 was written to protect (a `k3d image import` + # stores a tag alias with no resolvable digest; see the #569 note at + # controlPlanePullPolicy). And `image-refresh` never reconciles this site: + # `kubectl set image` only names `api` and `pods-monitor-container`, so the + # chart render would be its ONLY driver, with no `set image` to move it + # forward. The pin buys nothing here anyway — it is a MySQL TCP wait on the + # same image. So keep the OPERATOR pin (`images.jobsManager.digest`) for an + # explicit, reproducible pin, but do NOT seed the floating annotation. image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} securityContext: # NO runAsUser, DELIBERATELY (Bugbot High, #942). OpenShift's @@ -265,7 +288,7 @@ spec: {{- end }} containers: - name: api - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.jobsManager.digest "annotationImage" "jobs-manager")) "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} # #569: IfNotPresent wherever an update path exists without `Always` -- # a pinned digest, or the image-refresh reconcile. `Always` forced a # registry round-trip on every (re)start, so an offline Docker Desktop / @@ -274,7 +297,7 @@ spec: # is NOT unconditional: on an edge the reconcile cannot reach (private # mirror, or imageRefresh disabled) a floating tag plus a restart is the # only update path there is, so it stays Always. - imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" .Values.images.jobsManager.digest "root" $) }} + imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.jobsManager.digest "annotationImage" "jobs-manager")) "root" $) }} securityContext: allowPrivilegeEscalation: false capabilities: @@ -906,10 +929,10 @@ spec: {{- end }} {{- end }} - name: pods-monitor-container - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/pods-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.podsMonitor.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + image: {{ include "tracebloc.image" (dict "repository" "tracebloc/pods-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.podsMonitor.digest "annotationImage" "pods-monitor")) "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} # #569: same policy as the api container above -- see # tracebloc.controlPlanePullPolicy. - imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" .Values.images.podsMonitor.digest "root" $) }} + imagePullPolicy: {{ include "tracebloc.controlPlanePullPolicy" (dict "digest" (include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" .Values.images.podsMonitor.digest "annotationImage" "pods-monitor")) "root" $) }} securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/client/templates/requests-proxy-deployment.yaml b/client/templates/requests-proxy-deployment.yaml index 0ac6505d..0cb1db2a 100644 --- a/client/templates/requests-proxy-deployment.yaml +++ b/client/templates/requests-proxy-deployment.yaml @@ -41,6 +41,11 @@ spec: these keys. */}} {{- $rpDigest := (dig "requestsProxy" "digest" "" $rpImages) | default (dig "jobsManager" "digest" "" $rpImages) }} + {{- /* client-runtime#199: seed the digest image-refresh last observed so a + `helm upgrade` renders repo@digest, not the bare :tag (see + tracebloc.controlPlaneDigest). requests-proxy runs the SAME + jobs-manager image and follows its annotation. */}} + {{- $rpDigest = include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" $rpDigest "annotationImage" "jobs-manager") }} image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" $rpDigest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} # #569: IfNotPresent wherever an update path exists without `Always` -- # a pinned digest, or the image-refresh reconcile (see diff --git a/client/templates/resource-monitor-daemonset.yaml b/client/templates/resource-monitor-daemonset.yaml index fca159ff..814c8dc9 100644 --- a/client/templates/resource-monitor-daemonset.yaml +++ b/client/templates/resource-monitor-daemonset.yaml @@ -136,6 +136,12 @@ spec: entry). `dig` is not usable here — it rejects chartutil.Values. */}} {{- $rmDigest := (default (dict) (default (dict) .Values.images).resourceMonitor).digest | default "" }} + {{- /* client-runtime#199: seed the digest image-refresh last observed so a + `helm upgrade` renders repo@digest, not the bare :tag (see + tracebloc.controlPlaneDigest). The last-refreshed-resource-monitor-digest + annotation lives on the jobs-manager Deployment in the release + namespace, which the helper reads. */}} + {{- $rmDigest = include "tracebloc.controlPlaneDigest" (dict "root" $ "operatorDigest" $rmDigest "annotationImage" "resource-monitor") }} image: {{ include "tracebloc.image" (dict "repository" "tracebloc/resource-monitor" "tag" (include "tracebloc.clientEnv" .) "digest" $rmDigest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} # #569: IfNotPresent wherever an update path exists without `Always` -- # see tracebloc.controlPlanePullPolicy. `Always` made every node's pod diff --git a/client/tests/control_plane_digest_test.yaml b/client/tests/control_plane_digest_test.yaml new file mode 100644 index 00000000..4e4d1b58 --- /dev/null +++ b/client/tests/control_plane_digest_test.yaml @@ -0,0 +1,392 @@ +suite: Control-plane digest seeding (tracebloc.controlPlaneDigest) +# client-runtime#199 (@LukasWodka on #1013): a `helm upgrade` used to re-render +# the bare `repo:tag` for every control-plane image and so DROP the digest that +# image-refresh had pinned out-of-band with `kubectl set image repo@digest`; a +# stale-`:tag` node cache then silently ran an OLD control-plane build. +# tracebloc.controlPlaneDigest closes that by reading the digest image-refresh +# last OBSERVED off the jobs-manager Deployment's +# `tracebloc.io/last-refreshed--digest` annotation via `lookup`, so the +# upgrade renders `repo@digest` and preserves the pin. +# +# The whole point of the PR is that lookup path, and it was previously claimed to +# be untestable without a cluster. It is NOT: helm-unittest 0.5.2 (the version CI +# pins) fakes `lookup` via `kubernetesProvider`, so the annotation seed, the +# operator-pin precedence, the malformed-value guard and the suspend decision are +# all covered clusterless here. `Helm unit tests` is a REQUIRED check on develop; +# the k3d e2e that also exercises this path is path-filtered and NOT required, so +# without this suite a regression of the seed could merge green. +templates: + - templates/jobs-manager-deployment.yaml + - templates/requests-proxy-deployment.yaml + - templates/resource-monitor-daemonset.yaml +set: + clientId: "test-id" + clientPassword: "test" + dockerRegistry: + server: https://index.docker.io/v1/ + username: test + password: test + email: test@test.com +# Under helm-unittest .Release.Name is RELEASE-NAME and .Release.Namespace is +# NAMESPACE, so tracebloc.fullname is RELEASE-NAME and the helper looks up +# RELEASE-NAME-jobs-manager in NAMESPACE. Distinct digests per key prove each +# site reads its OWN annotation rather than a shared value. +tests: + # ── the five image sites render their last-observed digest ────────────────── + - it: api renders the last-refreshed jobs-manager digest (the lookup seed) + # FIRST case deliberately: this is the one that reddens if the helper is + # neutered (e.g. the lookup result discarded). It is the clusterless proof + # that branch 2 works, which the k3d e2e used to be the only witness of. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + tracebloc.io/last-refreshed-pods-monitor-digest: "sha256:2222222222222222222222222222222222222222222222222222222222222222" + tracebloc.io/last-refreshed-resource-monitor-digest: "sha256:3333333333333333333333333333333333333333333333333333333333333333" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: docker.io/tracebloc/jobs-manager@sha256:1111111111111111111111111111111111111111111111111111111111111111 + # #569: a rendered digest is immutable, so the pull policy stays IfNotPresent. + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].imagePullPolicy' + value: IfNotPresent + + - it: the wait-for-mysql init container is NOT seeded from the annotation (floats on :tag) + # @shujaatTracebloc #1013 (High): the init container is deliberately NOT seeded + # from the last-refreshed annotation. A digest a node never pulled would fail as + # Init:ImagePullBackOff (pod never starts) on exactly the k3d/Docker-Desktop edge + # #569 protects, and `kubectl set image` never touches this container so the chart + # would be its only, un-reconciled driver. So even with a valid annotation present, + # this init site stays on :tag. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'spec.template.spec.initContainers[?(@.name=="wait-for-mysql")].image' + value: docker.io/tracebloc/jobs-manager:prod + + - it: the wait-for-mysql init container STILL honours an explicit operator pin + # Dropping the annotation seed must not drop the operator's own reproducible pin. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + set: + images: + jobsManager: + digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + asserts: + - equal: + path: 'spec.template.spec.initContainers[?(@.name=="wait-for-mysql")].image' + value: docker.io/tracebloc/jobs-manager@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + + - it: pods-monitor-container follows its OWN last-refreshed annotation key + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + tracebloc.io/last-refreshed-pods-monitor-digest: "sha256:2222222222222222222222222222222222222222222222222222222222222222" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="pods-monitor-container")].image' + value: docker.io/tracebloc/pods-monitor@sha256:2222222222222222222222222222222222222222222222222222222222222222 + + - it: requests-proxy runs the jobs-manager image and follows the jobs-manager digest + template: templates/requests-proxy-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="proxy")].image' + value: docker.io/tracebloc/jobs-manager@sha256:1111111111111111111111111111111111111111111111111111111111111111 + + - it: resource-monitor DaemonSet renders its own last-refreshed digest + template: templates/resource-monitor-daemonset.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-resource-monitor-digest: "sha256:3333333333333333333333333333333333333333333333333333333333333333" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="tracebloc-resource-monitor")].image' + value: docker.io/tracebloc/resource-monitor@sha256:3333333333333333333333333333333333333333333333333333333333333333 + + # ── operator pin beats the annotation ─────────────────────────────────────── + - it: an operator values pin wins over the last-refreshed annotation + # images..digest is priority 1 in the helper: an operator who pins a + # digest gets exactly that, whatever image-refresh last observed. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + set: + images: + jobsManager: + digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: docker.io/tracebloc/jobs-manager@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + + # ── malformed annotation degrades to the floating :tag ────────────────────── + - it: a malformed annotation degrades to the floating :tag (validation guard) + # The annotation is written out-of-band by kubectl and reaches image: unchecked; + # tracebloc.image drops the tag when a digest is present, so a malformed value + # would render an unstartable ref (kubelet InvalidImageName). The helper's + # regexMatch against ^sha256:[a-f0-9]{64}$ rejects it and falls back to :tag. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:not-a-real-digest" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: docker.io/tracebloc/jobs-manager:prod + + # ── the suspend decision: suspend KEEPS the pin (@LukasWodka on #1013) ─────── + - it: a suspended image-refresh KEEPS the last-observed digest (does not un-pin) + # suspend stops the CronJob POLLING; it must not un-pin. The knob freezes an + # edge during an incident and values.schema.json promises it pauses "without + # removing the resources", so re-rendering the floating :tag here would roll + # jobs-manager (Recreate), requests-proxy and the resource-monitor DaemonSet + # and downgrade to whatever :tag points at now — the opposite of freezing. + # (This reverses an earlier gate that made suspend render :tag + Always; a + # newly joined node still gets a valid, previously-resolved digest to pull.) + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + set: + imageRefresh: + suspend: true + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: docker.io/tracebloc/jobs-manager@sha256:1111111111111111111111111111111111111111111111111111111111111111 + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].imagePullPolicy' + value: IfNotPresent + + # ── the non-refresh / mirror edges keep the floating :tag ─────────────────── + - it: imageRefresh disabled ignores the annotation and floats on :tag + # Branch 2 is gated on the refresh path being live; with it off the annotation + # is not the update path, so the edge floats on :tag (+ Always via pull policy). + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + set: + imageRefresh: + enabled: false + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: docker.io/tracebloc/jobs-manager:prod + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].imagePullPolicy' + value: Always + + - it: a private mirror ignores the annotation and floats on :tag + # The reconcile resolves digests from docker.io, so under a mirror it goes + # inert by design and the helper must not pin a digest the mirror may not hold. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + set: + global: + imageRegistry: "mirror.example.com" + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: mirror.example.com/tracebloc/jobs-manager:prod + + - it: no jobs-manager Deployment yet (first install) floats on :tag + # lookup is empty on the FIRST install / helm template / helm diff, so the + # helper degrades to "" -> :tag. Modelled here by an empty object set. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: [] + asserts: + - equal: + path: 'spec.template.spec.containers[?(@.name=="api")].image' + value: docker.io/tracebloc/jobs-manager:prod + + # ── the digest-source provenance annotation (@shujaatTracebloc #1013) ──────── + # tracebloc.io/controlplane-digest-source records WHICH branch produced the + # image ref, so a `lookup` that fails open (silently dropping the pin to :tag) + # is observable instead of a silent client-runtime#199 recurrence. + - it: source is "annotation" when a valid last-refreshed digest is preserved + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + annotations: + tracebloc.io/last-refreshed-jobs-manager-digest: "sha256:1111111111111111111111111111111111111111111111111111111111111111" + asserts: + - equal: + path: 'metadata.annotations["tracebloc.io/controlplane-digest-source"]' + value: annotation + + - it: source is "values" when an operator pins the digest + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + set: + images: + jobsManager: + digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + asserts: + - equal: + path: 'metadata.annotations["tracebloc.io/controlplane-digest-source"]' + value: values + + - it: source is "tag" when the Deployment exists but carries no valid annotation + # First-observation window: the Deployment is present but image-refresh has not + # written a digest yet. Legitimately :tag — not an anomaly. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: RELEASE-NAME-jobs-manager + namespace: NAMESPACE + asserts: + - equal: + path: 'metadata.annotations["tracebloc.io/controlplane-digest-source"]' + value: tag + + - it: source is "tag-lookup-failed" when the lookup returns empty ON AN UPGRADE + # The anomaly: on an upgrade the jobs-manager Deployment MUST exist, so an empty + # lookup is a read failure (RBAC/5xx/timeout), NOT a fresh install. Surface it so + # a silent fall-back to :tag (client-runtime#199) is alertable. + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + release: + upgrade: true + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: [] + asserts: + - equal: + path: 'metadata.annotations["tracebloc.io/controlplane-digest-source"]' + value: tag-lookup-failed + + - it: source is "tag" on a FIRST INSTALL with no Deployment (not an anomaly) + template: templates/jobs-manager-deployment.yaml + documentIndex: 0 + release: + upgrade: false + kubernetesProvider: + scheme: + "apps/v1/Deployment": {gvr: {group: apps, version: v1, resource: deployments}, namespaced: true} + objects: [] + asserts: + - equal: + path: 'metadata.annotations["tracebloc.io/controlplane-digest-source"]' + value: tag diff --git a/client/values.schema.json b/client/values.schema.json index 1a5abdaf..ea506aac 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -1337,7 +1337,7 @@ }, "imageRefresh": { "type": "object", - "description": "Image-refresh CronJob (issue tracebloc/client#154; reworked by #569). Polls Docker Hub for the jobs-manager, pods-monitor and resource-monitor digests under the floating CLIENT_ENV tag and, on a change, pins the new digest onto the running workloads with `kubectl set image` (jobs-manager + requests-proxy Deployments and the resource-monitor DaemonSet). It used to `kubectl rollout restart` a single Deployment; the reference change is what lets those pods run imagePullPolicy=IfNotPresent and survive an offline restart. Disable to keep the running image in place until manual restart — the chart then renders imagePullPolicy=Always on the unpinned control-plane images, because a floating tag plus a restart is the only update path left.", + "description": "Image-refresh CronJob (issue tracebloc/client#154; reworked by #569). Polls Docker Hub for the jobs-manager, pods-monitor and resource-monitor digests under the floating CLIENT_ENV tag and, on a change, pins the new digest onto the running workloads with `kubectl set image` (jobs-manager + requests-proxy Deployments and the resource-monitor DaemonSet). It used to `kubectl rollout restart` a single Deployment; the reference change is what lets those pods run imagePullPolicy=IfNotPresent and survive an offline restart on a node that already cached that digest (client#1013 narrows this: once observed, the digest is seeded into the chart via tracebloc.controlPlaneDigest, so an offline/side-loaded k3d node that never pulled it restarts into ImagePullBackOff). Disable to keep the running image in place until manual restart — the chart then renders imagePullPolicy=Always on the unpinned control-plane images, because a floating tag plus a restart is the only update path left.", "properties": { "enabled": { "type": "boolean", diff --git a/client/values.yaml b/client/values.yaml index 564c0962..2097aee6 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -1547,7 +1547,13 @@ podTokenTtlSeconds: 604800 # - NOTE (#569): a chart upgrade no longer picks up new IMAGE content as a # side effect. That used to work because the control-plane pods ran # `imagePullPolicy: Always`, so any restart re-resolved the floating tag; -# they now run IfNotPresent so an offline restart cannot fail. New image +# they now run IfNotPresent so an offline restart cannot fail *provided the +# node already cached the running image*. NARROWED (client#1013): once +# image-refresh has observed a digest, the chart seeds that registry +# `@digest` onto the control-plane images (see tracebloc.controlPlaneDigest), +# so a node that never pulled that exact digest — an offline/side-loaded k3d +# or Docker-Desktop edge — now restarts into ImagePullBackOff instead of the +# cached tag. New image # content arrives via the image-refresh CronJob's `kubectl set image` # instead. This CronJob still delivers everything else a chart release # carries — template, RBAC, values-default and schema changes. @@ -1722,7 +1728,10 @@ autoUpgrade: # seeing the digest recorded but the workload still on repo:tag, pins # it (client-runtime#199) — so a fresh edge becomes reproducible ~one interval # post-install, not at the next upstream release. Restart-safe offline -# throughout. +# throughout — NARROWED (client#1013): once a digest has been recorded, the +# chart ALSO seeds it (tracebloc.controlPlaneDigest) and a later chart render +# pins repo@digest, so from then on an offline restart is restart-safe only +# on a node that already cached that digest. # - Idle-cheap: when the recorded digest matches today's digest AND the # workload already runs that digest, the script exits without touching # anything. If a helm re-render reverted the pin back to repo:tag it diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md index b7bc44be..89c7d127 100644 --- a/docs/MIGRATIONS.md +++ b/docs/MIGRATIONS.md @@ -245,6 +245,8 @@ Two things to know before you force: 1. **`--force-conflicts` re-takes *every* field the chart renders that a non-Helm manager owns — not only `limits.cpu`.** Helm prints the full conflict list on the failed attempt; read it first. On a long-running fleet the list can include `image-refresh`'s pinned digests (`kubectl set image`); forcing reverts them to the chart's rendered image, and `image-refresh` re-pins on its next tick. That is recoverable but is a real, if brief, image churn — don't force blind. + **From chart 1.9.111 (#1013 / client-runtime#199) the `.image` conflict narrows.** The chart now renders the last-refreshed `@digest` itself (`tracebloc.controlPlaneDigest` reads image-refresh's annotation via `lookup`), so in the **steady state** Helm applies the same value the `kubectl-set` manager holds and the `.spec…containers[…].image` conflict disappears. **This whole note is about the hand-run server-side path** (`helm upgrade --server-side=true --force-conflicts`) that reports a `.image` conflict at all — the automated fleet path is Helm 3.16 client-side (`alpine/helm:3.16.4`, no `--server-side`), whose 3-way merge **overwrites** the live value rather than conflicting, so it never surfaces here. On the server-side path, a `.image` conflict on 1.9.111+ therefore signals **annotation-vs-live LAG** — the rollout-timeout / `helm rollback` window where the annotation trails the live spec — not steady-state drift. Wait one refresh tick for the reconcile to re-pin, then re-run; or `--force-conflicts` to take the chart's rendered (annotation) digest, accepting it may briefly trail until the next tick. + 2. **`--server-side=true` is not optional, even though Helm 4 defaults to it.** After a rollback (including the automatic one the `auto-upgrade` CronJob performs when it reads a `pending-upgrade` release as a wedge — `backend#2877`), the release's stored apply method can revert to client-side. Then `--force-conflicts` **alone** fails with: ``` diff --git a/docs/SEAL-CHECK.md b/docs/SEAL-CHECK.md index 88962c8c..e47a6094 100644 --- a/docs/SEAL-CHECK.md +++ b/docs/SEAL-CHECK.md @@ -287,11 +287,17 @@ client-runtime#199. run a build carrying client-runtime#416 (the HF-offline injection) *before* the seal, or NLP templates fail by network block instead of the clean closed door. On each cluster the chart renders control-plane images as `repository:tag` + -`IfNotPresent`, and the `image-refresh` CronJob pins the live digest. dev now -tracks its `:dev` tag (auto-refresh); staging/prod **pin the #416 digest** in -values (`images.jobsManager.digest`) because their tag node-caches were stale -(pre-#416) — pinning is deterministic and survives `--reset-then-reuse-values`, -but disables `image-refresh` auto-tracking until the pin is bumped. +`IfNotPresent` on a fresh install, and the `image-refresh` CronJob pins the live +digest out-of-band. **As of chart 1.9.111 (#1013 / client-runtime#199) a +`helm upgrade` no longer reverts that pin:** `tracebloc.controlPlaneDigest` reads +image-refresh's last-refreshed annotation via `lookup` and re-renders +`repository@digest`, so `--reset-then-reuse-values` preserves the digest on its +own — you no longer need a values pin merely to survive an upgrade. dev now +tracks its `:dev` tag (auto-refresh); staging/prod still **pin the #416 digest** +in values (`images.jobsManager.digest`) because their tag node-caches were stale +(pre-#416) and a values pin is the deterministic, image-refresh-independent +choice (it wins over the annotation), but that pin disables `image-refresh` +auto-tracking until it is bumped. ## Runbook: flip the §8.2 egress lockdown on a real fleet diff --git a/replies/bugbot-flap.md b/replies/bugbot-flap.md new file mode 100644 index 00000000..3f80bb3e --- /dev/null +++ b/replies/bugbot-flap.md @@ -0,0 +1 @@ +Confirmed — same finding as @shujaatTracebloc's BLOCKING thread on this helper. The fix is A-side (#1008): write each workload's `last-refreshed-*` annotation right after its own `rollout status` and before the flap-guard `exit 0`, so the annotation can't lag the live spec into a backward move. B is held behind A; tracked there, leaving open. diff --git a/replies/bugbot-lookup.md b/replies/bugbot-lookup.md new file mode 100644 index 00000000..d415d27a --- /dev/null +++ b/replies/bugbot-lookup.md @@ -0,0 +1 @@ +Addressed in 1c78acf — `tracebloc.controlPlaneDigestSource` records `tracebloc.io/controlplane-digest-source` (`values`|`annotation`|`tag`|`tag-lookup-failed`) on the jobs-manager Deployment; on an upgrade an empty lookup ⇒ `tag-lookup-failed`, since the Deployment must exist. See the fuller reply on @shujaatTracebloc's thread. diff --git a/replies/bugbot-tag.md b/replies/bugbot-tag.md new file mode 100644 index 00000000..ea481aa3 --- /dev/null +++ b/replies/bugbot-tag.md @@ -0,0 +1 @@ +Documented in 1c78acf next to the suspend note; the actual fix (record the tag alongside the digest, honour the annotation only when it matches the current env) is an A-side change — see @shujaatTracebloc's thread. Leaving open until A lands. diff --git a/replies/clientenv.md b/replies/clientenv.md new file mode 100644 index 00000000..38f5d23f --- /dev/null +++ b/replies/clientenv.md @@ -0,0 +1 @@ +Documented in 1c78acf next to the suspend note in `controlPlaneDigest` (your "at minimum, call it out"). The actual containment — image-refresh recording the resolved tag alongside the digest (`tracebloc.io/last-refreshed--tag`) so the helper honours the annotation only when it equals the current `tracebloc.clientEnv`, else `:tag` — is an **A-side** change (image-refresh writes that annotation), so I'm leaving this thread open until A carries it; the B-side "honour only if tag matches" lands on top of that. Flagging it for #1008. diff --git a/replies/flap.md b/replies/flap.md new file mode 100644 index 00000000..90a3dedb --- /dev/null +++ b/replies/flap.md @@ -0,0 +1 @@ +Real, and you're right that #1008 as written doesn't heal it — its live-spec reconcile sits in the `recorded == latest` branch, which the flap-lockout state never reaches. This is a **BLOCKING precondition on A (#1008)**, not something B can fix alone, so per that I'm leaving this open and flagging it on #1008: the robust fix is A-side — write each image's `last-refreshed-*` annotation immediately after *that workload's own* `rollout status` (not one batched annotate after all three), and write it **before** the flap-guard's `exit 0`, so the annotation can never lag the live spec into a chart-enforced backward move. Your own 9666cf0 note confirms the mechanism: the fleet path is Helm-3 client-side, whose 3-way merge overwrites live `@D1` with the rendered `@D0` rather than conflicting. B stays held behind A until A covers this. (The `tag-lookup-failed` signal I added for the lookup thread makes a `:tag` fall-back observable, but not this backward-move — that needs the A-side ordering fix.) diff --git a/replies/init.md b/replies/init.md new file mode 100644 index 00000000..f1da33c3 --- /dev/null +++ b/replies/init.md @@ -0,0 +1 @@ +Fixed in 1c78acf. The `wait-for-mysql` init container no longer seeds from the annotation — it renders `:tag` (or an explicit `images.jobsManager.digest` operator pin) via `tracebloc.image` directly. Your reasoning is exactly right: it's an init container so an unpullable digest is `Init:ImagePullBackOff` (pod never starts) on the very k3d/Docker-Desktop edge #569 protects, and `kubectl set image` never names it so the chart would be its only un-reconciled driver. `control_plane_digest_test.yaml` flips the case (init stays `:tag` with a valid annotation present; still honours an operator pin), mutation-checked: re-seeding the init reddens that guard. diff --git a/replies/lookup.md b/replies/lookup.md new file mode 100644 index 00000000..0152ffcc --- /dev/null +++ b/replies/lookup.md @@ -0,0 +1 @@ +Addressed in 1c78acf, as a recorded signal rather than a hard `fail` (your steer — a `fail` would break `helm diff upgrade`, where lookup is empty too). New `tracebloc.controlPlaneDigestSource` renders `tracebloc.io/controlplane-digest-source` on the jobs-manager Deployment: `values` | `annotation` | `tag` | `tag-lookup-failed`. The key distinction you named — on an **upgrade** the jobs-manager Deployment MUST exist, so `.Release.IsUpgrade` + an empty lookup ⇒ a **read failure**, not a fresh install — is rendered as `tag-lookup-failed`, so a monitored edge can alert on the silent fall-back instead of discovering the #199 revert after the fact. Metadata-only (no pod roll). Covered clusterless in `control_plane_digest_test.yaml` incl. the `tag-lookup-failed`-on-upgrade case; `helm template --is-upgrade` renders it, plain render stays `tag`. diff --git a/scripts/tests/e2e-auto-upgrade.sh b/scripts/tests/e2e-auto-upgrade.sh index b2eb5784..a2775b8d 100755 --- a/scripts/tests/e2e-auto-upgrade.sh +++ b/scripts/tests/e2e-auto-upgrade.sh @@ -116,6 +116,19 @@ local_prod_digest() { awk -F'"' '/^[[:space:]]*prodDigest:[[:space:]]*"/ {print $2; exit}' "$CHART_DIR/values.yaml" } +# The control-plane image DIGEST the jobs-manager container is ACTUALLY running, +# read off the LIVE Deployment and extracted from `image:` (repo@sha256:…). This +# is what `tracebloc.controlPlaneDigest` must render from the last-refreshed +# annotation (client-runtime#199) — assert the rendered image, not merely that +# the annotation survived: the annotation is a k8s object helm never touches, so +# it survives even if the helper is removed and the image reverts to a floating +# `:tag`. Empty output means no digest is pinned on `image:` (the revert). +jm_controlplane_image_digest() { + kubectl get -n "$NS" "$(jm_deploy)" \ + -o jsonpath='{.spec.template.spec.containers[0].image}' \ + | sed -n 's/.*@\(sha256:[a-f0-9]\{64\}\).*/\1/p' +} + # Decode one key from the release Secret. Defined here with the other # cluster-reading helpers (it used to live in path 5) so the baseline # root-rotation assertion below can use it too (backend#3384). @@ -216,8 +229,31 @@ BASELINE_EGRESS_PROXY_URL="$(jm_egress_proxy_url)" echo " baseline egress posture: external_443=$([ "$BASELINE_EXTERNAL_443" = 1 ] && echo present || echo absent) egress_proxy_url=${BASELINE_EGRESS_PROXY_URL:-}" echo "── simulate an image-refresh-managed annotation (must survive upgrades) ──" +# A VALID sha256 digest (64 hex): tracebloc.controlPlaneDigest validates the +# annotation against `^sha256:[a-f0-9]{64}$` before rendering it onto `image:`, so +# a placeholder like `sha256:e2e-sentinel` would degrade to `:tag` and stop +# exercising branch 2 (the digest render this test exists to protect). Bound once +# and reused by both the survival check and the rendered-image check in path 2. +# +# DELIBERATELY UNPULLABLE, and asserted as such below. This 64-hex digest does +# not resolve to any image in the k3d cache (a side-loaded `k3d image import` +# stores a tag alias with no resolvable digest — see the #569 offline note in +# _helpers.tpl), so any upgrade that renders the WORKING-TREE chart (path 1's +# `--reuse-values`, path 2's `--reset-then-reuse-values`) Recreates jobs-manager +# onto an unpullable ref and it sits in Init:ImagePullBackOff. (The intermediate +# reset to the PUBLISHED $PREV chart between paths 1 and 2 MAY briefly un-wedge it +# by rendering `:tag` — but only while $PREV predates tracebloc.controlPlaneDigest; +# once 1.9.111+ is the published $PREV the reset renders this sentinel too and the +# un-wedge stops happening. Do not depend on it either way.) That +# is harmless to paths 2-5 BY CONSTRUCTION regardless: the only readiness wait in this script +# is `kubectl wait … nodes` at the top (before this seed), and every assertion +# after this point is a spec-only read (`kubectl get … -o jsonpath`), never a +# live-pod / rollout / readiness read. If you add a step below that needs a +# running jobs-manager pod, seed a pullable digest here (or clear the annotation +# first) — do not assume the pod is up. +E2E_REFRESH_DIGEST="sha256:e2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee2ee" kubectl annotate -n "$NS" "$(jm_deploy)" \ - "tracebloc.io/last-refreshed-jobs-manager-digest=sha256:e2e-sentinel" --overwrite + "tracebloc.io/last-refreshed-jobs-manager-digest=$E2E_REFRESH_DIGEST" --overwrite echo "── path 1: manual-operator habit — helm upgrade --reuse-values ──" # Old stored values replayed against the new chart: every new key is absent and @@ -249,7 +285,16 @@ else [ "$(jm_ingestor_digest)" = "$BASELINE_PROD_DIGEST" ] \ || fail "--reuse-values did not replay the baseline prod pin verbatim: got '$(jm_ingestor_digest)', want '$BASELINE_PROD_DIGEST' (stored computed values must win over new chart defaults on this path)" fi -echo " OK: upgrade succeeded, egress posture replayed from the baseline verbatim, ingestor pin matches the baseline era (${BASELINE_PROD_DIGEST:-floating})" +# Control-plane pin also survives --reuse-values: this upgrade renders the working +# tree chart, so tracebloc.controlPlaneDigest reads the seeded annotation and pins +# the jobs-manager image to the sentinel. Spec-only read (no live-pod dependency) — +# and this is the upgrade that first Recreates jobs-manager onto the deliberately +# unpullable sentinel, asserting explicitly here so the Init:ImagePullBackOff that +# follows is a documented, expected state rather than a surprise to the next author. +CP_DIGEST_P1="$(jm_controlplane_image_digest)" +[ "$CP_DIGEST_P1" = "$E2E_REFRESH_DIGEST" ] \ + || fail "--reuse-values did not seed the jobs-manager control-plane image from the last-refreshed digest: got '${CP_DIGEST_P1:-}', want '$E2E_REFRESH_DIGEST' (tracebloc.controlPlaneDigest did not preserve the pin on this path — client-runtime#199)" +echo " OK: upgrade succeeded, egress posture replayed from the baseline verbatim, ingestor pin matches the baseline era (${BASELINE_PROD_DIGEST:-floating}), control-plane image pinned to the last-refreshed digest (jobs-manager now Init:ImagePullBackOff on the unpullable sentinel BY DESIGN — later paths read spec only)" echo "── isolate path 2 from path 1's --reuse-values contamination (#459) ──" # path 1's --reuse-values rewrote THIS release's recorded values to the baseline's FULL @@ -301,7 +346,22 @@ kubectl get deploy "${NS}-egress-proxy" -n "$NS" >/dev/null \ || fail "auto-upgrade did not deploy the egress gateway (new defaults did not flow)" ANNOT="$(kubectl get -n "$NS" "$(jm_deploy)" \ -o jsonpath='{.metadata.annotations.tracebloc\.io/last-refreshed-jobs-manager-digest}')" -[ "$ANNOT" = "sha256:e2e-sentinel" ] || fail "image-refresh annotation was clobbered by the upgrade" +[ "$ANNOT" = "$E2E_REFRESH_DIGEST" ] || fail "image-refresh annotation was clobbered by the upgrade" +# THE #1013 CONTRACT: the auto-upgrade must SEED the control-plane image from that +# annotation, so the jobs-manager container renders repo@ instead of the +# floating :tag a stale node would run old (client-runtime#199). The annotation +# check above is necessary but not sufficient — helm never touches the annotation, +# so it survives even if tracebloc.controlPlaneDigest is removed or always returns +# empty and the image silently reverts to :tag. Assert the RENDERED image digest, +# which is what actually protects the pin: this is the assertion that reddens if +# the helper is dropped. (This test's upgrade tolerates the unpullable sentinel +# digest because *this script's* helm upgrade passes no --wait and every check here +# is a spec-only read — NOT a property of the fleet path: auto-upgrade-cronjob.yaml +# runs `--atomic --cleanup-on-fail --timeout`, and --atomic implies --wait, so on a +# real edge an unpullable seed blocks on readiness and rolls back. See the :245 note.) +CP_DIGEST="$(jm_controlplane_image_digest)" +[ "$CP_DIGEST" = "$E2E_REFRESH_DIGEST" ] \ + || fail "auto-upgrade did not seed the jobs-manager control-plane image from the last-refreshed digest: got '${CP_DIGEST:-}', want '$E2E_REFRESH_DIGEST' (tracebloc.controlPlaneDigest did not render the pin — client-runtime#199 revert not prevented)" DEPLOYED="$(helm list -n "$NS" --filter "^${NS}\$" -o yaml \ | awk '/^[[:space:]]*chart:/ {print $2; exit}')" [ "$DEPLOYED" = "client-${LOCAL_VERSION}" ] || fail "deployed chart is $DEPLOYED, expected client-${LOCAL_VERSION}"