fix(chart): reconcile the telemetry Collector in place once its token Secret exists

.github/workflows/helm-unit.yaml

@@ -89,3 +89,11 @@ jobs:
 
       - name: Run unit tests
         run: helm unittest ./client
+
+      # Mutation (d) of the telemetry-reconcile gate: restoring the old
+      # "resolves itself" wording must redden the status-record unit test. It
+      # needs the plugin installed above, which is why it runs here and not in
+      # `make drift` (that job has no helm-unittest and read the missing plugin
+      # as a wrong-reason red on 2026-09-11).
+      - name: Prove the status-record honesty test can fail
+        run: MODE=record bash scripts/tests/auto-upgrade-telemetry-reconcile-mutations.sh

Makefile

@@ -390,6 +390,8 @@ DRIFT_GUARDS := scripts/gen-manifest.sh --check|\
   bash scripts/tests/gate-default-prose-mutations.sh|\
   bash scripts/tests/guards-survive-spaced-paths.sh|\
   bash scripts/tests/auto-upgrade-inflight-vs-wedge.sh|\
+  bash scripts/tests/auto-upgrade-telemetry-reconcile.sh|\
+  bash scripts/tests/auto-upgrade-telemetry-reconcile-mutations.sh|\
   bash scripts/tests/hostpath-reads-guarded.sh|\
   bash scripts/tests/jobs-manager-waits-for-mysql.sh|\
   bash scripts/tests/control-plane-footprint.sh|\
@@ -585,3 +587,4 @@ helm-unittest:
 	       echo "  helm plugin install https://github.com/helm-unittest/helm-unittest --version 0.5.2"; \
 	       exit 1; }
 	helm unittest ./client
+	MODE=record bash scripts/tests/auto-upgrade-telemetry-reconcile-mutations.sh

client/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.117
-appVersion: "1.9.117"
+version: 1.9.118
+appVersion: "1.9.118"
 keywords:
   - tracebloc
   - kubernetes

client/templates/_helpers.tpl

@@ -1331,6 +1331,29 @@ can be kept above the configured helm timeout.
 {{- printf "%s-telemetry-collector" (include "tracebloc.fullname" .) | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{/*
+  tracebloc.telemetryStatusName — the ConfigMap that records what the Collector
+  decided, and why (templates/telemetry-collector-status.yaml). ONE resolver for
+  its two readers: the status template that writes it and the auto-upgrade
+  CronJob that reads it back out of the stored release manifest to decide whether
+  a same-version re-render is due (backend#3550). Deliberately NOT under the
+  `telemetry-collector` prefix — see the status template for why five shell gates
+  depend on that.
+*/}}
+{{- define "tracebloc.telemetryStatusName" -}}
+{{- printf "%s-telemetry-status" (include "tracebloc.fullname" .) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+  tracebloc.telemetryStateAnnotation — the annotation key the status ConfigMap
+  carries its resolved state under. Shared by the writer (status template) and the
+  reader (auto-upgrade script, which greps it out of `helm get manifest` and out of
+  a server-side dry-run render) so the two cannot disagree about the spelling.
+*/}}
+{{- define "tracebloc.telemetryStateAnnotation" -}}
+tracebloc.io/telemetry-collector-state
+{{- end -}}
+
 {{/*
   tracebloc.telemetryTokenLegacyName — the pre-backend#2625 fixed Secret name.
 

client/templates/auto-upgrade-cronjob.yaml

@@ -116,6 +116,138 @@ data:
       }'
     }
 
+    # --- telemetry Collector same-version reconcile: begin (backend#3550) -------
+    # The Collector's three-state rule (`telemetryCollector.enabled` unset ->
+    # collect iff the token Secret exists, else record `skipped-no-token`) is
+    # decided by a template `lookup`, and a `lookup` runs ONLY inside a
+    # `helm upgrade`. This tick used to upgrade only onto a NEWER published chart,
+    # so the Secret appearing (jobs-manager writes it minutes after install)
+    # triggered nothing: measured on a fresh edge, three hourly ticks passed with
+    # the Secret present and no Collector, which then rendered a day later only
+    # because a new chart happened to ship. So a tick that has no newer chart to
+    # install now asks the one question that matters and re-renders the DEPLOYED
+    # version in place when the answer is yes.
+    #
+    # DERIVED, NOT RESTATED. The decision is read from the chart's own decider,
+    # twice: the stored state out of `helm get manifest` (what the last render
+    # decided) and the would-be state out of a server-side dry-run of the SAME
+    # chart version (`--dry-run=server`, helm 3.13+, runs the templates' real
+    # `lookup`s against the cluster). No Secret name, Class A rule or DaemonSet
+    # gate is copied here; the three names this needs arrive in env from the same
+    # helpers the templates render with (TELEMETRY_STATUS_NAME,
+    # TELEMETRY_STATUS_ANNOTATION, TELEMETRY_COLLECTOR_NAME).
+    #
+    # TRI-STATE, and "cannot tell" never acts: 0 reconcile / 1 nothing to do /
+    # 2 cannot tell. An unreadable manifest, a failed dry-run, a contradictory
+    # record, or a deployed version the repo cannot serve is not evidence either
+    # way, so the tick leaves the release alone and says so on stdout.
+    #
+    # AHEAD OF THE INDEX IS THE ONE STATE THIS CANNOT FIX, said plainly: a release
+    # deployed from a dev chart (current > latest) runs a version the repo does
+    # not offer, and nothing here can re-render a chart it cannot fetch. That edge
+    # converges on the next published chart, and the verdict names it every tick.
+
+    # Parses the `version:` field out of `helm search repo ... -o yaml` output on
+    # stdin. Search results are newest-first, so with no --version filter on the
+    # search itself the first hit is the latest; with one, it's the served
+    # version for that pin (or nothing, if the repo does not carry it). One
+    # parser for both call sites below so they cannot drift in how they read the
+    # same `helm search repo` output shape.
+    parse_repo_version() {
+      awk '/^[[:space:]]*version:/ {print $2; exit}'
+    }
+
+    # Resolved Collector state carried by the manifest on stdin: the state
+    # annotation of the ConfigMap named TELEMETRY_STATUS_NAME. Prints nothing when
+    # the record is absent (a release rendered by a chart that predates it).
+    telemetry_state_of() {
+      awk -v name="$TELEMETRY_STATUS_NAME" -v key="$TELEMETRY_STATUS_ANNOTATION" '
+        /^---/ { doc = ""; next }
+        /^  name: / && $2 == name { doc = "status"; next }
+        doc == "status" && $1 == key ":" { v = $2; gsub(/"/, "", v); print v; exit }'
+    }
+
+    # Exit 0 iff the manifest on stdin carries a DaemonSet named
+    # TELEMETRY_COLLECTOR_NAME, i.e. the last render DID enable the Collector.
+    telemetry_manifest_has_collector() {
+      awk -v name="$TELEMETRY_COLLECTOR_NAME" '
+        /^---/ { kind = ""; next }
+        /^kind: / { kind = $2; next }
+        kind == "DaemonSet" && /^  name: / && $2 == name { found = 1; exit }
+        END { exit found ? 0 : 1 }'
+    }
+
+    # THE condition, in one place, so the test and its mutation drive this code
+    # and not a copy. Reads helm; prints one reason line; returns the tri-state.
+    # Reconcile iff: the stored record says skipped-no-token (which by the
+    # decider's construction means `enabled` was unset) AND the stored manifest
+    # carries no Collector DaemonSet AND a server-side dry-run of the deployed
+    # chart version now resolves to `enabled` (the token Secret has arrived).
+    telemetry_reconcile_verdict() {
+      if [ -z "${TELEMETRY_STATUS_NAME:-}" ] || [ -z "${TELEMETRY_STATUS_ANNOTATION:-}" ] \
+         || [ -z "${TELEMETRY_COLLECTOR_NAME:-}" ]; then
+        echo "reconcile inputs missing from env (TELEMETRY_STATUS_NAME / TELEMETRY_STATUS_ANNOTATION / TELEMETRY_COLLECTOR_NAME)"
+        return 2
+      fi
+      _manifest="$(helm get manifest "$RELEASE_NAME" -n "$RELEASE_NAMESPACE" 2>/dev/null)" \
+        || { echo "cannot read the deployed manifest (helm get manifest failed)"; return 2; }
+      [ -n "$_manifest" ] || { echo "cannot read the deployed manifest (empty)"; return 2; }
+      _stored="$(printf '%s\n' "$_manifest" | telemetry_state_of)"
+      case "$_stored" in
+        "") echo "release carries no telemetry status record (chart predates it); nothing to reconcile"; return 1 ;;
+        skipped-no-token) ;;
+        *) echo "telemetry Collector state is '$_stored'; nothing to reconcile"; return 1 ;;
+      esac
+      if printf '%s\n' "$_manifest" | telemetry_manifest_has_collector; then
+        echo "status record says skipped-no-token but the release already carries DaemonSet $TELEMETRY_COLLECTOR_NAME: contradictory, not touching it"
+        return 2
+      fi
+      _search="$(helm search repo "${REPO_NAME}/${CHART_NAME}" --version "$CURRENT" -o yaml 2>&1)" \
+        || { echo "cannot ask $REPO_URL whether it serves $CURRENT: $(printf '%s\n' "$_search" | tail -n 1)"; return 2; }
+      _served="$(printf '%s\n' "$_search" | parse_repo_version)"
+      [ "$_served" = "$CURRENT" ] \
+        || { echo "deployed chart $CURRENT is not served by $REPO_URL (newest there: $LATEST): a same-version re-render is impossible, the Collector renders on the next published chart"; return 2; }
+      # Bounded at 60s: a server-side dry-run only templates and diffs against
+      # the API server, it never waits on a rollout, so it should complete in
+      # seconds. Left unbounded it would share helm's 5m default, which could eat
+      # deep into the 300s repo-fetch buffer activeDeadlineSeconds reserves below
+      # for the REAL --atomic upgrade that follows it on this same tick.
+      _render="$(helm upgrade "$RELEASE_NAME" "${REPO_NAME}/${CHART_NAME}" \
+          --namespace "$RELEASE_NAMESPACE" --version "$CURRENT" \
+          --reset-then-reuse-values --dry-run=server --hide-secret --timeout 60s 2>&1)" \
+        || { echo "server-side dry-run of $CURRENT failed: $(printf '%s\n' "$_render" | tail -n 1)"; return 2; }
+      _would="$(printf '%s\n' "$_render" | telemetry_state_of)"
+      case "$_would" in
+        enabled) echo "token Secret is now present, a re-render of $CURRENT enables the Collector"; return 0 ;;
+        "") echo "dry-run render of $CURRENT carries no telemetry status record; cannot tell"; return 2 ;;
+        *) echo "a re-render would still decide '$_would'; nothing to reconcile"; return 1 ;;
+      esac
+    }
+
+    # Runs where this tick has no newer chart to install. The reconcile IS the
+    # upgrade at the bottom of this script, pinned to the DEPLOYED version, so it
+    # keeps every flag and the same values discipline. Only verdict 0 acts; 2 is
+    # logged because "cannot tell" must never be silent; 1 stays quiet, as today.
+    telemetry_reconcile_if_needed() {
+      if _why="$(telemetry_reconcile_verdict)"; then
+        log "reconcile: $_why; re-rendering $CURRENT in place (same-version upgrade, values reused)"
+        helm upgrade "$RELEASE_NAME" "${REPO_NAME}/${CHART_NAME}" \
+          --namespace "$RELEASE_NAMESPACE" \
+          --version "$CURRENT" \
+          --reset-then-reuse-values \
+          --atomic \
+          --cleanup-on-fail \
+          --timeout "$UPGRADE_TIMEOUT"
+        log "reconcile complete: $CURRENT re-rendered in place"
+      else
+        _rc=$?
+        if [ "$_rc" -eq 2 ]; then
+          log "reconcile: NOT re-rendering, cannot tell whether the Collector is due: $_why"
+        fi
+      fi
+    }
+    # --- telemetry Collector same-version reconcile: end ------------------------
+
     log "release=$RELEASE_NAME namespace=$RELEASE_NAMESPACE repo=$REPO_URL"
 
     helm repo add "$REPO_NAME" "$REPO_URL" >/dev/null
@@ -205,10 +337,8 @@ data:
         ;;
     esac
 
-    # Latest published version. Helm orders search results newest-first,
-    # so the first `version:` line in the YAML output is the latest.
-    LATEST="$(helm search repo "${REPO_NAME}/${CHART_NAME}" -o yaml \
-      | awk '/^[[:space:]]*version:/ {print $2; exit}')"
+    # Latest published version.
+    LATEST="$(helm search repo "${REPO_NAME}/${CHART_NAME}" -o yaml | parse_repo_version)"
 
     # Currently deployed chart version. `helm list -o yaml` emits
     # `chart: client-<version>` for the matching release.
@@ -229,14 +359,16 @@ data:
     log "current=$CURRENT latest=$LATEST"
 
     if [ "$LATEST" = "$CURRENT" ]; then
-      log "already at latest; nothing to do"
+      log "already at latest; nothing to upgrade"
+      telemetry_reconcile_if_needed
       exit 0
     fi
 
     # Semver-aware compare via `sort -V` so 1.10.0 > 1.9.0.
     NEWER="$(printf '%s\n%s\n' "$CURRENT" "$LATEST" | sort -V | tail -n 1)"
     if [ "$NEWER" != "$LATEST" ]; then
-      log "deployed version is ahead of repo (current=$CURRENT > latest=$LATEST); skipping"
+      log "deployed version is ahead of repo (current=$CURRENT > latest=$LATEST); skipping the upgrade"
+      telemetry_reconcile_if_needed
       exit 0
     fi
 
@@ -374,6 +506,16 @@ spec:
                 # predates the key (same pattern as activeDeadlineSeconds).
                 - name: WEDGE_MIN_AGE_SECONDS
                   value: {{ include "tracebloc.durationSeconds" (.Values.autoUpgrade.pendingWedgeMinAge | default "45m") | quote }}
+                # backend#3550: the names the same-version reconcile reads the
+                # stored manifest and the dry-run render by. Rendered from the SAME
+                # helpers the status template and the Collector DaemonSet use, so
+                # the reader and the writers cannot disagree about a spelling.
+                - name: TELEMETRY_STATUS_NAME
+                  value: {{ include "tracebloc.telemetryStatusName" . | quote }}
+                - name: TELEMETRY_STATUS_ANNOTATION
+                  value: {{ include "tracebloc.telemetryStateAnnotation" . | quote }}
+                - name: TELEMETRY_COLLECTOR_NAME
+                  value: {{ include "tracebloc.telemetryCollectorName" . | quote }}
               securityContext:
                 allowPrivilegeEscalation: false
                 readOnlyRootFilesystem: true

client/templates/telemetry-collector-status.yaml

@@ -30,19 +30,40 @@
   label selectors.
 */}}
 {{- $state := include "tracebloc.telemetryCollectorState" . -}}
+{{- /*
+  WHAT ACTUALLY RE-RENDERS A SKIP (backend#3550). The record used to say the
+  no-token skip "resolves itself once jobs-manager writes the Secret". It does
+  not: the `lookup` that decides the state runs only inside a `helm upgrade`, and
+  the Secret appearing triggers none. Measured on a fresh edge — the Secret was
+  present within minutes, three hourly ticks passed, no Collector; it rendered a
+  day later, when a NEWER chart happened to be published. So the trigger is a
+  helm upgrade, and the record must name the one that will actually happen. The
+  auto-upgrade tick now performs a same-version re-render when the stored state is
+  skipped-no-token and a server-side dry-run says the Secret has arrived
+  (auto-upgrade-cronjob.yaml), but only when the deployed chart version is one
+  its repo can serve — an edge running a version AHEAD of the published index
+  (a dev chart) has nothing the tick can re-render, and waits for the next
+  published chart. Both the schedule and the repo are read from the values that
+  drive that CronJob, never restated here.
+*/ -}}
+{{- $au := default (dict) .Values.autoUpgrade -}}
+{{- $trigger := "autoUpgrade is off in this release, so nothing re-renders it for you: run `helm upgrade` on this release once the Secret exists" -}}
+{{- if $au.enabled -}}
+{{- $trigger = printf "the next auto-upgrade tick (CronJob schedule %q) re-renders this release in place once the Secret exists, provided the deployed chart version %s is one the repo %s serves — a version ahead of that repo's index cannot be re-rendered and waits for the next published chart" ($au.schedule | default "unset" | toString) .Chart.Version ($au.repoUrl | default "unset" | toString) -}}
+{{- end -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ printf "%s-telemetry-status" (include "tracebloc.fullname" .) }}
+  name: {{ include "tracebloc.telemetryStatusName" . }}
   namespace: {{ .Values.nodeAgents.namespace.name }}
   labels:
     {{- include "tracebloc.labels" . | nindent 4 }}
     app.kubernetes.io/component: telemetry-collector-status
   annotations:
     # Sweepable across a cluster without parsing the data block:
     #   kubectl get cm -A -l app.kubernetes.io/component=telemetry-collector-status \
-    #     -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,STATE:.metadata.annotations.tracebloc\.io/telemetry-collector-state
-    tracebloc.io/telemetry-collector-state: {{ $state | quote }}
+    #     -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,STATE:.metadata.annotations.{{ include "tracebloc.telemetryStateAnnotation" . | replace "." "\\." }}
+    {{ include "tracebloc.telemetryStateAnnotation" . }}: {{ $state | quote }}
 data:
   state: {{ $state | quote }}
   # WHY, in the operator's terms — the state alone does not say what to do next.
@@ -51,7 +72,7 @@ data:
   {{- else if eq $state "disabled-by-operator" -}}
     "telemetryCollector.enabled is explicitly false in this release's values. Nothing is collected, by choice."
   {{- else if eq $state "skipped-incomplete-values" -}}
-    {{ printf "No Collector: nobody set telemetryCollector.enabled, and this release carries no Class A container lists — helm DELETES a key set to null and does not coalesce chart defaults back under --reuse-values, so a release predating the Collector arrives without them. Upgrading with --reset-then-reuse-values (which auto-upgrade uses) restores them and this resolves itself." | quote }}
+    {{ printf "No Collector: nobody set telemetryCollector.enabled, and this release carries no Class A container lists — helm DELETES a key set to null and does not coalesce chart defaults back under --reuse-values, so a release predating the Collector arrives without them. Upgrading with --reset-then-reuse-values (which auto-upgrade uses) restores them — on the next chart upgrade, which is the next auto-upgrade tick that finds a newer published chart or a `helm upgrade` you run; nothing re-renders this release by itself." | quote }}
   {{- else -}}
-    {{ printf "No Collector: nobody set telemetryCollector.enabled, and its token Secret is not present in namespace %q (looked for %q, and the legacy %q). jobs-manager writes that Secret when it next re-authenticates (backend#2274); this resolves itself once it does. Set telemetryCollector.enabled: false to opt out permanently." .Values.nodeAgents.namespace.name (include "tracebloc.telemetryTokenSecretName" .) (include "tracebloc.telemetryTokenLegacyName" .) | quote }}
+    {{ printf "No Collector: nobody set telemetryCollector.enabled, and its token Secret is not present in namespace %q (looked for %q, and the legacy %q). jobs-manager writes that Secret when it next re-authenticates (backend#2274). The Secret appearing triggers nothing by itself — the Collector renders on the next helm upgrade of this release: %s. Set telemetryCollector.enabled: false to opt out permanently." .Values.nodeAgents.namespace.name (include "tracebloc.telemetryTokenSecretName" .) (include "tracebloc.telemetryTokenLegacyName" .) $trigger | quote }}
   {{- end }}