From 4c29e869224ce0289f3df656541a9e52b74bc96e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Tue, 27 Oct 2026 21:16:28 +1300 Subject: [PATCH] fix(tamanu): release the previous program's field ownership on adopted objects The handover to Argo CD adopted the Deployments, Services and database objects the previous program created under stable names, but left the field managers it applied them with. Server-side apply lets Argo remove only the fields it owns, so a field only the old manager owned stays on the object: from Tamanu 2.60 the render drops the raw and reporting database variables, the stub deletes their Secrets, and the facility servers can no longer start. Every update of a handed-over stack now applies an empty object as each leftover pulumi-kubernetes manager, before anything is declared, so its fields are given up and the ones Argo does not also own are removed. --- .../runbooks/application-stuck-progressing.md | 1 + docs/runbooks/stub-refuses-update.md | 1 + pulumi/tamanu/on-k8s/src/argo/migrate.ts | 69 +++++++++++++++++-- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/docs/runbooks/application-stuck-progressing.md b/docs/runbooks/application-stuck-progressing.md index d8ac7bf9..030828d7 100644 --- a/docs/runbooks/application-stuck-progressing.md +++ b/docs/runbooks/application-stuck-progressing.md @@ -37,6 +37,7 @@ Waves are in [object names](../reference/tamanu-deployment-objects.md#rendered-o | A database `Cluster` is Progressing | Argo CD reports a cluster Healthy only once its Ready condition is true, and wave 0 waits for it ([K8STDR](../spec/kubernetes/tamanu/deployment-rollout.md#database-health)). | [Database not coming up](database-not-coming-up.md). | | Pods in `ImagePullBackOff` | The image tag does not exist. | [Image tag missing](image-pull-backoff.md). | | A `*-reporting-grants` pod stays in `ContainerCreating` | It reads a Secret the database operator creates with the cluster, and a missing Secret means the pod never starts, so the Job neither completes nor fails. | Check the database first: `-db-superuser` exists once its cluster is created ([object names](../reference/tamanu-deployment-objects.md#created-by-cloudnativepg)). | +| Server pods in `CreateContainerConfigError` with `secret "-raw-db" not found` or `"-reporting-db"`, while the Application is `Synced` | The deployment came from the previous program, which still owns fields on the adopted Deployments, so variables the render dropped from Tamanu 2.60 stay on them. Each update of the stub releases those fields first (`pulumi/tamanu/on-k8s/src/argo/migrate.ts`). | Run the update again ([change a deployment's config](../sops/change-deployment-config.md)). **[dev-OTS]** | | Pods `Pending` with a scheduling message | Servers need a node with `bes.node.purpose=workload` and the deployment's architecture, and replicas of a server must spread across nodes, so a pod that would break the spread stays unscheduled ([K8STDR](../spec/kubernetes/tamanu/deployment-rollout.md#placement)). | Read the pod's events. Capacity or architecture mismatches are a cluster matter: escalate. | | A hook Job pod is running for a long time | The migrator or provisioner is working or waiting on the database. | Read its logs ([read logs](../sops/read-logs.md#hook-jobs)). **[diagnose, sensitive-data]** | | A claim `-blobs` is `Pending` | Shared blob storage cannot be provisioned. | Escalate: [K8SVOL](../spec/kubernetes/volumes.md) and the `efs` storage class. | diff --git a/docs/runbooks/stub-refuses-update.md b/docs/runbooks/stub-refuses-update.md index ca7186e7..99463c9f 100644 --- a/docs/runbooks/stub-refuses-update.md +++ b/docs/runbooks/stub-refuses-update.md @@ -37,6 +37,7 @@ Know what it does first: - The config ConfigMaps are copied across first, so hand edits are kept. - The auth secrets are new, so every user is signed out. - Each Gateway's Certificate is deleted with it, and the issued TLS Secret is kept, so the new Gateway's Certificate finds it. If a certificate is ordered anyway, it is a renewal for the same hostnames. +- Every later update releases the fields the previous program still owns on the objects Argo CD adopted, so fields a later render drops are removed from them. ## 5. Escalate diff --git a/pulumi/tamanu/on-k8s/src/argo/migrate.ts b/pulumi/tamanu/on-k8s/src/argo/migrate.ts index f56c62b4..274d60ec 100644 --- a/pulumi/tamanu/on-k8s/src/argo/migrate.ts +++ b/pulumi/tamanu/on-k8s/src/argo/migrate.ts @@ -3,10 +3,12 @@ import { CoreV1Api, CustomObjectsApi, type KubernetesObject, + KubernetesObjectApi, + PatchStrategy, } from '@kubernetes/client-node'; import { log, runtime } from '@pulumi/pulumi'; import { argoNamespace } from '../../../../common/k8s/argo.ts'; -import { k8sClient } from '../../../../common/k8s/client.ts'; +import { k8sClient, k8sConfig } from '../../../../common/k8s/client.ts'; import { namespaceName } from '../namespace.ts'; // A stack deployed by the previous program has Kubernetes objects that Pulumi created. @@ -27,6 +29,7 @@ const WAIT_MINUTES = 10; const core = k8sClient(CoreV1Api); const discovery = k8sClient(ApisApi); const custom = k8sClient(CustomObjectsApi); +const objects = KubernetesObjectApi.makeApiClient(k8sConfig()); interface Kind { group: string; @@ -135,7 +138,11 @@ async function copyConfig(): Promise { await core.createNamespacedConfigMap({ namespace: namespaceName, body: { - metadata: { name: target, namespace: namespaceName, labels: { [TEMPLATED_FROM]: from } }, + metadata: { + name: target, + namespace: namespaceName, + labels: { [TEMPLATED_FROM]: from }, + }, data: cm.data, binaryData: cm.binaryData, }, @@ -161,7 +168,10 @@ async function removeReplaced(): Promise { for (const service of services.items.filter(autonamed)) { log.info(`Deleting service ${service.metadata?.name}`); await orNothing( - core.deleteNamespacedService({ name: service.metadata!.name!, namespace: namespaceName }), + core.deleteNamespacedService({ + name: service.metadata!.name!, + namespace: namespaceName, + }), ); } const configMaps = await core.listNamespacedConfigMap({ namespace: namespaceName }); @@ -191,16 +201,67 @@ async function waitForRemoval(): Promise { } } +// What Argo adopts from the previous program under the same names. +const ADOPTED = [ + { apiVersion: 'apps/v1', kind: 'Deployment' }, + { apiVersion: 'v1', kind: 'Service' }, + { apiVersion: 'postgresql.cnpg.io/v1', kind: 'Cluster' }, + { apiVersion: 'postgresql.cnpg.io/v1', kind: 'ScheduledBackup' }, + { apiVersion: 'barmancloud.cnpg.io/v1', kind: 'ObjectStore' }, +]; + +const PULUMI_MANAGER = /^pulumi-kubernetes-/; + +// An adopted object keeps the field managers the previous program applied it with, and +// server-side apply lets Argo remove only the fields it owns itself. A field only Pulumi +// owns, such as a variable a later render drops, would otherwise stay on the object for +// good. Applying an empty object as each of those managers gives up their fields, and the +// ones Argo does not also own are removed. +async function releaseLegacyFields(): Promise { + for (const { apiVersion, kind } of ADOPTED) { + const { items } = (await orNothing(objects.list(apiVersion, kind, namespaceName))) ?? { + items: [], + }; + for (const object of items) { + if (!object.metadata?.annotations?.[TRACKING]) continue; + const managers = (object.metadata.managedFields ?? []) + .map((entry) => entry.manager) + .filter((manager): manager is string => !!manager && PULUMI_MANAGER.test(manager)); + for (const manager of managers) { + log.info(`Releasing the fields ${manager} owns on ${kind} ${object.metadata.name}`); + if (runtime.isDryRun()) continue; + await objects.patch( + { + apiVersion, + kind, + metadata: { name: object.metadata.name, namespace: namespaceName }, + }, + undefined, + undefined, + manager, + undefined, + PatchStrategy.ServerSideApply, + ); + } + } + } +} + /** * Hands a stack deployed by the previous program over to Argo CD. Does nothing for a new * stack or one that has already been handed over. Fails before changing anything if the * previous program has not marked its resources, since Pulumi would otherwise delete them. + * Once handed over, every update releases what the previous program still owns on the + * objects Argo adopted. * * During a preview it only checks. */ export async function migrateFromLegacy(): Promise<'new' | 'migrated' | 'migrating'> { await assertArgoInstalled(); - if (await isMigrated()) return 'migrated'; + if (await isMigrated()) { + await releaseLegacyFields(); + return 'migrated'; + } if (!(await hasLegacyObjects())) return 'new'; if (!(await isMarked())) {