From 065febb8d0b41b9d8fa4524c12630d4ae8453e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Gon=C3=A7alves?= Date: Fri, 5 Jun 2026 15:03:14 +0100 Subject: [PATCH 1/4] Replace Engine with Worker --- chart/templates/_helpers.tpl | 4 +- chart/templates/deployment-app.yaml | 2 + ...ent-engine.yaml => deployment-worker.yaml} | 41 +++++++++---------- chart/templates/external-secret.yaml | 4 +- .../{hpa-engine.yaml => hpa-worker.yaml} | 20 ++++----- chart/templates/networkpolicy.yaml | 18 ++++---- .../{pdb-engine.yaml => pdb-worker.yaml} | 10 ++--- chart/templates/secret-env.yaml | 2 +- ...ervice-engine.yaml => service-worker.yaml} | 10 ++--- ...engine.yaml => serviceaccount-worker.yaml} | 8 ++-- chart/templates/servicemonitor.yaml | 10 ++--- chart/values.ci.yaml | 2 +- chart/values.production.yaml | 2 +- chart/values.schema.json | 4 +- chart/values.yaml | 12 +++--- 15 files changed, 75 insertions(+), 74 deletions(-) rename chart/templates/{deployment-engine.yaml => deployment-worker.yaml} (73%) rename chart/templates/{hpa-engine.yaml => hpa-worker.yaml} (54%) rename chart/templates/{pdb-engine.yaml => pdb-worker.yaml} (60%) rename chart/templates/{service-engine.yaml => service-worker.yaml} (65%) rename chart/templates/{serviceaccount-engine.yaml => serviceaccount-worker.yaml} (68%) diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl index abf6c4b..1c62de6 100644 --- a/chart/templates/_helpers.tpl +++ b/chart/templates/_helpers.tpl @@ -144,8 +144,8 @@ Service URLs {{- printf "http://%s" .Values.app.name -}} {{- end }} -{{- define "openops.engineServiceUrl" -}} -{{- printf "http://%s:3005" .Values.engine.name -}} +{{- define "openops.workerServiceUrl" -}} +{{- printf "http://%s:3000" .Values.worker.name -}} {{- end }} {{- define "openops.tablesServiceUrl" -}} diff --git a/chart/templates/deployment-app.yaml b/chart/templates/deployment-app.yaml index b19bb48..a6eb81b 100644 --- a/chart/templates/deployment-app.yaml +++ b/chart/templates/deployment-app.yaml @@ -74,6 +74,8 @@ spec: value: "--no-node-snapshot" - name: OPS_COMPONENT value: app + - name: OPS_CONTAINER_TYPE + value: APP {{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnv "skipDuplicateSecrets" true) | nindent 12 }} {{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnvSecrets) | nindent 12 }} ports: diff --git a/chart/templates/deployment-engine.yaml b/chart/templates/deployment-worker.yaml similarity index 73% rename from chart/templates/deployment-engine.yaml rename to chart/templates/deployment-worker.yaml index 6b567c6..a4c46e9 100644 --- a/chart/templates/deployment-engine.yaml +++ b/chart/templates/deployment-worker.yaml @@ -1,9 +1,9 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ .Values.engine.name }} + name: {{ .Values.worker.name }} labels: - {{- include "openops.componentLabels" (dict "root" . "component" "engine") | nindent 4 }} + {{- include "openops.componentLabels" (dict "root" . "component" "worker") | nindent 4 }} {{- with .Values.global.commonLabels }} {{- toYaml . | nindent 4 }} {{- end }} @@ -12,22 +12,22 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if not .Values.engine.autoscaling.enabled }} - replicas: {{ .Values.engine.replicas }} + {{- if not .Values.worker.autoscaling.enabled }} + replicas: {{ .Values.worker.replicas }} {{- end }} {{- include "openops.deploymentStrategy" . | nindent 2 }} selector: matchLabels: - {{- include "openops.componentSelectorLabels" (dict "root" . "component" "engine") | nindent 6 }} + {{- include "openops.componentSelectorLabels" (dict "root" . "component" "worker") | nindent 6 }} template: metadata: labels: - {{- include "openops.componentSelectorLabels" (dict "root" . "component" "engine") | nindent 8 }} + {{- include "openops.componentSelectorLabels" (dict "root" . "component" "worker") | nindent 8 }} {{- with .Values.global.commonLabels }} {{- toYaml . | nindent 8 }} {{- end }} annotations: - {{- include "openops.prometheusAnnotations" (dict "root" . "component" "engine") | nindent 8 }} + {{- include "openops.prometheusAnnotations" (dict "root" . "component" "worker") | nindent 8 }} {{- with include "openops.secretChecksum" . }} checksum/secret-env: {{ . }} {{- end }} @@ -35,34 +35,33 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - serviceAccountName: {{ include "openops.serviceAccountName" (dict "root" . "component" "engine") }} + serviceAccountName: {{ include "openops.serviceAccountName" (dict "root" . "component" "worker") }} {{- include "openops.imagePullSecrets" . | nindent 6 }} - {{- include "openops.podSecurityContext" (dict "root" . "component" "engine") | nindent 6 }} + {{- include "openops.podSecurityContext" (dict "root" . "component" "worker") | nindent 6 }} {{- include "openops.priorityClassName" . | nindent 6 }} {{- include "openops.terminationGracePeriodSeconds" . | nindent 6 }} - {{- include "openops.topologySpreadConstraints" (dict "root" . "component" "engine") | nindent 6 }} - {{- include "openops.affinity" (dict "root" . "component" "engine") | nindent 6 }} + {{- include "openops.topologySpreadConstraints" (dict "root" . "component" "worker") | nindent 6 }} + {{- include "openops.affinity" (dict "root" . "component" "worker") | nindent 6 }} {{- include "openops.nodeSelector" . | nindent 6 }} {{- include "openops.tolerations" . | nindent 6 }} - {{- include "openops.initContainers" (dict "root" . "component" "engine") | nindent 6 }} + {{- include "openops.initContainers" (dict "root" . "component" "worker") | nindent 6 }} containers: - - name: {{ .Values.engine.name }} - image: "{{ include "openops.imageRepository" (dict "root" . "component" "engine") }}/{{ .Values.engine.image }}:{{ .Values.global.version }}" + - name: {{ .Values.worker.name }} + image: "{{ include "openops.imageRepository" (dict "root" . "component" "worker") }}/{{ .Values.worker.image }}:{{ .Values.global.version }}" imagePullPolicy: {{ .Values.image.pullPolicy }} {{- include "openops.containerSecurityContext" (dict "root" .) | nindent 10 }} - command: [ "/bin/sh", "-c", "cp -r /var/tmp-base/. /tmp/ && node main.js" ] env: - - name: NODE_OPTIONS - value: "--no-node-snapshot" - name: OPS_COMPONENT - value: engine + value: worker + - name: OPS_CONTAINER_TYPE + value: WORKER {{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnv "skipDuplicateSecrets" true) | nindent 12 }} {{ include "openops.renderEnv" (dict "root" . "env" .Values.openopsEnvSecrets) | nindent 12 }} -{{ include "openops.renderEnv" (dict "root" . "env" .Values.engine.env) | nindent 12 }} +{{ include "openops.renderEnv" (dict "root" . "env" .Values.worker.env) | nindent 12 }} ports: - - containerPort: 3005 + - containerPort: 3000 name: http -{{- with .Values.engine.resources }} +{{- with .Values.worker.resources }} resources: {{ toYaml . | nindent 12 }} {{- end }} diff --git a/chart/templates/external-secret.yaml b/chart/templates/external-secret.yaml index 6d74d71..2030451 100644 --- a/chart/templates/external-secret.yaml +++ b/chart/templates/external-secret.yaml @@ -71,8 +71,8 @@ spec: {{- range $k, $v := .Values.tables.env }}{{ if not (hasKey $allEnv $k) }}{{ $_ := set $allEnv $k ($v | toString) }}{{ end }}{{ end }} {{- range $k, $v := .Values.analytics.env }}{{ if not (hasKey $allEnv $k) }}{{ $_ := set $allEnv $k ($v | toString) }}{{ end }}{{ end }} {{- range $k, $v := .Values.postgres.env }}{{ if not (hasKey $allEnv $k) }}{{ $_ := set $allEnv $k ($v | toString) }}{{ end }}{{ end }} -{{- if .Values.engine }}{{- if .Values.engine.env }} -{{- range $k, $v := .Values.engine.env }}{{ if not (hasKey $allEnv $k) }}{{ $_ := set $allEnv $k ($v | toString) }}{{ end }}{{ end }} +{{- if .Values.worker }}{{- if .Values.worker.env }} +{{- range $k, $v := .Values.worker.env }}{{ if not (hasKey $allEnv $k) }}{{ $_ := set $allEnv $k ($v | toString) }}{{ end }}{{ end }} {{- end }}{{- end }} {{- include "openops.collectSecretEntries" (dict "root" $ "env" $allEnv "secretName" .Values.externalSecrets.secretName) }} {{- end }} diff --git a/chart/templates/hpa-engine.yaml b/chart/templates/hpa-worker.yaml similarity index 54% rename from chart/templates/hpa-engine.yaml rename to chart/templates/hpa-worker.yaml index 608f9fc..1021e94 100644 --- a/chart/templates/hpa-engine.yaml +++ b/chart/templates/hpa-worker.yaml @@ -1,10 +1,10 @@ -{{- if .Values.engine.autoscaling.enabled }} +{{- if .Values.worker.autoscaling.enabled }} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: - name: {{ .Values.engine.name }}-hpa + name: {{ .Values.worker.name }}-hpa labels: - {{- include "openops.componentLabels" (dict "root" . "component" "engine") | nindent 4 }} + {{- include "openops.componentLabels" (dict "root" . "component" "worker") | nindent 4 }} {{- with .Values.global.commonLabels }} {{- toYaml . | nindent 4 }} {{- end }} @@ -12,24 +12,24 @@ spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ .Values.engine.name }} - minReplicas: {{ .Values.engine.autoscaling.minReplicas }} - maxReplicas: {{ .Values.engine.autoscaling.maxReplicas }} + name: {{ .Values.worker.name }} + minReplicas: {{ .Values.worker.autoscaling.minReplicas }} + maxReplicas: {{ .Values.worker.autoscaling.maxReplicas }} metrics: - {{- if .Values.engine.autoscaling.targetCPUUtilizationPercentage }} + {{- if .Values.worker.autoscaling.targetCPUUtilizationPercentage }} - type: Resource resource: name: cpu target: type: Utilization - averageUtilization: {{ .Values.engine.autoscaling.targetCPUUtilizationPercentage }} + averageUtilization: {{ .Values.worker.autoscaling.targetCPUUtilizationPercentage }} {{- end }} - {{- if .Values.engine.autoscaling.targetMemoryUtilizationPercentage }} + {{- if .Values.worker.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory target: type: Utilization - averageUtilization: {{ .Values.engine.autoscaling.targetMemoryUtilizationPercentage }} + averageUtilization: {{ .Values.worker.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} {{- end }} diff --git a/chart/templates/networkpolicy.yaml b/chart/templates/networkpolicy.yaml index c9dc2b1..6a4eccf 100644 --- a/chart/templates/networkpolicy.yaml +++ b/chart/templates/networkpolicy.yaml @@ -40,10 +40,10 @@ spec: - to: - podSelector: matchLabels: - app.kubernetes.io/component: engine + app.kubernetes.io/component: worker ports: - protocol: TCP - port: 3005 + port: 3000 - to: - podSelector: matchLabels: @@ -71,17 +71,17 @@ spec: - protocol: TCP port: 443 --- -# Network policy for engine +# Network policy for worker apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: {{ .Values.engine.name }}-netpol + name: {{ .Values.worker.name }}-netpol labels: - {{- include "openops.componentLabels" (dict "root" . "component" "engine") | nindent 4 }} + {{- include "openops.componentLabels" (dict "root" . "component" "worker") | nindent 4 }} spec: podSelector: matchLabels: - {{- include "openops.componentSelectorLabels" (dict "root" . "component" "engine") | nindent 6 }} + {{- include "openops.componentSelectorLabels" (dict "root" . "component" "worker") | nindent 6 }} policyTypes: - Ingress - Egress @@ -92,7 +92,7 @@ spec: app.kubernetes.io/component: app ports: - protocol: TCP - port: 3005 + port: 3000 egress: - to: - podSelector: @@ -306,7 +306,7 @@ spec: - from: - podSelector: matchLabels: - app.kubernetes.io/component: engine + app.kubernetes.io/component: worker ports: - protocol: TCP port: 5432 @@ -358,7 +358,7 @@ spec: - from: - podSelector: matchLabels: - app.kubernetes.io/component: engine + app.kubernetes.io/component: worker ports: - protocol: TCP port: 6379 diff --git a/chart/templates/pdb-engine.yaml b/chart/templates/pdb-worker.yaml similarity index 60% rename from chart/templates/pdb-engine.yaml rename to chart/templates/pdb-worker.yaml index b0e670e..17a54ac 100644 --- a/chart/templates/pdb-engine.yaml +++ b/chart/templates/pdb-worker.yaml @@ -1,16 +1,16 @@ -{{- if .Values.engine.podDisruptionBudget.enabled }} +{{- if .Values.worker.podDisruptionBudget.enabled }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - name: {{ .Values.engine.name }}-pdb + name: {{ .Values.worker.name }}-pdb labels: - {{- include "openops.componentLabels" (dict "root" . "component" "engine") | nindent 4 }} + {{- include "openops.componentLabels" (dict "root" . "component" "worker") | nindent 4 }} {{- with .Values.global.commonLabels }} {{- toYaml . | nindent 4 }} {{- end }} spec: - minAvailable: {{ .Values.engine.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.worker.podDisruptionBudget.minAvailable }} selector: matchLabels: - {{- include "openops.componentSelectorLabels" (dict "root" . "component" "engine") | nindent 6 }} + {{- include "openops.componentSelectorLabels" (dict "root" . "component" "worker") | nindent 6 }} {{- end }} diff --git a/chart/templates/secret-env.yaml b/chart/templates/secret-env.yaml index 7d50968..926b7a2 100644 --- a/chart/templates/secret-env.yaml +++ b/chart/templates/secret-env.yaml @@ -10,7 +10,7 @@ {{- $existingSecret := default "" $secretSettings.existingSecret -}} {{- if and $create (not $existingSecret) -}} {{- $autoSecretData := dict -}} -{{- $envSources := list .Values.openopsEnvSecrets .Values.openopsEnv .Values.engine.env .Values.tables.env .Values.analytics.env .Values.postgres.env -}} +{{- $envSources := list .Values.openopsEnvSecrets .Values.openopsEnv .Values.worker.env .Values.tables.env .Values.analytics.env .Values.postgres.env -}} {{- range $env := $envSources }} {{- if $env }} {{- range $k, $v := $env }} diff --git a/chart/templates/service-engine.yaml b/chart/templates/service-worker.yaml similarity index 65% rename from chart/templates/service-engine.yaml rename to chart/templates/service-worker.yaml index 8fa522d..0fda388 100644 --- a/chart/templates/service-engine.yaml +++ b/chart/templates/service-worker.yaml @@ -1,15 +1,15 @@ apiVersion: v1 kind: Service metadata: - name: {{ .Values.engine.name }} + name: {{ .Values.worker.name }} labels: - {{- include "openops.componentLabels" (dict "root" . "component" "engine") | nindent 4 }} + {{- include "openops.componentLabels" (dict "root" . "component" "worker") | nindent 4 }} spec: type: ClusterIP ports: - - port: 3005 - targetPort: 3005 + - port: 3000 + targetPort: 3000 protocol: TCP name: http selector: - {{- include "openops.componentSelectorLabels" (dict "root" . "component" "engine") | nindent 4 }} \ No newline at end of file + {{- include "openops.componentSelectorLabels" (dict "root" . "component" "worker") | nindent 4 }} \ No newline at end of file diff --git a/chart/templates/serviceaccount-engine.yaml b/chart/templates/serviceaccount-worker.yaml similarity index 68% rename from chart/templates/serviceaccount-engine.yaml rename to chart/templates/serviceaccount-worker.yaml index 461b0bf..2d1dfdb 100644 --- a/chart/templates/serviceaccount-engine.yaml +++ b/chart/templates/serviceaccount-worker.yaml @@ -1,14 +1,14 @@ -{{- if .Values.engine.serviceAccount.create }} +{{- if .Values.worker.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "openops.serviceAccountName" (dict "root" . "component" "engine") }} + name: {{ include "openops.serviceAccountName" (dict "root" . "component" "worker") }} labels: - {{- include "openops.componentLabels" (dict "root" . "component" "engine") | nindent 4 }} + {{- include "openops.componentLabels" (dict "root" . "component" "worker") | nindent 4 }} {{- with .Values.global.commonLabels }} {{- toYaml . | nindent 4 }} {{- end }} - {{- with .Values.engine.serviceAccount.annotations }} + {{- with .Values.worker.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} diff --git a/chart/templates/servicemonitor.yaml b/chart/templates/servicemonitor.yaml index ff16021..94cda23 100644 --- a/chart/templates/servicemonitor.yaml +++ b/chart/templates/servicemonitor.yaml @@ -20,23 +20,23 @@ spec: interval: {{ .Values.serviceMonitor.interval }} scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} --- -# ServiceMonitor for engine +# ServiceMonitor for worker apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: {{ .Values.engine.name }} + name: {{ .Values.worker.name }} labels: - {{- include "openops.componentLabels" (dict "root" . "component" "engine") | nindent 4 }} + {{- include "openops.componentLabels" (dict "root" . "component" "worker") | nindent 4 }} {{- with .Values.serviceMonitor.labels }} {{- toYaml . | nindent 4 }} {{- end }} spec: selector: matchLabels: - {{- include "openops.componentSelectorLabels" (dict "root" . "component" "engine") | nindent 6 }} + {{- include "openops.componentSelectorLabels" (dict "root" . "component" "worker") | nindent 6 }} endpoints: - port: http - path: {{ .Values.engine.metrics.path }} + path: {{ .Values.worker.metrics.path }} interval: {{ .Values.serviceMonitor.interval }} scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} --- diff --git a/chart/values.ci.yaml b/chart/values.ci.yaml index dd1fb2f..41eef3c 100644 --- a/chart/values.ci.yaml +++ b/chart/values.ci.yaml @@ -41,7 +41,7 @@ app: cpu: "500m" memory: "1Gi" -engine: +worker: replicas: 1 podDisruptionBudget: enabled: false diff --git a/chart/values.production.yaml b/chart/values.production.yaml index f88c63d..e9fd02a 100644 --- a/chart/values.production.yaml +++ b/chart/values.production.yaml @@ -64,7 +64,7 @@ app: memory: "4Gi" cpu: "2000m" -engine: +worker: replicas: 3 resources: requests: diff --git a/chart/values.schema.json b/chart/values.schema.json index fa7f547..1d2df46 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -55,7 +55,7 @@ } } }, - "engine": { + "worker": { "type": "object", "required": ["name", "image", "replicas"], "properties": { @@ -67,7 +67,7 @@ }, "repository": { "type": "string", - "description": "Override global image.repository for engine (empty = use global)" + "description": "Override global image.repository for worker (empty = use global)" }, "replicas": { "type": "integer", diff --git a/chart/values.yaml b/chart/values.yaml index 54d5790..6c55908 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -106,8 +106,8 @@ openopsEnv: OPS_TELEMETRY_MODE: COLLECTOR OPS_TELEMETRY_COLLECTOR_URL: https://telemetry.openops.com/save - # Engine - OPS_ENGINE_URL: '{{ include "openops.engineServiceUrl" . }}/execute' + # Worker + OPS_WORKER_URL: '{{ include "openops.workerServiceUrl" . }}' OPS_EXECUTION_MODE: SANDBOX_CODE_ONLY # Queue @@ -214,9 +214,9 @@ app: port: 8080 path: /metrics -engine: - name: openops-engine - image: openops-engine +worker: + name: openops-worker + image: openops-worker repository: "" # Override global image.repository (empty = use global) replicas: 2 # Minimum 2 for high availability resources: @@ -598,7 +598,7 @@ ingress: - paths: - path: / pathType: Prefix - # serviceName must match a service defined in this chart (nginx, app, engine, tables, analytics) + # serviceName must match a service defined in this chart (nginx, app, worker, tables, analytics) serviceName: nginx servicePort: 80 # TLS-related settings below apply only to nginx ingress controller From 4b953cb2f230211b0385c0a7c46274715738df65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Gon=C3=A7alves?= Date: Thu, 13 Aug 2026 12:26:34 +0100 Subject: [PATCH 2/4] Use maxUnavailable for PodDisruptionBudgets Every PDB in the chart set minAvailable, while analytics and tables default to a single replica. minAvailable: 1 against one replica evaluates to disruptionsAllowed: 0, so those pods cannot be evicted and every voluntary disruption fails: cluster upgrades, node image upgrades and autoscaler scale-down all stall while it is set. This is not theoretical. It left an Azure AKS estate on three-month-old node images across four clusters, with the pools reporting Failed and nothing alerting on it, because the drain could never complete. maxUnavailable: 1 is equivalent at two replicas and drainable at one, so all five components now default to it rather than only fixing the two that deadlock today. Overriding replicas down to 1 is a supported thing to do and should not reintroduce the deadlock. minAvailable still works when maxUnavailable is unset, so existing overrides are unaffected. A PDB may not set both; maxUnavailable takes precedence. One behaviour change to note: at three or more replicas maxUnavailable: 1 permits one pod down at a time where minAvailable: 1 permitted all but one. That is safer but makes drains slower. tables uses ReadWriteOnce storage and cannot be scaled past one replica, so it still incurs brief downtime while its node drains. This makes the drain possible, not seamless. Verified by rendering the chart: all five PDBs emit maxUnavailable: 1 with default values, and an override of maxUnavailable: null with minAvailable: 2 still emits minAvailable: 2. helm lint and both CI template steps pass. Note that CI never exercises this path, since values.ci.yaml disables PDBs. Part of OPS-4725 --- AGENTS.md | 2 +- README.md | 51 ++++++++++++++++++++++-------- chart/templates/pdb-analytics.yaml | 6 +++- chart/templates/pdb-app.yaml | 6 +++- chart/templates/pdb-engine.yaml | 6 +++- chart/templates/pdb-nginx.yaml | 6 +++- chart/templates/pdb-tables.yaml | 6 +++- chart/values.yaml | 10 +++--- docs/DEPLOY_TO_AWS_EKS.md | 27 +++++++++------- 9 files changed, 84 insertions(+), 36 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1e59fe0..88a6057 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ - **Security-first design**: Security contexts enabled by default (runAsNonRoot, drop ALL capabilities, seccomp RuntimeDefault profile). - **Service accounts**: Dedicated service accounts for each component (app, engine, tables, analytics, nginx, postgres, redis) with configurable annotations for AWS IAM roles (IRSA), GCP Workload Identity, or Azure Managed Identity. - **External Secrets Operator**: Built-in support for AWS Secrets Manager, HashiCorp Vault, GCP Secret Manager, and Azure Key Vault integration. -- **PodDisruptionBudgets (PDBs)**: Configured for all stateless components to ensure minimum availability during voluntary disruptions (node drains, upgrades). +- **PodDisruptionBudgets (PDBs)**: Configured for all stateless components, defaulting to `maxUnavailable: 1` so voluntary disruptions (node drains, upgrades) can always proceed. Each component also accepts `minAvailable` when `maxUnavailable` is unset, but avoid it where a component runs a single replica: `minAvailable: 1` then evaluates to `disruptionsAllowed: 0` and blocks every drain. `analytics` and `tables` default to one replica. - **HorizontalPodAutoscalers (HPAs)**: Optional autoscaling for app, engine, analytics, and nginx based on CPU/memory metrics. - **NetworkPolicy**: Optional network segmentation to restrict pod-to-pod communication and enforce least-privilege networking with explicit allow rules. - **LimitRange**: Optional namespace-level resource defaults and constraints to prevent resource exhaustion. diff --git a/README.md b/README.md index 3dc1650..da628b9 100644 --- a/README.md +++ b/README.md @@ -829,26 +829,51 @@ global: topologyKey: topology.kubernetes.io/zone whenUnsatisfiable: DoNotSchedule -# Enable PodDisruptionBudgets -pdb: - enabled: true - app: +# PodDisruptionBudgets are enabled by default with maxUnavailable: 1 +app: + podDisruptionBudget: enabled: true - minAvailable: 2 - engine: + maxUnavailable: 1 +engine: + podDisruptionBudget: enabled: true - minAvailable: 2 - nginx: + maxUnavailable: 1 +nginx: + podDisruptionBudget: enabled: true - minAvailable: 1 - analytics: + maxUnavailable: 1 +analytics: + podDisruptionBudget: enabled: true - minAvailable: 1 - tables: + maxUnavailable: 1 +tables: + podDisruptionBudget: enabled: true - minAvailable: 1 + maxUnavailable: 1 +``` + +Each component takes either `maxUnavailable` or `minAvailable` — a PodDisruptionBudget +may not set both, and `maxUnavailable` wins if you set both. + +Prefer `maxUnavailable`. `minAvailable: 1` on a component running a single replica +evaluates to `disruptionsAllowed: 0`, which makes the pod impossible to evict and blocks +every node drain — cluster upgrades, node image upgrades and autoscaler scale-down all +fail while it is set. `analytics` and `tables` default to one replica, and `tables` uses +`ReadWriteOnce` storage so it cannot be scaled out of the problem. + +To use `minAvailable` instead, unset `maxUnavailable` explicitly: + +```yaml +app: + podDisruptionBudget: + maxUnavailable: null + minAvailable: 2 ``` +Note that `maxUnavailable: 1` allows only one pod down at a time. At three or more +replicas that is stricter than `minAvailable: 1`, which permits all but one to go at +once — safer, but drains take longer. + ### Monitoring and observability **Prometheus metrics:** diff --git a/chart/templates/pdb-analytics.yaml b/chart/templates/pdb-analytics.yaml index 369f90d..85b74ba 100644 --- a/chart/templates/pdb-analytics.yaml +++ b/chart/templates/pdb-analytics.yaml @@ -9,7 +9,11 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - minAvailable: {{ .Values.analytics.podDisruptionBudget.minAvailable }} + {{- if .Values.analytics.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.analytics.podDisruptionBudget.maxUnavailable }} + {{- else }} + minAvailable: {{ .Values.analytics.podDisruptionBudget.minAvailable | default 1 }} + {{- end }} selector: matchLabels: {{- include "openops.componentSelectorLabels" (dict "root" . "component" "analytics") | nindent 6 }} diff --git a/chart/templates/pdb-app.yaml b/chart/templates/pdb-app.yaml index 12f9064..5180d26 100644 --- a/chart/templates/pdb-app.yaml +++ b/chart/templates/pdb-app.yaml @@ -9,7 +9,11 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - minAvailable: {{ .Values.app.podDisruptionBudget.minAvailable }} + {{- if .Values.app.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.app.podDisruptionBudget.maxUnavailable }} + {{- else }} + minAvailable: {{ .Values.app.podDisruptionBudget.minAvailable | default 1 }} + {{- end }} selector: matchLabels: {{- include "openops.componentSelectorLabels" (dict "root" . "component" "app") | nindent 6 }} diff --git a/chart/templates/pdb-engine.yaml b/chart/templates/pdb-engine.yaml index b0e670e..51b82aa 100644 --- a/chart/templates/pdb-engine.yaml +++ b/chart/templates/pdb-engine.yaml @@ -9,7 +9,11 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - minAvailable: {{ .Values.engine.podDisruptionBudget.minAvailable }} + {{- if .Values.engine.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.engine.podDisruptionBudget.maxUnavailable }} + {{- else }} + minAvailable: {{ .Values.engine.podDisruptionBudget.minAvailable | default 1 }} + {{- end }} selector: matchLabels: {{- include "openops.componentSelectorLabels" (dict "root" . "component" "engine") | nindent 6 }} diff --git a/chart/templates/pdb-nginx.yaml b/chart/templates/pdb-nginx.yaml index d452c88..85bf478 100644 --- a/chart/templates/pdb-nginx.yaml +++ b/chart/templates/pdb-nginx.yaml @@ -9,7 +9,11 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - minAvailable: {{ .Values.nginx.podDisruptionBudget.minAvailable }} + {{- if .Values.nginx.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.nginx.podDisruptionBudget.maxUnavailable }} + {{- else }} + minAvailable: {{ .Values.nginx.podDisruptionBudget.minAvailable | default 1 }} + {{- end }} selector: matchLabels: {{- include "openops.componentSelectorLabels" (dict "root" . "component" "nginx") | nindent 6 }} diff --git a/chart/templates/pdb-tables.yaml b/chart/templates/pdb-tables.yaml index b3a7cfa..368bf27 100644 --- a/chart/templates/pdb-tables.yaml +++ b/chart/templates/pdb-tables.yaml @@ -9,7 +9,11 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - minAvailable: {{ .Values.tables.podDisruptionBudget.minAvailable }} + {{- if .Values.tables.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.tables.podDisruptionBudget.maxUnavailable }} + {{- else }} + minAvailable: {{ .Values.tables.podDisruptionBudget.minAvailable | default 1 }} + {{- end }} selector: matchLabels: {{- include "openops.componentSelectorLabels" (dict "root" . "component" "tables") | nindent 6 }} diff --git a/chart/values.yaml b/chart/values.yaml index 139198f..ef6990a 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -193,7 +193,7 @@ app: # Pod Disruption Budget - ENABLED BY DEFAULT podDisruptionBudget: enabled: true - minAvailable: 1 + maxUnavailable: 1 # Horizontal Pod Autoscaler autoscaling: enabled: false @@ -229,7 +229,7 @@ engine: # Pod Disruption Budget - ENABLED BY DEFAULT podDisruptionBudget: enabled: true - minAvailable: 1 + maxUnavailable: 1 # Horizontal Pod Autoscaler autoscaling: enabled: false @@ -269,7 +269,7 @@ tables: # Pod Disruption Budget podDisruptionBudget: enabled: true - minAvailable: 1 + maxUnavailable: 1 # Horizontal Pod Autoscaler autoscaling: enabled: false @@ -332,7 +332,7 @@ analytics: # Pod Disruption Budget podDisruptionBudget: enabled: true - minAvailable: 1 + maxUnavailable: 1 # Horizontal Pod Autoscaler autoscaling: enabled: false @@ -545,7 +545,7 @@ nginx: # Pod Disruption Budget - ENABLED BY DEFAULT podDisruptionBudget: enabled: true - minAvailable: 1 + maxUnavailable: 1 # Horizontal Pod Autoscaler autoscaling: enabled: false diff --git a/docs/DEPLOY_TO_AWS_EKS.md b/docs/DEPLOY_TO_AWS_EKS.md index 3d17f15..254ca3c 100644 --- a/docs/DEPLOY_TO_AWS_EKS.md +++ b/docs/DEPLOY_TO_AWS_EKS.md @@ -751,23 +751,26 @@ nginx: # service.beta.kubernetes.io/aws-load-balancer-internal: "true" # Pod Disruption Budgets for HA -pdb: - enabled: true - app: +app: + podDisruptionBudget: enabled: true - minAvailable: 2 - engine: + maxUnavailable: 1 +engine: + podDisruptionBudget: enabled: true - minAvailable: 2 - nginx: + maxUnavailable: 1 +nginx: + podDisruptionBudget: enabled: true - minAvailable: 1 - analytics: + maxUnavailable: 1 +analytics: + podDisruptionBudget: enabled: true - minAvailable: 1 - tables: + maxUnavailable: 1 +tables: + podDisruptionBudget: enabled: true - minAvailable: 1 + maxUnavailable: 1 # Horizontal Pod Autoscaling hpa: From 5bcde25174365e73ced453a269fa26039a66ac48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Gon=C3=A7alves?= Date: Thu, 13 Aug 2026 12:36:30 +0100 Subject: [PATCH 3/4] Respect an explicit maxUnavailable of zero MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The template guarded on truthiness, and Go templates treat 0 as false, so maxUnavailable: 0 fell through to rendering minAvailable: 1 — silently producing a PDB that differs from the values that asked for it. 0 is a valid PodDisruptionBudget value. kindIs "invalid" tests for nil instead, which keeps all four cases correct: an explicit 0 renders as 0, an explicit null falls back to minAvailable, an absent key falls back, and a set value renders. hasKey would not work here, since it is true for maxUnavailable: null and would render an empty field. Also correct two documentation errors. The README claimed a PDB "may not set both" fields and then that maxUnavailable "wins if you set both", conflating the rendered resource with the values schema. AGENTS.md described PDBs as covering "all stateless components" while tables, which has one, is stateful. Part of OPS-4725 --- AGENTS.md | 2 +- README.md | 7 ++++--- chart/templates/pdb-analytics.yaml | 2 +- chart/templates/pdb-app.yaml | 2 +- chart/templates/pdb-engine.yaml | 2 +- chart/templates/pdb-nginx.yaml | 2 +- chart/templates/pdb-tables.yaml | 2 +- 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 88a6057..7e5dee5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ - **Security-first design**: Security contexts enabled by default (runAsNonRoot, drop ALL capabilities, seccomp RuntimeDefault profile). - **Service accounts**: Dedicated service accounts for each component (app, engine, tables, analytics, nginx, postgres, redis) with configurable annotations for AWS IAM roles (IRSA), GCP Workload Identity, or Azure Managed Identity. - **External Secrets Operator**: Built-in support for AWS Secrets Manager, HashiCorp Vault, GCP Secret Manager, and Azure Key Vault integration. -- **PodDisruptionBudgets (PDBs)**: Configured for all stateless components, defaulting to `maxUnavailable: 1` so voluntary disruptions (node drains, upgrades) can always proceed. Each component also accepts `minAvailable` when `maxUnavailable` is unset, but avoid it where a component runs a single replica: `minAvailable: 1` then evaluates to `disruptionsAllowed: 0` and blocks every drain. `analytics` and `tables` default to one replica. +- **PodDisruptionBudgets (PDBs)**: Configured for `app`, `engine`, `nginx`, `analytics` and `tables` — note that `tables` is stateful (PVC, ReadWriteOnce), while the Postgres and Redis StatefulSets have no PDB. All default to `maxUnavailable: 1` so voluntary disruptions (node drains, upgrades) can always proceed. Each component also accepts `minAvailable` when `maxUnavailable` is unset, but avoid it where a component runs a single replica: `minAvailable: 1` then evaluates to `disruptionsAllowed: 0` and blocks every drain. `analytics` and `tables` default to one replica. - **HorizontalPodAutoscalers (HPAs)**: Optional autoscaling for app, engine, analytics, and nginx based on CPU/memory metrics. - **NetworkPolicy**: Optional network segmentation to restrict pod-to-pod communication and enforce least-privilege networking with explicit allow rules. - **LimitRange**: Optional namespace-level resource defaults and constraints to prevent resource exhaustion. diff --git a/README.md b/README.md index da628b9..b58f20c 100644 --- a/README.md +++ b/README.md @@ -852,8 +852,9 @@ tables: maxUnavailable: 1 ``` -Each component takes either `maxUnavailable` or `minAvailable` — a PodDisruptionBudget -may not set both, and `maxUnavailable` wins if you set both. +A PodDisruptionBudget resource carries only one of the two fields. The chart renders +`maxUnavailable` whenever that value is non-nil, and `minAvailable` otherwise; +`maxUnavailable: 0` counts as set and is rendered as `0`. Prefer `maxUnavailable`. `minAvailable: 1` on a component running a single replica evaluates to `disruptionsAllowed: 0`, which makes the pod impossible to evict and blocks @@ -861,7 +862,7 @@ every node drain — cluster upgrades, node image upgrades and autoscaler scale- fail while it is set. `analytics` and `tables` default to one replica, and `tables` uses `ReadWriteOnce` storage so it cannot be scaled out of the problem. -To use `minAvailable` instead, unset `maxUnavailable` explicitly: +To use `minAvailable` instead, set `maxUnavailable` to `null` explicitly: ```yaml app: diff --git a/chart/templates/pdb-analytics.yaml b/chart/templates/pdb-analytics.yaml index 85b74ba..87245ad 100644 --- a/chart/templates/pdb-analytics.yaml +++ b/chart/templates/pdb-analytics.yaml @@ -9,7 +9,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if .Values.analytics.podDisruptionBudget.maxUnavailable }} + {{- if not (kindIs "invalid" .Values.analytics.podDisruptionBudget.maxUnavailable) }} maxUnavailable: {{ .Values.analytics.podDisruptionBudget.maxUnavailable }} {{- else }} minAvailable: {{ .Values.analytics.podDisruptionBudget.minAvailable | default 1 }} diff --git a/chart/templates/pdb-app.yaml b/chart/templates/pdb-app.yaml index 5180d26..6df94f4 100644 --- a/chart/templates/pdb-app.yaml +++ b/chart/templates/pdb-app.yaml @@ -9,7 +9,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if .Values.app.podDisruptionBudget.maxUnavailable }} + {{- if not (kindIs "invalid" .Values.app.podDisruptionBudget.maxUnavailable) }} maxUnavailable: {{ .Values.app.podDisruptionBudget.maxUnavailable }} {{- else }} minAvailable: {{ .Values.app.podDisruptionBudget.minAvailable | default 1 }} diff --git a/chart/templates/pdb-engine.yaml b/chart/templates/pdb-engine.yaml index 51b82aa..e3a2829 100644 --- a/chart/templates/pdb-engine.yaml +++ b/chart/templates/pdb-engine.yaml @@ -9,7 +9,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if .Values.engine.podDisruptionBudget.maxUnavailable }} + {{- if not (kindIs "invalid" .Values.engine.podDisruptionBudget.maxUnavailable) }} maxUnavailable: {{ .Values.engine.podDisruptionBudget.maxUnavailable }} {{- else }} minAvailable: {{ .Values.engine.podDisruptionBudget.minAvailable | default 1 }} diff --git a/chart/templates/pdb-nginx.yaml b/chart/templates/pdb-nginx.yaml index 85bf478..261a6d9 100644 --- a/chart/templates/pdb-nginx.yaml +++ b/chart/templates/pdb-nginx.yaml @@ -9,7 +9,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if .Values.nginx.podDisruptionBudget.maxUnavailable }} + {{- if not (kindIs "invalid" .Values.nginx.podDisruptionBudget.maxUnavailable) }} maxUnavailable: {{ .Values.nginx.podDisruptionBudget.maxUnavailable }} {{- else }} minAvailable: {{ .Values.nginx.podDisruptionBudget.minAvailable | default 1 }} diff --git a/chart/templates/pdb-tables.yaml b/chart/templates/pdb-tables.yaml index 368bf27..fe63e3b 100644 --- a/chart/templates/pdb-tables.yaml +++ b/chart/templates/pdb-tables.yaml @@ -9,7 +9,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if .Values.tables.podDisruptionBudget.maxUnavailable }} + {{- if not (kindIs "invalid" .Values.tables.podDisruptionBudget.maxUnavailable) }} maxUnavailable: {{ .Values.tables.podDisruptionBudget.maxUnavailable }} {{- else }} minAvailable: {{ .Values.tables.podDisruptionBudget.minAvailable | default 1 }} From 1118c7998202a95d3b340c089dadff8b77657729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Gon=C3=A7alves?= Date: Thu, 13 Aug 2026 12:47:29 +0100 Subject: [PATCH 4/4] Fix lint issues --- chart/templates/pdb-worker.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chart/templates/pdb-worker.yaml b/chart/templates/pdb-worker.yaml index 91cc646..60d023e 100644 --- a/chart/templates/pdb-worker.yaml +++ b/chart/templates/pdb-worker.yaml @@ -9,10 +9,10 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if not (kindIs "invalid" .Values.engine.podDisruptionBudget.maxUnavailable) }} - maxUnavailable: {{ .Values.engine.podDisruptionBudget.maxUnavailable }} + {{- if not (kindIs "invalid" .Values.worker.podDisruptionBudget.maxUnavailable) }} + maxUnavailable: {{ .Values.worker.podDisruptionBudget.maxUnavailable }} {{- else }} - minAvailable: {{ .Values.engine.podDisruptionBudget.minAvailable | default 1 }} + minAvailable: {{ .Values.worker.podDisruptionBudget.minAvailable | default 1 }} {{- end }} selector: matchLabels: