From fba28c9138da6539b03d16c0e45c1dcb38a2005e Mon Sep 17 00:00:00 2001 From: bennerv <10840174+bennerv@users.noreply.github.com> Date: Thu, 9 Apr 2026 16:14:07 -0400 Subject: [PATCH] ARO-25877: Add periodic cleanup job for expired ARO-HCP app registrations Add new cluster profiles (aro-hcp-red-hat-tenant, aro-hcp-msft-test-tenant) and a periodic job to clean up orphaned app registrations with expired credentials left by e2e test runs. Depends on openshift/ci-tools#5097 for cluster profile registration. Co-Authored-By: Claude Opus 4.6 --- .../ARO-HCP/Azure-ARO-HCP-main__periodic.yaml | 22 +++ .../ARO-HCP/Azure-ARO-HCP-main-periodics.yaml | 146 ++++++++++++++++++ .../expired-app-registrations/OWNERS | 14 ++ ...sion-expired-app-registrations-commands.sh | 16 ++ ...xpired-app-registrations-ref.metadata.json | 21 +++ ...ovision-expired-app-registrations-ref.yaml | 22 +++ core-services/prow/02_config/_boskos.yaml | 8 + .../prow/02_config/generate-boskos.py | 6 + 8 files changed, 255 insertions(+) create mode 100644 ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/OWNERS create mode 100644 ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/aro-hcp-deprovision-expired-app-registrations-commands.sh create mode 100644 ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/aro-hcp-deprovision-expired-app-registrations-ref.metadata.json create mode 100644 ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/aro-hcp-deprovision-expired-app-registrations-ref.yaml diff --git a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main__periodic.yaml b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main__periodic.yaml index afa328efc59fe..da3fbb9a79e2c 100644 --- a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main__periodic.yaml +++ b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main__periodic.yaml @@ -18,6 +18,28 @@ tests: steps: test: - ref: aro-hcp-deprovision-kusto-role-assignments +- as: delete-expired-red-hat-tenant-app-registrations + cron: 0 5 * * * + steps: + env: + VAULT_SECRET_PROFILE: red-hat-tenant + leases: + - count: 1 + env: ENV_QUOTA_LEASED_RESOURCE + resource_type: aro-hcp-red-hat-tenant-quota-slice + test: + - ref: aro-hcp-deprovision-expired-app-registrations +- as: delete-expired-msft-test-tenant-app-registrations + cron: 0 5 * * * + steps: + env: + VAULT_SECRET_PROFILE: msft-test-tenant + leases: + - count: 1 + env: ENV_QUOTA_LEASED_RESOURCE + resource_type: aro-hcp-msft-test-tenant-quota-slice + test: + - ref: aro-hcp-deprovision-expired-app-registrations - as: delete-expired-development-resource-groups cron: 7,37 * * * * steps: diff --git a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-periodics.yaml b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-periodics.yaml index 788d8f81c49df..c8737c9ecdd4c 100644 --- a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-periodics.yaml +++ b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-periodics.yaml @@ -307,6 +307,79 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build09 + cron: 0 5 * * * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: main + org: Azure + repo: ARO-HCP + labels: + ci-operator.openshift.io/variant: periodic + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-Azure-ARO-HCP-main-periodic-delete-expired-msft-test-tenant-app-registrations + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=delete-expired-msft-test-tenant-app-registrations + - --variant=periodic + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build09 cron: 7,37 * * * * @@ -388,6 +461,79 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build09 + cron: 0 5 * * * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: main + org: Azure + repo: ARO-HCP + labels: + ci-operator.openshift.io/variant: periodic + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-Azure-ARO-HCP-main-periodic-delete-expired-red-hat-tenant-app-registrations + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=delete-expired-red-hat-tenant-app-registrations + - --variant=periodic + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build09 cron: 7,37 * * * * diff --git a/ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/OWNERS b/ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/OWNERS new file mode 100644 index 0000000000000..ef1de966b8267 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/OWNERS @@ -0,0 +1,14 @@ +approvers: +- geoberle +- jharrington22 +- mmazur +- roivaz +- venkateshsredhat +- deads2k +reviewers: +- geoberle +- jharrington22 +- mmazur +- roivaz +- venkateshsredhat +- deads2k diff --git a/ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/aro-hcp-deprovision-expired-app-registrations-commands.sh b/ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/aro-hcp-deprovision-expired-app-registrations-commands.sh new file mode 100644 index 0000000000000..2ddde319aa53d --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/aro-hcp-deprovision-expired-app-registrations-commands.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}" + +# Disable tracing due to credential handling +export AZURE_CLIENT_ID; AZURE_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/client-id") +export AZURE_TENANT_ID; AZURE_TENANT_ID=$(cat "${CLUSTER_PROFILE_DIR}/tenant") +export AZURE_CLIENT_SECRET; AZURE_CLIENT_SECRET=$(cat "${CLUSTER_PROFILE_DIR}/client-secret") +export AZURE_TOKEN_CREDENTIALS=prod + +set -o xtrace + +./test/aro-hcp-tests cleanup app-registrations diff --git a/ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/aro-hcp-deprovision-expired-app-registrations-ref.metadata.json b/ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/aro-hcp-deprovision-expired-app-registrations-ref.metadata.json new file mode 100644 index 0000000000000..06b47ee0eae0b --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/aro-hcp-deprovision-expired-app-registrations-ref.metadata.json @@ -0,0 +1,21 @@ +{ + "path": "aro-hcp/deprovision/expired-app-registrations/aro-hcp-deprovision-expired-app-registrations-ref.yaml", + "owners": { + "approvers": [ + "geoberle", + "jharrington22", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ], + "reviewers": [ + "geoberle", + "jharrington22", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/aro-hcp-deprovision-expired-app-registrations-ref.yaml b/ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/aro-hcp-deprovision-expired-app-registrations-ref.yaml new file mode 100644 index 0000000000000..3ddec4e3486ea --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/deprovision/expired-app-registrations/aro-hcp-deprovision-expired-app-registrations-ref.yaml @@ -0,0 +1,22 @@ +ref: + as: aro-hcp-deprovision-expired-app-registrations + from: aro-hcp-e2e-tools + commands: aro-hcp-deprovision-expired-app-registrations-commands.sh + resources: + requests: + cpu: 100m + memory: 300Mi + credentials: + - namespace: test-credentials + name: cluster-secrets-aro-hcp-red-hat-tenant + mount_path: /var/run/aro-hcp-red-hat-tenant + - namespace: test-credentials + name: cluster-secrets-aro-hcp-msft-test-tenant + mount_path: /var/run/aro-hcp-msft-test-tenant + env: + - name: VAULT_SECRET_PROFILE + default: "red-hat-tenant" + documentation: |- + Selects which environment's cluster secrets to use (red-hat-tenant, msft-test-tenant). + documentation: |- + Clean up expired e2e app registrations that were left by test runs. diff --git a/core-services/prow/02_config/_boskos.yaml b/core-services/prow/02_config/_boskos.yaml index ed6be9fb12858..d23f3a3470dc4 100644 --- a/core-services/prow/02_config/_boskos.yaml +++ b/core-services/prow/02_config/_boskos.yaml @@ -67,6 +67,10 @@ resources: min-count: 1 state: free type: aro-hcp-int-quota-slice +- max-count: 1 + min-count: 1 + state: free + type: aro-hcp-msft-test-tenant-quota-slice - names: - aro-hcp-msi-mock-cs-sp-dev-0 - aro-hcp-msi-mock-cs-sp-dev-1 @@ -94,6 +98,10 @@ resources: min-count: 10 state: free type: aro-hcp-prod-quota-slice +- max-count: 1 + min-count: 1 + state: free + type: aro-hcp-red-hat-tenant-quota-slice - max-count: 1 min-count: 1 state: free diff --git a/core-services/prow/02_config/generate-boskos.py b/core-services/prow/02_config/generate-boskos.py index 3997256e8a626..857e4ee2e1dc8 100755 --- a/core-services/prow/02_config/generate-boskos.py +++ b/core-services/prow/02_config/generate-boskos.py @@ -291,6 +291,12 @@ 'aro-hcp-dev-global-pipeline-quota-slice': { 'default': 1, }, + 'aro-hcp-red-hat-tenant-quota-slice': { + 'default': 1, + }, + 'aro-hcp-msft-test-tenant-quota-slice': { + 'default': 1, + }, 'aro-hcp-test-msi-containers-dev': {}, 'aro-hcp-test-msi-containers-int': {}, 'aro-hcp-test-msi-containers-stg': {},