From f31daf4f6ac72067af60abfff80ff649a8085c11 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Thu, 16 Jul 2026 16:04:21 +0300 Subject: [PATCH 1/2] K8SPG-691: add repo host autoscaling https://perconadev.atlassian.net/browse/K8SPG-691 --- .../pgv2.percona.com_perconapgclusters.yaml | 4 + .../pgv2.percona.com_perconapgclusters.yaml | 4 + ...eam.pgv2.percona.com_postgresclusters.yaml | 4 + deploy/bundle.yaml | 8 + deploy/crd.yaml | 8 + deploy/cw-bundle.yaml | 8 + e2e-tests/run-pr.csv | 1 + e2e-tests/run-release.csv | 1 + .../tests/repo-host-autogrow/00-assert.yaml | 7 + .../00-deploy-operator.yaml | 12 ++ .../tests/repo-host-autogrow/01-assert.yaml | 17 ++ .../repo-host-autogrow/01-create-cluster.yaml | 17 ++ .../tests/repo-host-autogrow/02-assert.yaml | 42 +++++ .../02-grow-repo-volume.yaml | 33 ++++ .../tests/repo-host-autogrow/03-assert.yaml | 49 ++++++ .../99-remove-cluster-gracefully.yaml | 22 +++ .../controller/postgrescluster/pgbackrest.go | 147 ++++++++++++++++-- .../postgrescluster/pgbackrest_test.go | 147 ++++++++++++++---- .../controller/postgrescluster/watches.go | 14 ++ .../postgrescluster/watches_test.go | 15 ++ internal/naming/annotations.go | 27 ++++ internal/pgbackrest/config.go | 42 ++++- internal/pgbackrest/config_test.go | 34 ++-- internal/pgbackrest/rbac.go | 3 +- internal/pgbackrest/rbac_test.go | 1 + internal/pgbackrest/reconcile.go | 24 ++- internal/pgbackrest/reconcile_test.go | 39 +++++ .../v1beta1/pgbackrest_types.go | 4 + 28 files changed, 669 insertions(+), 65 deletions(-) create mode 100644 e2e-tests/tests/repo-host-autogrow/00-assert.yaml create mode 100644 e2e-tests/tests/repo-host-autogrow/00-deploy-operator.yaml create mode 100644 e2e-tests/tests/repo-host-autogrow/01-assert.yaml create mode 100644 e2e-tests/tests/repo-host-autogrow/01-create-cluster.yaml create mode 100644 e2e-tests/tests/repo-host-autogrow/02-assert.yaml create mode 100644 e2e-tests/tests/repo-host-autogrow/02-grow-repo-volume.yaml create mode 100644 e2e-tests/tests/repo-host-autogrow/03-assert.yaml create mode 100644 e2e-tests/tests/repo-host-autogrow/99-remove-cluster-gracefully.yaml diff --git a/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml b/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml index f681b7dc1b..7d9eed2a47 100644 --- a/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml +++ b/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml @@ -28953,6 +28953,10 @@ spec: description: Whether or not the pgBackRest repository PersistentVolumeClaim is bound to a volume type: boolean + desiredRepoVolume: + description: Desired size of the PersistentVolumeClaim containing + the repository. + type: string name: description: The name of the pgBackRest repository type: string diff --git a/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml b/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml index b6cbab607b..449ad76ed7 100644 --- a/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml +++ b/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml @@ -29648,6 +29648,10 @@ spec: description: Whether or not the pgBackRest repository PersistentVolumeClaim is bound to a volume type: boolean + desiredRepoVolume: + description: Desired size of the PersistentVolumeClaim containing + the repository. + type: string name: description: The name of the pgBackRest repository type: string diff --git a/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml b/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml index e8ba977e3c..3dc3dde0a2 100644 --- a/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml +++ b/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml @@ -30951,6 +30951,10 @@ spec: description: Whether or not the pgBackRest repository PersistentVolumeClaim is bound to a volume type: boolean + desiredRepoVolume: + description: Desired size of the PersistentVolumeClaim containing + the repository. + type: string name: description: The name of the pgBackRest repository type: string diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 737b9667eb..306acfa9ee 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -29945,6 +29945,10 @@ spec: description: Whether or not the pgBackRest repository PersistentVolumeClaim is bound to a volume type: boolean + desiredRepoVolume: + description: Desired size of the PersistentVolumeClaim containing + the repository. + type: string name: description: The name of the pgBackRest repository type: string @@ -69021,6 +69025,10 @@ spec: description: Whether or not the pgBackRest repository PersistentVolumeClaim is bound to a volume type: boolean + desiredRepoVolume: + description: Desired size of the PersistentVolumeClaim containing + the repository. + type: string name: description: The name of the pgBackRest repository type: string diff --git a/deploy/crd.yaml b/deploy/crd.yaml index cee8c24318..2fd8eb9dbb 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -29945,6 +29945,10 @@ spec: description: Whether or not the pgBackRest repository PersistentVolumeClaim is bound to a volume type: boolean + desiredRepoVolume: + description: Desired size of the PersistentVolumeClaim containing + the repository. + type: string name: description: The name of the pgBackRest repository type: string @@ -69021,6 +69025,10 @@ spec: description: Whether or not the pgBackRest repository PersistentVolumeClaim is bound to a volume type: boolean + desiredRepoVolume: + description: Desired size of the PersistentVolumeClaim containing + the repository. + type: string name: description: The name of the pgBackRest repository type: string diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index e2ab9b11ab..526c9b8f01 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -29945,6 +29945,10 @@ spec: description: Whether or not the pgBackRest repository PersistentVolumeClaim is bound to a volume type: boolean + desiredRepoVolume: + description: Desired size of the PersistentVolumeClaim containing + the repository. + type: string name: description: The name of the pgBackRest repository type: string @@ -69021,6 +69025,10 @@ spec: description: Whether or not the pgBackRest repository PersistentVolumeClaim is bound to a volume type: boolean + desiredRepoVolume: + description: Desired size of the PersistentVolumeClaim containing + the repository. + type: string name: description: The name of the pgBackRest repository type: string diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index 08617ee7ed..f67cf8d615 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -19,6 +19,7 @@ ldap-tls monitoring monitoring-pmm3 one-pod +repo-host-autogrow operator-self-healing pitr scaling diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv index 4a6fb3d525..5411a66778 100644 --- a/e2e-tests/run-release.csv +++ b/e2e-tests/run-release.csv @@ -24,6 +24,7 @@ ldap-tls monitoring monitoring-pmm3 one-pod +repo-host-autogrow operator-self-healing pitr scaling diff --git a/e2e-tests/tests/repo-host-autogrow/00-assert.yaml b/e2e-tests/tests/repo-host-autogrow/00-assert.yaml new file mode 100644 index 0000000000..868d09b93f --- /dev/null +++ b/e2e-tests/tests/repo-host-autogrow/00-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-operator-deploy-status +timeout: 120 +commands: + - script: kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 diff --git a/e2e-tests/tests/repo-host-autogrow/00-deploy-operator.yaml b/e2e-tests/tests/repo-host-autogrow/00-deploy-operator.yaml new file mode 100644 index 0000000000..708a3fb05a --- /dev/null +++ b/e2e-tests/tests/repo-host-autogrow/00-deploy-operator.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + init_temp_dir + + PGO_FEATURE_GATES="AutoGrowVolumes=true" deploy_operator diff --git a/e2e-tests/tests/repo-host-autogrow/01-assert.yaml b/e2e-tests/tests/repo-host-autogrow/01-assert.yaml new file mode 100644 index 0000000000..5ef3e7ed85 --- /dev/null +++ b/e2e-tests/tests/repo-host-autogrow/01-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: repo-host-autogrow +status: + state: ready +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: repo-host-autogrow-repo-host +status: + readyReplicas: 1 diff --git a/e2e-tests/tests/repo-host-autogrow/01-create-cluster.yaml b/e2e-tests/tests/repo-host-autogrow/01-create-cluster.yaml new file mode 100644 index 0000000000..86d941f85b --- /dev/null +++ b/e2e-tests/tests/repo-host-autogrow/01-create-cluster.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + get_cr \ + | yq eval ' + .spec.instances[0].replicas = 1 | + .spec.proxy.pgBouncer.replicas = 1 | + .spec.backups.pgbackrest.repos[0].volume.volumeClaimSpec.resources.requests.storage = "256Mi" | + .spec.backups.pgbackrest.repos[0].volume.volumeClaimSpec.resources.limits.storage = "300Mi"' - \ + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/repo-host-autogrow/02-assert.yaml b/e2e-tests/tests/repo-host-autogrow/02-assert.yaml new file mode 100644 index 0000000000..fbae311ca0 --- /dev/null +++ b/e2e-tests/tests/repo-host-autogrow/02-assert.yaml @@ -0,0 +1,42 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: |- + set -o errexit + set -o nounset + set -o pipefail + set -o xtrace + + cluster=repo-host-autogrow + selector="postgres-operator.crunchydata.com/cluster=${cluster},postgres-operator.crunchydata.com/pgbackrest-dedicated" + + repo_pod=$(kubectl -n "${NAMESPACE}" get pods -l "${selector}" -o jsonpath='{.items[0].metadata.name}') + + suggested_size= + for _ in $(seq 1 60); do + suggested_size=$(kubectl -n "${NAMESPACE}" get pod "${repo_pod}" \ + -o jsonpath='{.metadata.annotations.suggested-pgbackrest-repo1-pvc-size}') + if [[ -n "${suggested_size}" ]]; then + break + fi + sleep 5 + done + + if [[ -z "${suggested_size}" ]]; then + echo "Expected repo1 PVC size suggestion" + exit 1 + fi + if [[ "${suggested_size}" != *Mi ]]; then + echo "Expected repo1 PVC size suggestion in Mi, got ${suggested_size}" + exit 1 + fi + + suggested_mebibytes=${suggested_size%Mi} + if ((suggested_mebibytes <= 300)); then + echo "Expected repo1 PVC size suggestion greater than 300Mi, got ${suggested_size}" + exit 1 + fi + + kubectl -n "${NAMESPACE}" exec "${repo_pod}" -c pgbackrest-config -- \ + rm -f /pgbackrest/repo1/autogrow-e2e.fill diff --git a/e2e-tests/tests/repo-host-autogrow/02-grow-repo-volume.yaml b/e2e-tests/tests/repo-host-autogrow/02-grow-repo-volume.yaml new file mode 100644 index 0000000000..2ea8ba7387 --- /dev/null +++ b/e2e-tests/tests/repo-host-autogrow/02-grow-repo-volume.yaml @@ -0,0 +1,33 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: |- + set -o errexit + set -o nounset + set -o pipefail + set -o xtrace + + cluster=repo-host-autogrow + selector="postgres-operator.crunchydata.com/cluster=${cluster},postgres-operator.crunchydata.com/pgbackrest-dedicated" + pvc_selector="postgres-operator.crunchydata.com/cluster=${cluster},postgres-operator.crunchydata.com/pgbackrest-volume,postgres-operator.crunchydata.com/pgbackrest-repo=repo1" + + repo_pod=$(kubectl -n "${NAMESPACE}" get pods -l "${selector}" -o jsonpath='{.items[0].metadata.name}') + repo_pvc=$(kubectl -n "${NAMESPACE}" get pvc -l "${pvc_selector}" -o jsonpath='{.items[0].metadata.name}') + storage=$(kubectl -n "${NAMESPACE}" get pvc "${repo_pvc}" -o jsonpath='{.spec.resources.requests.storage}') + + if [[ "${storage}" != "256Mi" ]]; then + echo "Expected repo1 PVC storage request to be 256Mi, got ${storage}" + exit 1 + fi + + read -r size used <<<"$( + kubectl -n "${NAMESPACE}" exec "${repo_pod}" -c pgbackrest-config -- \ + df -Pm /pgbackrest/repo1 | awk 'FNR == 2 {print $2, $3}' + )" + fill_mebibytes=$(((size * 80 + 99) / 100 - used)) + if ((fill_mebibytes > 0)); then + kubectl -n "${NAMESPACE}" exec "${repo_pod}" -c pgbackrest-config -- \ + dd if=/dev/zero of=/pgbackrest/repo1/autogrow-e2e.fill \ + bs=1M count="${fill_mebibytes}" conv=fsync status=none + fi diff --git a/e2e-tests/tests/repo-host-autogrow/03-assert.yaml b/e2e-tests/tests/repo-host-autogrow/03-assert.yaml new file mode 100644 index 0000000000..121f2803e4 --- /dev/null +++ b/e2e-tests/tests/repo-host-autogrow/03-assert.yaml @@ -0,0 +1,49 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: |- + set -o errexit + set -o nounset + set -o pipefail + set -o xtrace + + cluster=repo-host-autogrow + selector="postgres-operator.crunchydata.com/cluster=${cluster},postgres-operator.crunchydata.com/pgbackrest-dedicated" + + repo_pod=$(kubectl -n "${NAMESPACE}" get pods -l "${selector}" -o jsonpath='{.items[0].metadata.name}') + suggested_size=$(kubectl -n "${NAMESPACE}" get pod "${repo_pod}" \ + -o jsonpath='{.metadata.annotations.suggested-pgbackrest-repo1-pvc-size}') + + if [[ -z "${suggested_size}" ]]; then + echo "Expected repo1 PVC size suggestion" + exit 1 + fi + + status_size= + for _ in $(seq 1 60); do + status_size=$(kubectl -n "${NAMESPACE}" get \ + postgrescluster.upstream.pgv2.percona.com/"${cluster}" \ + -o jsonpath='{.status.pgbackrest.repos[?(@.name=="repo1")].desiredRepoVolume}') + if [[ "${status_size}" == "${suggested_size}" ]]; then + break + fi + sleep 5 + done + + if [[ "${status_size}" != "${suggested_size}" ]]; then + echo "Expected desired repo volume ${suggested_size}, got ${status_size}" + exit 1 + fi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + postgres-operator.crunchydata.com/cluster: repo-host-autogrow + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 + postgres-operator.crunchydata.com/pgbackrest-volume: "" +spec: + resources: + requests: + storage: 300Mi diff --git a/e2e-tests/tests/repo-host-autogrow/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/repo-host-autogrow/99-remove-cluster-gracefully.yaml new file mode 100644 index 0000000000..338c02b183 --- /dev/null +++ b/e2e-tests/tests/repo-host-autogrow/99-remove-cluster-gracefully.yaml @@ -0,0 +1,22 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGCluster + metadata: + name: repo-host-autogrow + - apiVersion: upstream.pgv2.percona.com/v1beta1 + kind: PostgresCluster + metadata: + name: repo-host-autogrow +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + remove_all_finalizers + check_operator_panic + destroy_operator + timeout: 60 diff --git a/internal/controller/postgrescluster/pgbackrest.go b/internal/controller/postgrescluster/pgbackrest.go index 4a51d94709..5aeee27256 100644 --- a/internal/controller/postgrescluster/pgbackrest.go +++ b/internal/controller/postgrescluster/pgbackrest.go @@ -23,6 +23,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" utilerrors "k8s.io/apimachinery/pkg/util/errors" @@ -179,12 +180,16 @@ func (r *Reconciler) applyRepoHostIntent(ctx context.Context, postgresCluster *v // representing a repository. func (r *Reconciler) applyRepoVolumeIntent(ctx context.Context, postgresCluster *v1beta1.PostgresCluster, spec corev1.PersistentVolumeClaimSpec, - repoName string, repoResources *RepoResources, + repoName string, repoResources *RepoResources, desiredVolume string, ) (*corev1.PersistentVolumeClaim, error) { repo, err := r.generateRepoVolumeIntent(postgresCluster, spec, repoName, repoResources) if err != nil { return nil, errors.WithStack(err) } + r.setRepoVolumeSize(ctx, postgresCluster, repo, repoName, desiredVolume) + // Clear any set limit before applying PVC. This is needed to allow the limit + // value to change later. + repo.Spec.Resources.Limits = nil if err := r.apply(ctx, repo); err != nil { return nil, r.handlePersistentVolumeClaimError(postgresCluster, @@ -194,6 +199,50 @@ func (r *Reconciler) applyRepoVolumeIntent(ctx context.Context, return repo, nil } +// setRepoVolumeSize applies an observed automatic-growth suggestion without +// exceeding the ceiling configured in the repository PVC resource limits. +func (r *Reconciler) setRepoVolumeSize(ctx context.Context, cluster *v1beta1.PostgresCluster, + pvc *corev1.PersistentVolumeClaim, repoName, desiredVolume string, +) { + limit := pvc.Spec.Resources.Limits.Storage() + request := pvc.Spec.Resources.Requests.Storage() + if limit.IsZero() { + return + } + + if request.Cmp(*limit) > 0 { + r.Recorder.Eventf(cluster, corev1.EventTypeWarning, "RepoVolumeRequestOverLimit", + "pgBackRest repository volume request (%v) for %s/%s is greater than its limit (%v); the limit will be used.", + request, cluster.Name, repoName, limit) + request = limit + } else if feature.Enabled(ctx, feature.AutoGrowVolumes) && desiredVolume != "" { + desired, err := resource.ParseQuantity(desiredVolume) + if err != nil { + logging.FromContext(ctx).Error(err, "Unable to parse pgBackRest repository volume request", + "repo", repoName, "value", desiredVolume) + return + } + if desired.Cmp(*request) > 0 { + request = &desired + } + if request.Cmp(*limit) >= 0 { + if request.Cmp(*limit) > 0 { + r.Recorder.Eventf(cluster, corev1.EventTypeWarning, "DesiredRepoVolumeAboveLimit", + "The desired size (%v) for the %s/%s pgBackRest repository volume is greater than its limit (%v).", + request, cluster.Name, repoName, limit) + } + request = limit + r.Recorder.Eventf(cluster, corev1.EventTypeNormal, "RepoVolumeLimitReached", + "pgBackRest repository volume for %s/%s is at its size limit (%v).", + cluster.Name, repoName, limit) + } + } + + pvc.Spec.Resources.Requests = corev1.ResourceList{ + corev1.ResourceStorage: *resource.NewQuantity(request.Value(), resource.BinarySI), + } +} + // +kubebuilder:rbac:groups="apps",resources="statefulsets",verbs={list} // +kubebuilder:rbac:groups="batch",resources="cronjobs",verbs={list} // +kubebuilder:rbac:groups="batch",resources="jobs",verbs={list} @@ -687,13 +736,15 @@ func (r *Reconciler) generateRepoHostIntent(ctx context.Context, postgresCluster // - https://docs.k8s.io/tasks/configure-pod-container/share-process-namespace/ repo.Spec.Template.Spec.ShareProcessNamespace = new(true) - // pgBackRest does not make any Kubernetes API calls. Use the default - // ServiceAccount and do not mount its credentials. - repo.Spec.Template.Spec.AutomountServiceAccountToken = new(false) + // The repo host only needs Kubernetes credentials when it reports automatic + // volume-growth suggestions through Pod annotations. + autoGrowRepoVolumes := len(pgbackrest.AutoGrowRepoNames(ctx, postgresCluster)) > 0 + repo.Spec.Template.Spec.AutomountServiceAccountToken = new(autoGrowRepoVolumes) // K8SPG-138 currVersion, err := gover.NewVersion(postgresCluster.Labels[naming.LabelVersion]) - if err == nil && currVersion.GreaterThanOrEqual(gover.Must(gover.NewVersion("2.4.0"))) { + if autoGrowRepoVolumes || + (err == nil && currVersion.GreaterThanOrEqual(gover.Must(gover.NewVersion("2.4.0")))) { repo.Spec.Template.Spec.ServiceAccountName = naming.PGBackRestRBAC(postgresCluster).Name } @@ -713,9 +764,12 @@ func (r *Reconciler) generateRepoHostIntent(ctx context.Context, postgresCluster pgbackrest.MakePGBackrestLogDir(&repo.Spec.Template, postgresCluster) // add pgBackRest repo volumes to pod + containerNames := []string{naming.PGBackRestRepoContainerName} + if autoGrowRepoVolumes { + containerNames = append(containerNames, naming.ContainerPGBackRestConfig) + } if err := pgbackrest.AddRepoVolumesToPod(postgresCluster, &repo.Spec.Template, - getRepoPVCNames(postgresCluster, repoResources.pvcs), - naming.PGBackRestRepoContainerName); err != nil { + getRepoPVCNames(postgresCluster, repoResources.pvcs), containerNames...); err != nil { return nil, errors.WithStack(err) } } @@ -2894,6 +2948,10 @@ func (r *Reconciler) reconcileRepos(ctx context.Context, errMsg := "reconciling repository volume" repoVols := []*corev1.PersistentVolumeClaim{} var replicaCreateRepo v1beta1.PGBackRestRepo + desiredVolumes, err := r.observeDesiredRepoVolumes(ctx, postgresCluster) + if err != nil { + return replicaCreateRepo, err + } for i, repo := range postgresCluster.Spec.Backups.PGBackRest.Repos { // the repo at index 0 is the replica creation repo if i == 0 { @@ -2904,7 +2962,7 @@ func (r *Reconciler) reconcileRepos(ctx context.Context, continue } repo, err := r.applyRepoVolumeIntent(ctx, postgresCluster, repo.Volume.VolumeClaimSpec, - repo.Name, repoResources) + repo.Name, repoResources, desiredVolumes[repo.Name]) if err != nil { log.Error(err, errMsg) errors = append(errors, err) @@ -2915,12 +2973,71 @@ func (r *Reconciler) reconcileRepos(ctx context.Context, } } - postgresCluster.Status.PGBackRest.Repos = getRepoVolumeStatus(postgresCluster.Status.PGBackRest.Repos, repoVols, extConfigHashes, - replicaCreateRepo.Name) + postgresCluster.Status.PGBackRest.Repos = getRepoVolumeStatus(postgresCluster.Status.PGBackRest.Repos, + repoVols, extConfigHashes, replicaCreateRepo.Name, desiredVolumes) return replicaCreateRepo, utilerrors.NewAggregate(errors) } +// observeDesiredRepoVolumes reads automatic-growth suggestions from the +// dedicated repo-host Pod and retains the latest values through Pod restarts. +func (r *Reconciler) observeDesiredRepoVolumes(ctx context.Context, + cluster *v1beta1.PostgresCluster, +) (map[string]string, error) { + desired := make(map[string]string) + if !feature.Enabled(ctx, feature.AutoGrowVolumes) { + return desired, nil + } + + if cluster.Status.PGBackRest != nil { + for _, status := range cluster.Status.PGBackRest.Repos { + if status.DesiredRepoVolume != "" { + desired[status.Name] = status.DesiredRepoVolume + } + } + } + + pods := &corev1.PodList{} + if err := r.Client.List( + ctx, pods, + client.InNamespace(cluster.Namespace), + client.MatchingLabelsSelector{Selector: naming.PGBackRestDedicatedSelector(cluster.Name)}, + ); err != nil { + return nil, errors.Wrap(err, "list pods") + } + + limits := make(map[string]bool) + for _, repo := range cluster.Spec.Backups.PGBackRest.Repos { + limits[repo.Name] = repo.Volume != nil && + !repo.Volume.VolumeClaimSpec.Resources.Limits.Storage().IsZero() + } + for _, pod := range pods.Items { + for annotation, value := range pod.Annotations { + repoName, ok := naming.PGBackRestRepoFromVolumeSizeAnnotation(annotation) + if !ok || !limits[repoName] || value == "" { + continue + } + + current, err := resource.ParseQuantity(value) + if err != nil { + return nil, errors.Wrap(err, "parse quantity") + } + previous, err := resource.ParseQuantity(desired[repoName]) + if err != nil { + previous = resource.Quantity{} + } + if current.Cmp(previous) > 0 { + desired[repoName] = value + r.Recorder.Eventf(cluster, corev1.EventTypeNormal, "RepoVolumeAutoGrow", + "pgBackRest repository volume expansion to %v requested for %s/%s.", + current.String(), cluster.Name, repoName) + } + } + } + + return desired, nil +} + // +kubebuilder:rbac:groups="",resources="pods",verbs={get,list} // +kubebuilder:rbac:groups="",resources="pods/exec",verbs={create} @@ -3081,7 +3198,7 @@ func getRepoHostStatus(repoHost *appsv1.StatefulSet) *v1beta1.RepoHostStatus { // (i.e. PVCs) reconciled for the cluster, and the hashes calculated for the configuration for any // external repositories defined for the cluster. func getRepoVolumeStatus(repoStatus []v1beta1.RepoStatus, repoVolumes []*corev1.PersistentVolumeClaim, - configHashes map[string]string, replicaCreateRepoName string, + configHashes map[string]string, replicaCreateRepoName string, desiredVolumes map[string]string, ) []v1beta1.RepoStatus { // the new repository status that will be generated and returned updatedRepoStatus := []v1beta1.RepoStatus{} @@ -3118,6 +3235,7 @@ func getRepoVolumeStatus(repoStatus []v1beta1.RepoStatus, repoVolumes []*corev1. rs.ReplicaCreateBackupComplete = false } rs.VolumeName = rv.Spec.VolumeName + rs.DesiredRepoVolume = desiredVolumes[repoName] updatedRepoStatus = append(updatedRepoStatus, rs) break @@ -3125,9 +3243,10 @@ func getRepoVolumeStatus(repoStatus []v1beta1.RepoStatus, repoVolumes []*corev1. } if newRepoVolStatus { updatedRepoStatus = append(updatedRepoStatus, v1beta1.RepoStatus{ - Bound: (rv.Status.Phase == corev1.ClaimBound), - Name: repoName, - VolumeName: rv.Spec.VolumeName, + Bound: (rv.Status.Phase == corev1.ClaimBound), + Name: repoName, + VolumeName: rv.Spec.VolumeName, + DesiredRepoVolume: desiredVolumes[repoName], }) } } diff --git a/internal/controller/postgrescluster/pgbackrest_test.go b/internal/controller/postgrescluster/pgbackrest_test.go index e41a11e83e..df792c9ccb 100644 --- a/internal/controller/postgrescluster/pgbackrest_test.go +++ b/internal/controller/postgrescluster/pgbackrest_test.go @@ -36,6 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/percona/percona-postgresql-operator/v2/internal/controller/runtime" + "github.com/percona/percona-postgresql-operator/v2/internal/feature" "github.com/percona/percona-postgresql-operator/v2/internal/initialize" "github.com/percona/percona-postgresql-operator/v2/internal/naming" "github.com/percona/percona-postgresql-operator/v2/internal/pgbackrest" @@ -48,6 +49,27 @@ import ( var testCronSchedule string = "*/15 * * * *" +func TestSetRepoVolumeSize(t *testing.T) { + gate := feature.NewGate() + assert.NilError(t, gate.SetFromMap(map[string]bool{feature.AutoGrowVolumes: true})) + ctx := feature.NewContext(t.Context(), gate) + + pvc := &corev1.PersistentVolumeClaim{Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}, + Limits: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("5Gi")}, + }, + }} + new(Reconciler).setRepoVolumeSize(ctx, &v1beta1.PostgresCluster{}, pvc, "repo1", "3Gi") + + assert.Equal(t, pvc.Spec.Resources.Requests.Storage().String(), "3Gi") + + status := getRepoVolumeStatus(nil, []*corev1.PersistentVolumeClaim{{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{naming.LabelPGBackRestRepo: "repo1"}}, + }}, nil, "", map[string]string{"repo1": "3Gi"}) + assert.Equal(t, status[0].DesiredRepoVolume, "3Gi") +} + func fakePostgresCluster(clusterName, namespace, clusterUID string, includeDedicatedRepo bool, ) *v1beta1.PostgresCluster { @@ -308,7 +330,8 @@ func TestReconcilePGBackRest(t *testing.T) { naming.LabelPGBackRestDedicated: "", } expectedLabelsSelector, err := metav1.LabelSelectorAsSelector( - metav1.SetAsLabelSelector(expectedLabels)) + metav1.SetAsLabelSelector(expectedLabels), + ) if err != nil { t.Error(err) } @@ -961,9 +984,10 @@ func TestReconcileReplicaCreateBackup(t *testing.T) { case "COMMAND": assert.Assert(t, env.Value == "backup") case "COMMAND_OPTS": - assert.Assert(t, env.Value == "--stanza=db --repo=1"+ - // K8SPG-506: adding label to get info about this backup in `pgbackrest info` - fmt.Sprintf(` --annotation="%s"="%s"`, v2.PGBackrestAnnotationJobType, naming.BackupReplicaCreate), + assert.Assert( + t, env.Value == "--stanza=db --repo=1"+ + // K8SPG-506: adding label to get info about this backup in `pgbackrest info` + fmt.Sprintf(` --annotation="%s"="%s"`, v2.PGBackrestAnnotationJobType, naming.BackupReplicaCreate), ) case "COMPARE_HASH": @@ -1663,7 +1687,8 @@ func TestGetPGBackRestResources(t *testing.T) { }, Spec: appsv1.StatefulSetSpec{ Selector: metav1.SetAsLabelSelector( - naming.PGBackRestDedicatedLabels(clusterName)), + naming.PGBackRestDedicatedLabels(clusterName), + ), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: naming.PGBackRestDedicatedLabels(clusterName), @@ -1708,7 +1733,8 @@ func TestGetPGBackRestResources(t *testing.T) { }, Spec: appsv1.StatefulSetSpec{ Selector: metav1.SetAsLabelSelector( - naming.PGBackRestDedicatedLabels(clusterName)), + naming.PGBackRestDedicatedLabels(clusterName), + ), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: naming.PGBackRestDedicatedLabels(clusterName), @@ -2176,7 +2202,8 @@ func TestReconcileCloudBasedDataSource(t *testing.T) { if tc.dataSource != nil { pgclusterDataSource = tc.dataSource.PGBackRest } - err := r.reconcileCloudBasedDataSource(ctx, + err := r.reconcileCloudBasedDataSource( + ctx, cluster, pgclusterDataSource, "testhash", @@ -2374,7 +2401,8 @@ func TestCopyConfigurationResources(t *testing.T) { sc := sourceCluster("0") assert.Check(t, apierrors.IsNotFound( - r.copyConfigurationResources(ctx, cluster("0", sc.Name, sc.Namespace), sc))) + r.copyConfigurationResources(ctx, cluster("0", sc.Name, sc.Namespace), sc), + )) }) t.Run("Only Secret", func(t *testing.T) { secret := secret("1") @@ -2386,7 +2414,8 @@ func TestCopyConfigurationResources(t *testing.T) { sc := sourceCluster("1") assert.Check(t, apierrors.IsNotFound( - r.copyConfigurationResources(ctx, cluster("1", sc.Name, sc.Namespace), sc))) + r.copyConfigurationResources(ctx, cluster("1", sc.Name, sc.Namespace), sc), + )) }) t.Run("Only ConfigMap", func(t *testing.T) { configMap := configMap("2") @@ -2398,7 +2427,8 @@ func TestCopyConfigurationResources(t *testing.T) { sc := sourceCluster("2") assert.Check(t, apierrors.IsNotFound( - r.copyConfigurationResources(ctx, cluster("2", sc.Name, sc.Namespace), sc))) + r.copyConfigurationResources(ctx, cluster("2", sc.Name, sc.Namespace), sc), + )) }) t.Run("Secret and ConfigMap, neither optional", func(t *testing.T) { secret := secret("3") @@ -2595,7 +2625,8 @@ volumes: ImagePullPolicy: corev1.PullAlways, }, } - job := generateBackupJobSpecIntent(ctx, + job := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, @@ -2610,7 +2641,8 @@ volumes: cluster.Spec.Backups = v1beta1.Backups{ PGBackRest: v1beta1.PGBackRestArchive{}, } - job := generateBackupJobSpecIntent(ctx, + job := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, @@ -2627,12 +2659,14 @@ volumes: }, }, } - job := generateBackupJobSpecIntent(ctx, + job := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, ) - assert.DeepEqual(t, job.Template.Spec.Containers[0].Resources, + assert.DeepEqual( + t, job.Template.Spec.Containers[0].Resources, corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1m"), @@ -2667,7 +2701,8 @@ volumes: }, }, } - job := generateBackupJobSpecIntent(ctx, + job := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, @@ -2680,7 +2715,8 @@ volumes: cluster.Spec.Backups.PGBackRest.Jobs = &v1beta1.BackupJobs{ PriorityClassName: new("some-priority-class"), } - job := generateBackupJobSpecIntent(ctx, + job := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, @@ -2698,7 +2734,8 @@ volumes: cluster.Spec.Backups.PGBackRest.Jobs = &v1beta1.BackupJobs{ Tolerations: tolerations, } - job := generateBackupJobSpecIntent(ctx, + job := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, @@ -2712,14 +2749,16 @@ volumes: t.Run("Undefined", func(t *testing.T) { cluster.Spec.Backups.PGBackRest.Jobs = nil - spec := generateBackupJobSpecIntent(ctx, + spec := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, ) assert.Assert(t, spec.TTLSecondsAfterFinished == nil) cluster.Spec.Backups.PGBackRest.Jobs = &v1beta1.BackupJobs{} - spec = generateBackupJobSpecIntent(ctx, + spec = generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, ) assert.Assert(t, spec.TTLSecondsAfterFinished == nil) @@ -2730,7 +2769,8 @@ volumes: TTLSecondsAfterFinished: new(int32(0)), } - spec := generateBackupJobSpecIntent(ctx, + spec := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, ) if assert.Check(t, spec.TTLSecondsAfterFinished != nil) { @@ -2743,7 +2783,8 @@ volumes: TTLSecondsAfterFinished: new(int32(100)), } - spec := generateBackupJobSpecIntent(ctx, + spec := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, ) if assert.Check(t, spec.TTLSecondsAfterFinished != nil) { @@ -2770,7 +2811,8 @@ volumes: }, }, } - job := generateBackupJobSpecIntent(ctx, + job := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, @@ -2805,7 +2847,8 @@ volumes: }, }, } - job := generateBackupJobSpecIntent(ctx, + job := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, @@ -2838,7 +2881,8 @@ volumes: }, }, } - job := generateBackupJobSpecIntent(ctx, + job := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, @@ -2871,7 +2915,8 @@ volumes: }, }, } - job := generateBackupJobSpecIntent(ctx, + job := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, @@ -2934,7 +2979,8 @@ volumes: }, }, } - job := generateBackupJobSpecIntent(ctx, + job := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, @@ -2986,7 +3032,8 @@ volumes: }, }, } - job := generateBackupJobSpecIntent(ctx, + job := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, @@ -3054,7 +3101,8 @@ volumes: }, }, } - job := generateBackupJobSpecIntent(ctx, + job := generateBackupJobSpecIntent( + ctx, cluster, v1beta1.PGBackRestRepo{}, "", "", nil, nil, @@ -3099,6 +3147,46 @@ func TestGenerateRepoHostIntent(t *testing.T) { } }) + t.Run("AutoGrowServiceAccountAndMount", func(t *testing.T) { + gate := feature.NewGate() + assert.NilError(t, gate.SetFromMap(map[string]bool{feature.AutoGrowVolumes: true})) + ctx := feature.NewContext(ctx, gate) + + cluster := &v1beta1.PostgresCluster{ObjectMeta: metav1.ObjectMeta{ + Name: "hippo", + Labels: map[string]string{naming.LabelVersion: "2.5.0"}, + }} + cluster.Spec.Backups.PGBackRest.Repos = []v1beta1.PGBackRestRepo{{ + Name: "repo1", + Volume: &v1beta1.RepoPVC{VolumeClaimSpec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}, + Limits: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("10Gi")}, + }, + }}, + }} + + sts, err := r.generateRepoHostIntent(ctx, cluster, "", &RepoResources{}, &observedInstances{}) + assert.NilError(t, err) + assert.Equal(t, *sts.Spec.Template.Spec.AutomountServiceAccountToken, true) + assert.Equal(t, sts.Spec.Template.Spec.ServiceAccountName, naming.PGBackRestRBAC(cluster).Name) + + var config *corev1.Container + for i := range sts.Spec.Template.Spec.Containers { + if sts.Spec.Template.Spec.Containers[i].Name == naming.ContainerPGBackRestConfig { + config = &sts.Spec.Template.Spec.Containers[i] + break + } + } + if assert.Check(t, config != nil) { + var found bool + for _, mount := range config.VolumeMounts { + found = found || mount.Name == "repo1" && mount.MountPath == "/pgbackrest/repo1" + } + assert.Assert(t, found) + } + }) + t.Run("Replicas", func(t *testing.T) { assert.Equal(t, *sts.Spec.Replicas, int32(1)) }) @@ -3418,7 +3506,8 @@ func TestGenerateRestoreJobIntent(t *testing.T) { assert.Equal(t, envVar.Name, "TEST_ENV") assert.Equal(t, envVar.Value, "VALUE3") - assert.DeepEqual(t, container.EnvFrom, + assert.DeepEqual( + t, container.EnvFrom, []corev1.EnvFromSource{ { Prefix: "restore", diff --git a/internal/controller/postgrescluster/watches.go b/internal/controller/postgrescluster/watches.go index 2175bef865..52e95540fd 100644 --- a/internal/controller/postgrescluster/watches.go +++ b/internal/controller/postgrescluster/watches.go @@ -89,6 +89,20 @@ func (*Reconciler) watchPods() handler.Funcs { }}) return } + + // Reconcile when any pgBackRest repository volume suggestion changes. + if len(cluster) != 0 { + for annotation := range newAnnotations { + if _, ok := naming.PGBackRestRepoFromVolumeSizeAnnotation(annotation); ok && + oldAnnotations[annotation] != newAnnotations[annotation] { + q.Add(reconcile.Request{NamespacedName: client.ObjectKey{ + Namespace: e.ObjectNew.GetNamespace(), + Name: cluster, + }}) + return + } + } + } }, } } diff --git a/internal/controller/postgrescluster/watches_test.go b/internal/controller/postgrescluster/watches_test.go index 6b9d106bbe..952bdf9747 100644 --- a/internal/controller/postgrescluster/watches_test.go +++ b/internal/controller/postgrescluster/watches_test.go @@ -214,4 +214,19 @@ func TestWatchPodsUpdate(t *testing.T) { }, }, queue) assert.Equal(t, queue.Len(), 1) + item, _ = queue.Get() + queue.Done(item) + + // A repo-host volume suggestion also triggers reconciliation. + update(ctx, event.UpdateEvent{ + ObjectOld: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"suggested-pgbackrest-repo1-pvc-size": "5Gi"}, + Labels: map[string]string{"postgres-operator.crunchydata.com/cluster": "starfish"}, + }}, + ObjectNew: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"suggested-pgbackrest-repo1-pvc-size": "8Gi"}, + Labels: map[string]string{"postgres-operator.crunchydata.com/cluster": "starfish"}, + }}, + }, queue) + assert.Equal(t, queue.Len(), 1) } diff --git a/internal/naming/annotations.go b/internal/naming/annotations.go index ec04eb0e9a..3ea8b12bbc 100644 --- a/internal/naming/annotations.go +++ b/internal/naming/annotations.go @@ -4,6 +4,8 @@ package naming +import "strings" + const ( annotationPrefix = labelPrefix perconaAnnotationPrefix = perconaLabelPrefix @@ -81,4 +83,29 @@ const ( // is present, the controller will not update the ConfigMap, allowing users to make custom // modifications that won't be overwritten during reconciliation. OverrideConfigAnnotation = perconaAnnotationPrefix + "override-config" + + // SuggestedPGBackRestRepoVolumeSizeAnnotationPrefix identifies Pod annotations + // containing a desired size for a volume-backed pgBackRest repository. + SuggestedPGBackRestRepoVolumeSizeAnnotationPrefix = "suggested-pgbackrest-" ) + +// SuggestedPGBackRestRepoVolumeSizeAnnotation returns the Pod annotation used +// to communicate an automatically calculated PVC size for repoName. +func SuggestedPGBackRestRepoVolumeSizeAnnotation(repoName string) string { + return SuggestedPGBackRestRepoVolumeSizeAnnotationPrefix + repoName + "-pvc-size" +} + +// PGBackRestRepoFromVolumeSizeAnnotation extracts a repository name from an +// automatic volume-size annotation. +func PGBackRestRepoFromVolumeSizeAnnotation(annotation string) (string, bool) { + if !strings.HasPrefix(annotation, SuggestedPGBackRestRepoVolumeSizeAnnotationPrefix) || + !strings.HasSuffix(annotation, "-pvc-size") { + return "", false + } + + repoName := strings.TrimSuffix( + strings.TrimPrefix(annotation, SuggestedPGBackRestRepoVolumeSizeAnnotationPrefix), + "-pvc-size", + ) + return repoName, repoName != "" +} diff --git a/internal/pgbackrest/config.go b/internal/pgbackrest/config.go index 0ed478dbd3..1210d39cbd 100644 --- a/internal/pgbackrest/config.go +++ b/internal/pgbackrest/config.go @@ -491,7 +491,7 @@ func getExternalRepoConfigs(repo v1beta1.PGBackRestRepo) map[string]string { // reloadCommand returns an entrypoint that convinces the pgBackRest TLS server // to reload its options and certificate files when they change. The process // will appear as name in `ps` and `top`. -func reloadCommand(name string, post250 bool) []string { +func reloadCommand(name string, post250 bool, autoGrowRepos []string) []string { // Use a Bash loop to periodically check the mtime of the mounted server // volume and configuration file. When either changes, signal pgBackRest // and print the observed timestamp. @@ -551,6 +551,46 @@ done ` } + if len(autoGrowRepos) > 0 { + var monitorCalls strings.Builder + for _, repoName := range autoGrowRepos { + fmt.Fprintf(&monitorCalls, " monitor_volume %q %q\n", + repoMountPath+"/"+repoName, + naming.SuggestedPGBackRestRepoVolumeSizeAnnotation(repoName)) + } + + autoGrowScript := ` +# Parameters for updating automatic volume-growth annotations. +APISERVER="https://kubernetes.default.svc" +SERVICEACCOUNT="/var/run/secrets/kubernetes.io/serviceaccount" +NAMESPACE=$(<"${SERVICEACCOUNT}/namespace") +TOKEN=$(<"${SERVICEACCOUNT}/token") +CACERT="${SERVICEACCOUNT}/ca.crt" + +monitor_volume() { + local path="$1" annotation="$2" size use size_int use_int new_size patch + size=$(df --human-readable --block-size=M "${path}" | awk 'FNR == 2 {print $2}') + use=$(df --human-readable "${path}" | awk 'FNR == 2 {print $5}') + size_int="${size//M/}" + use_int=$(sed 's/[[:punct:]]//g' <<<"${use}") + if ((use_int > 75)); then + new_size="$((size_int + size_int / 2))Mi" + patch='{"metadata":{"annotations":{"'"${annotation}"'":"'"${new_size}"'"}}}' + curl --fail --silent --show-error --cacert "${CACERT}" \ + --header "Authorization: Bearer ${TOKEN}" \ + --header "Content-Type: application/merge-patch+json" \ + --request PATCH \ + --data "${patch}" \ + "${APISERVER}/api/v1/namespaces/${NAMESPACE}/pods/${HOSTNAME}" + fi +} +` + + // Run each repository check from the existing five-second reload loop. + script = strings.Replace(script, "done\n", monitorCalls.String()+"done\n", 1) + script = autoGrowScript + script + } + // Elide the above script from `ps` and `top` by wrapping it in a function // and calling that. wrapper := `monitor() {` + script + `};` + diff --git a/internal/pgbackrest/config_test.go b/internal/pgbackrest/config_test.go index c931558e21..5e2fc1c43b 100644 --- a/internal/pgbackrest/config_test.go +++ b/internal/pgbackrest/config_test.go @@ -310,25 +310,27 @@ func TestMakePGBackrestLogDir(t *testing.T) { func TestReloadCommand(t *testing.T) { shellcheck := require.ShellCheck(t) - command := reloadCommand("some-name", true) - - // Expect a bash command with an inline script. - assert.DeepEqual(t, command[:3], []string{"bash", "-ceu", "--"}) - assert.Assert(t, len(command) > 3) - - // Write out that inline script. - dir := t.TempDir() - file := filepath.Join(dir, "script.bash") - assert.NilError(t, os.WriteFile(file, []byte(command[3]), 0o600)) - - // Expect shellcheck to be happy. - cmd := exec.Command(shellcheck, "--enable=all", file) - output, err := cmd.CombinedOutput() - assert.NilError(t, err, "%q\n%s", cmd.Args, output) + for _, repos := range [][]string{nil, {"repo1", "repo4"}} { + command := reloadCommand("some-name", true, repos) + + // Expect a bash command with an inline script. + assert.DeepEqual(t, command[:3], []string{"bash", "-ceu", "--"}) + assert.Assert(t, len(command) > 3) + + // Write out that inline script. + dir := t.TempDir() + file := filepath.Join(dir, "script.bash") + assert.NilError(t, os.WriteFile(file, []byte(command[3]), 0o600)) + + // Expect shellcheck to be happy. + cmd := exec.CommandContext(t.Context(), shellcheck, "--enable=all", file) + output, err := cmd.CombinedOutput() + assert.NilError(t, err, "%q\n%s", cmd.Args, output) + } } func TestReloadCommandPrettyYAML(t *testing.T) { - assert.Assert(t, cmp.MarshalContains(reloadCommand("any", true), "\n- |"), + assert.Assert(t, cmp.MarshalContains(reloadCommand("any", true, nil), "\n- |"), "expected literal block scalar") } diff --git a/internal/pgbackrest/rbac.go b/internal/pgbackrest/rbac.go index 3e0cedd909..d50f4ced03 100644 --- a/internal/pgbackrest/rbac.go +++ b/internal/pgbackrest/rbac.go @@ -16,13 +16,12 @@ import ( // Permissions returns the RBAC rules pgBackRest needs for a cluster. func Permissions(cluster *v1beta1.PostgresCluster) []rbacv1.PolicyRule { - rules := make([]rbacv1.PolicyRule, 0, 2) rules = append(rules, rbacv1.PolicyRule{ APIGroups: []string{corev1.SchemeGroupVersion.Group}, Resources: []string{"pods"}, - Verbs: []string{"list"}, + Verbs: []string{"list", "patch"}, }) rules = append(rules, rbacv1.PolicyRule{ diff --git a/internal/pgbackrest/rbac_test.go b/internal/pgbackrest/rbac_test.go index f4ff876cc0..cdc4da8c62 100644 --- a/internal/pgbackrest/rbac_test.go +++ b/internal/pgbackrest/rbac_test.go @@ -46,6 +46,7 @@ func TestPermissions(t *testing.T) { - pods verbs: - list + - patch - apiGroups: - "" resources: diff --git a/internal/pgbackrest/reconcile.go b/internal/pgbackrest/reconcile.go index f65b45814a..c946aa0e3b 100644 --- a/internal/pgbackrest/reconcile.go +++ b/internal/pgbackrest/reconcile.go @@ -277,6 +277,7 @@ func addServerContainerAndVolume( ctx context.Context, cluster *v1beta1.PostgresCluster, pod *corev1.PodSpec, certificates []corev1.VolumeProjection, resources *corev1.ResourceRequirements, + autoGrowRepos []string, ) { serverVolumeMount := corev1.VolumeMount{ Name: "pgbackrest-server", @@ -334,7 +335,7 @@ func addServerContainerAndVolume( reloader := corev1.Container{ Name: naming.ContainerPGBackRestConfig, - Command: reloadCommand(naming.ContainerPGBackRestConfig, cluster.CompareVersion("2.5.0") >= 0), + Command: reloadCommand(naming.ContainerPGBackRestConfig, cluster.CompareVersion("2.5.0") >= 0, autoGrowRepos), Image: container.Image, ImagePullPolicy: container.ImagePullPolicy, SecurityContext: initialize.RestrictedSecurityContext(cluster.CompareVersion("2.5.0") >= 0), @@ -374,7 +375,7 @@ func AddServerToInstancePod( resources = sidecars.PGBackRest.Resources } - addServerContainerAndVolume(ctx, cluster, pod, certificates, resources) + addServerContainerAndVolume(ctx, cluster, pod, certificates, resources, nil) } // AddServerToRepoPod adds the TLS server container and volume to pod for @@ -397,7 +398,24 @@ func AddServerToRepoPod( resources = &cluster.Spec.Backups.PGBackRest.RepoHost.Resources } - addServerContainerAndVolume(ctx, cluster, pod, certificates, resources) + addServerContainerAndVolume(ctx, cluster, pod, certificates, resources, + AutoGrowRepoNames(ctx, cluster)) +} + +func AutoGrowRepoNames(ctx context.Context, cluster *v1beta1.PostgresCluster) []string { + if !feature.Enabled(ctx, feature.AutoGrowVolumes) { + return nil + } + + names := make([]string, 0, len(cluster.Spec.Backups.PGBackRest.Repos)) + for _, repo := range cluster.Spec.Backups.PGBackRest.Repos { + if repo.Volume != nil && + !repo.Volume.VolumeClaimSpec.Resources.Limits.Storage().IsZero() { + names = append(names, repo.Name) + } + } + + return names } // InstanceCertificates populates the shared Secret with certificates needed to run pgBackRest. diff --git a/internal/pgbackrest/reconcile_test.go b/internal/pgbackrest/reconcile_test.go index 98a3dfbb98..a100393063 100644 --- a/internal/pgbackrest/reconcile_test.go +++ b/internal/pgbackrest/reconcile_test.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "reflect" + "strings" "testing" "github.com/google/go-cmp/cmp/cmpopts" @@ -976,6 +977,44 @@ func TestAddServerToRepoPod(t *testing.T) { }) } +func TestRepoVolumeAutoGrowMonitoring(t *testing.T) { + gate := feature.NewGate() + assert.NilError(t, gate.SetFromMap(map[string]bool{feature.AutoGrowVolumes: true})) + ctx := feature.NewContext(t.Context(), gate) + + cluster := new(v1beta1.PostgresCluster) + cluster.Name = "hippo" + cluster.Labels = map[string]string{naming.LabelVersion: "2.5.0"} + cluster.Spec.Backups.PGBackRest.Repos = []v1beta1.PGBackRestRepo{ + { + Name: "repo1", + Volume: &v1beta1.RepoPVC{VolumeClaimSpec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{Limits: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }}, + }}, + }, + {Name: "repo2", Volume: &v1beta1.RepoPVC{}}, + {Name: "repo3"}, + } + + assert.DeepEqual(t, AutoGrowRepoNames(ctx, cluster), []string{"repo1"}) + + instance := new(corev1.PodSpec) + AddServerToInstancePod(ctx, cluster, instance, "instance-secret") + instanceScript := instance.Containers[len(instance.Containers)-1].Command[3] + assert.Assert(t, !strings.Contains(instanceScript, "monitor_volume")) + assert.Assert(t, !strings.Contains(instanceScript, "suggested-pgbackrest-")) + + repoHost := new(corev1.PodSpec) + AddServerToRepoPod(ctx, cluster, repoHost) + repoHostScript := repoHost.Containers[len(repoHost.Containers)-1].Command[3] + assert.Assert(t, strings.Contains(repoHostScript, + `monitor_volume "/pgbackrest/repo1" "suggested-pgbackrest-repo1-pvc-size"`)) + assert.Assert(t, !strings.Contains(repoHostScript, "/pgbackrest/repo2")) + assert.Assert(t, !strings.Contains(repoHostScript, "/pgbackrest/repo3")) +} + func getContainerNames(containers []corev1.Container) []string { names := make([]string, len(containers)) for i, c := range containers { diff --git a/pkg/apis/upstream.pgv2.percona.com/v1beta1/pgbackrest_types.go b/pkg/apis/upstream.pgv2.percona.com/v1beta1/pgbackrest_types.go index 83cf8b4781..171a3738a2 100644 --- a/pkg/apis/upstream.pgv2.percona.com/v1beta1/pgbackrest_types.go +++ b/pkg/apis/upstream.pgv2.percona.com/v1beta1/pgbackrest_types.go @@ -434,6 +434,10 @@ type RepoStatus struct { // +optional VolumeName string `json:"volume,omitempty"` + // Desired size of the PersistentVolumeClaim containing the repository. + // +optional + DesiredRepoVolume string `json:"desiredRepoVolume,omitempty"` + // Specifies whether or not a stanza has been successfully created for the repository // +optional StanzaCreated bool `json:"stanzaCreated"` From af736b48dfa96a04a6854df6f702e3fd1b82ad14 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Thu, 16 Jul 2026 16:32:20 +0300 Subject: [PATCH 2/2] fix test --- internal/pgbackrest/config.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/pgbackrest/config.go b/internal/pgbackrest/config.go index 1210d39cbd..d68d0f74f2 100644 --- a/internal/pgbackrest/config.go +++ b/internal/pgbackrest/config.go @@ -568,11 +568,12 @@ TOKEN=$(<"${SERVICEACCOUNT}/token") CACERT="${SERVICEACCOUNT}/ca.crt" monitor_volume() { - local path="$1" annotation="$2" size use size_int use_int new_size patch - size=$(df --human-readable --block-size=M "${path}" | awk 'FNR == 2 {print $2}') - use=$(df --human-readable "${path}" | awk 'FNR == 2 {print $5}') + local path="$1" annotation="$2" df_output size use size_int use_int new_size patch + df_output=$(df --human-readable --block-size=M "${path}") + size=$(awk 'FNR == 2 {print $2}' <<<"${df_output}") + use=$(awk 'FNR == 2 {print $5}' <<<"${df_output}") size_int="${size//M/}" - use_int=$(sed 's/[[:punct:]]//g' <<<"${use}") + use_int="${use//[[:punct:]]/}" if ((use_int > 75)); then new_size="$((size_int + size_int / 2))Mi" patch='{"metadata":{"annotations":{"'"${annotation}"'":"'"${new_size}"'"}}}'