From 03c1f088eb54c093a386379ed835579e7eadfcfe Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Wed, 19 Aug 2026 16:34:57 +0530 Subject: [PATCH 01/12] =?UTF-8?q?Update=20e2e=20backup=20tests=20for=20bac?= =?UTF-8?q?kup-rework=20CLI=20changes=20-=20backup=20import:=20positional?= =?UTF-8?q?=20arg=20=E2=86=92=20--from-file=20flag=20(3=20locations)=20-?= =?UTF-8?q?=20backup=20restore:=20remove=20--cluster-id=20flag=20(no=20lon?= =?UTF-8?q?ger=20exists)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- e2e/e2e_tests/backup/test_backup_restore.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/e2e/e2e_tests/backup/test_backup_restore.py b/e2e/e2e_tests/backup/test_backup_restore.py index 623880776..a754f18e5 100644 --- a/e2e/e2e_tests/backup/test_backup_restore.py +++ b/e2e/e2e_tests/backup/test_backup_restore.py @@ -2204,7 +2204,7 @@ def run(self): self.ssh_obj.exec_command( self.mgmt_nodes[0], f"echo '{{not valid json}}' > {bad_json}") - out, err = self._sbcli(f"backup import {bad_json}") + out, err = self._sbcli(f"backup import --from-file {bad_json}") assert err or "error" in out.lower(), \ "TC-BCK-035: expected error for malformed JSON import" self.logger.info("TC-BCK-035: got expected error ✓") @@ -2214,7 +2214,7 @@ def run(self): self.ssh_obj.exec_command( self.mgmt_nodes[0], f"echo '[]' > {good_json}") - out, err = self._sbcli(f"backup import {good_json}") + out, err = self._sbcli(f"backup import --from-file {good_json}") # Empty list → 0 imported; should not error assert "error" not in out.lower() or "0" in out, \ f"TC-BCK-036: unexpected error for empty-list import: {err}" @@ -3820,7 +3820,7 @@ def _run_cli_cross_cluster_restore(self, backup_id: str, # TC-BCK-073: import metadata on Cluster-2 self.logger.info(f"TC-BCK-073: Cluster-2 — backup import {meta_file}") out, err = self._sbcli_c2( - f"backup import {meta_file} --cluster-id {self._cluster2_id}") + f"backup import --from-file {meta_file} --cluster-id {self._cluster2_id}") assert not (err and "error" in err.lower()), \ f"TC-BCK-073: backup import on Cluster-2 failed: {err}" self.logger.info(f"TC-BCK-073: import result: {out.strip()}") @@ -3867,8 +3867,7 @@ def _run_cli_cross_cluster_restore(self, backup_id: str, f"backup restore {backup_id} " f"--lvol {restored_name} " f"--pool {self._cluster2_pool_name} " - f"--node {c2_node_id} " - f"--cluster-id {self._cluster2_id}") + f"--node {c2_node_id}") assert not (err3 and "error" in err3.lower()), \ f"TC-BCK-075: restore on Cluster-2 failed: {err3}" self.logger.info( From 14992d2c39bd85e12ec26dc2989968b33993d900 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Fri, 21 Aug 2026 02:56:10 +0530 Subject: [PATCH 02/12] Fix _verify_lvol_crypto() to use correct field name crypto_bdev The LVol model uses `crypto_bdev` (a device name string like "crypto_LVOL_60"), not `crypto` (boolean). The assertion was always failing because `d.get("crypto")` returned None. --- e2e/e2e_tests/backup/test_backup_restore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/e2e_tests/backup/test_backup_restore.py b/e2e/e2e_tests/backup/test_backup_restore.py index a754f18e5..c7ea24025 100644 --- a/e2e/e2e_tests/backup/test_backup_restore.py +++ b/e2e/e2e_tests/backup/test_backup_restore.py @@ -1057,10 +1057,10 @@ def _verify_lvol_crypto(self, lvol_id: str, label: str = ""): details = self.sbcli_utils.get_lvol_details(lvol_id=lvol_id) assert details, f"{label}: get_lvol_details returned empty for {lvol_id}" d = details[0] if isinstance(details, list) else details - crypto_val = d.get("crypto") or d.get("encryption") or d.get("Crypto") - self.logger.info(f"{label}: lvol {lvol_id} crypto={crypto_val}") + crypto_val = d.get("crypto_bdev") or d.get("crypto") or d.get("encryption") + self.logger.info(f"{label}: lvol {lvol_id} crypto_bdev={crypto_val}") assert crypto_val, ( - f"{label}: restored lvol {lvol_id} expected crypto=True, " + f"{label}: restored lvol {lvol_id} expected crypto_bdev to be set, " f"got {crypto_val!r}. Full details: {d}") return d From 8ae760a14b41bf5592840ed1c928d34218fb4367 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Fri, 21 Aug 2026 14:24:37 +0530 Subject: [PATCH 03/12] Remove --max-subsys from deploy_storage_node() sn configure call max-subsys moved to cluster-level; sn configure no longer accepts it. --- e2e/utils/ssh_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/utils/ssh_utils.py b/e2e/utils/ssh_utils.py index fed82725f..ae5cdde4f 100644 --- a/e2e/utils/ssh_utils.py +++ b/e2e/utils/ssh_utils.py @@ -1917,7 +1917,7 @@ def deploy_storage_node(self, node, max_lvol, max_prov_gb, ifname="eth0", branch time.sleep(10) - configure_cmd = f"{self.base_cmd} -d sn configure --max-subsys {max_lvol} --nodes-per-socket {nodes_per_socket}" + configure_cmd = f"{self.base_cmd} -d sn configure --nodes-per-socket {nodes_per_socket}" deploy_cmd = f"{self.base_cmd} sn deploy --ifname {ifname}" self.logger.info(f"Deploying storage node: {node}") From 415e9e1af08f66e1b53e69954ec994e7e5f7fb9e Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Fri, 21 Aug 2026 15:27:34 +0530 Subject: [PATCH 04/12] Change MassCreateRapidRestart_6k NUM_SUBSYSTEMS to 60, NS_PER_SUBSYSTEM to 50 Adjust for 12k max-subsys limit: 60 subsystems x 50 ns/subsystem = 3000 lvols + 9000 snaps = 12000 entities. Both Docker and K8s variants. --- e2e/stress_test/mass_create_delete_stress.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/stress_test/mass_create_delete_stress.py b/e2e/stress_test/mass_create_delete_stress.py index 005e3af75..f01355319 100644 --- a/e2e/stress_test/mass_create_delete_stress.py +++ b/e2e/stress_test/mass_create_delete_stress.py @@ -5145,8 +5145,8 @@ class MassCreateRapidRestart_6k_3Snap_Docker(_MassCreateDeleteDocker): 30 rapid container stop/restart cycles per phase.""" PERSISTENT_RETRY = True MAX_ENTITY_COUNT = 12000 - NUM_SUBSYSTEMS = 40 - NS_PER_SUBSYSTEM = 75 + NUM_SUBSYSTEMS = 60 + NS_PER_SUBSYSTEM = 50 SNAPSHOTS_PER_LVOL = 3 RAPID_RESTART_ITERATIONS = 30 RAPID_RESTART_COOLDOWN = 60 @@ -5171,8 +5171,8 @@ class MassCreateRapidRestart_6k_3Snap_K8s(_MassCreateDeleteK8s): 30 rapid pod delete/restart cycles per phase.""" PERSISTENT_RETRY = True MAX_ENTITY_COUNT = 12000 - NUM_SUBSYSTEMS = 40 - NS_PER_SUBSYSTEM = 75 + NUM_SUBSYSTEMS = 60 + NS_PER_SUBSYSTEM = 50 SNAPSHOTS_PER_LVOL = 3 RAPID_RESTART_ITERATIONS = 30 RAPID_RESTART_COOLDOWN = 60 From b4932f8ddddb30c33b02611642a0a4d963ec1530 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Fri, 21 Aug 2026 16:12:14 +0530 Subject: [PATCH 05/12] Update workflows for CLI param migration and K8s operator CRD changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker workflows: fix --max-lvol → --max-subsys in bootstrap calls. K8s native workflows: move maxSubsystemCount and vcpuCount to StorageCluster spec, replace partitions with enableJournalDevice in StorageNodeSet spec, remove corePercentage (replaced by vcpuCount calculated from CPU count * percentage), rename max_lvol input to max_subsys across all workflows. --- .github/workflows/bare-metal-deploy.yml | 4 +- .github/workflows/k8s-e2e-ha.yaml | 2 +- .../k8s-native-cross-cluster-restore.yaml | 66 ++++++++++++++++--- .../workflows/k8s-native-e2e-add-node.yaml | 37 +++++++++-- .../k8s-native-e2e-node-migration.yaml | 37 +++++++++-- .github/workflows/k8s-native-e2e.yaml | 43 +++++++++--- .github/workflows/k8s-native-stress.yaml | 41 +++++++++--- .github/workflows/k8s-native-upgrade.yaml | 39 +++++++++-- .../workflows/monitoring-suite-docker.yaml | 4 +- .../monitoring-suite-k8s-native.yaml | 35 ++++++++-- .../workflows/topology-suite-k8s-add-node.yml | 4 +- .../topology-suite-k8s-migration.yml | 4 +- 12 files changed, 257 insertions(+), 59 deletions(-) diff --git a/.github/workflows/bare-metal-deploy.yml b/.github/workflows/bare-metal-deploy.yml index 8814991e6..455b65ae5 100755 --- a/.github/workflows/bare-metal-deploy.yml +++ b/.github/workflows/bare-metal-deploy.yml @@ -192,8 +192,8 @@ jobs: # If the cluster is running on k8s, we need to set the environment variable DEPLOY_CMD="./bootstrap-cluster.sh \ - --max-lvol 10 --max-snap 10 - --max-size 400G --number-of-devices 1 + --max-subsys 10 --max-snap 10 + --max-size 400G --number-of-devices 1 --sbcli-cmd sbcli-dev --spdk-debug" if [ "${{ inputs.k8s_snode }}" == "true" ]; then diff --git a/.github/workflows/k8s-e2e-ha.yaml b/.github/workflows/k8s-e2e-ha.yaml index 2190b2161..05b781d57 100755 --- a/.github/workflows/k8s-e2e-ha.yaml +++ b/.github/workflows/k8s-e2e-ha.yaml @@ -176,7 +176,7 @@ jobs: run: | cd $GITHUB_WORKSPACE/simplyBlockDeploy ./bootstrap-cluster.sh --sbcli-cmd "$SBCLI_CMD" \ - --max-lvol 10 --max-snap 10 --max-prov 500G --number-of-devices 1 \ + --max-subsys 10 --max-snap 10 --number-of-devices 1 \ --distr-ndcs $NDCS \ --distr-npcs $NPCS \ --distr-bs $BS \ diff --git a/.github/workflows/k8s-native-cross-cluster-restore.yaml b/.github/workflows/k8s-native-cross-cluster-restore.yaml index d7a24cb32..25bf0d994 100644 --- a/.github/workflows/k8s-native-cross-cluster-restore.yaml +++ b/.github/workflows/k8s-native-cross-cluster-restore.yaml @@ -58,7 +58,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -640,7 +640,7 @@ jobs: IFC_NAMES="${{ github.event.inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ github.event.inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ github.event.inputs.max_subsys || '30' }}" SB_REPO="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ github.event.inputs.simplyblock_image }}" SPDK_IMAGE="${{ github.event.inputs.spdk_image }}" @@ -650,6 +650,30 @@ jobs: FORCE_FORMAT_4K="${{ (github.event.inputs.cluster_environment == 'aws-openshift' || github.event.inputs.cluster_environment == 'openshift-local' || github.event.inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ github.event.inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${C1_WORKERS%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + # Build C1 worker nodes YAML WORKER_YAML="" IFS=',' read -ra C1_NODES <<< "${C1_WORKERS}" @@ -688,6 +712,8 @@ jobs: capacity: 96 provisionedCapacity: 98 ${BACKUP_SPEC} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -711,9 +737,7 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 @@ -735,7 +759,7 @@ jobs: IFC_NAMES="${{ github.event.inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ github.event.inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ github.event.inputs.max_subsys || '30' }}" SB_REPO="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ github.event.inputs.simplyblock_image }}" SPDK_IMAGE="${{ github.event.inputs.spdk_image }}" @@ -745,6 +769,30 @@ jobs: FORCE_FORMAT_4K="${{ (github.event.inputs.cluster_environment == 'aws-openshift' || github.event.inputs.cluster_environment == 'openshift-local' || github.event.inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ github.event.inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${C2_WORKERS%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + # Build C2 worker nodes YAML WORKER_YAML="" IFS=',' read -ra C2_NODES <<< "${C2_WORKERS}" @@ -783,6 +831,8 @@ jobs: capacity: 96 provisionedCapacity: 98 ${BACKUP_SPEC} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -806,9 +856,7 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/k8s-native-e2e-add-node.yaml b/.github/workflows/k8s-native-e2e-add-node.yaml index b935ce2ff..3c756e136 100644 --- a/.github/workflows/k8s-native-e2e-add-node.yaml +++ b/.github/workflows/k8s-native-e2e-add-node.yaml @@ -42,7 +42,7 @@ on: ifc_names: type: string default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: type: string default: '30' ssh_user: @@ -132,7 +132,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -919,7 +919,7 @@ jobs: IFC_NAMES="${{ inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ inputs.max_subsys || '30' }}" SB_REPO="${{ inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ inputs.simplyblock_image }}" SPDK_IMAGE="${{ inputs.spdk_image }}" @@ -931,6 +931,31 @@ jobs: FORCE_FORMAT_4K="${{ (inputs.cluster_environment == 'aws-openshift' || inputs.cluster_environment == 'openshift-local' || inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${{ inputs.worker_nodes }}" + FIRST_WORKER="${FIRST_WORKER%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + RESERVED_CPU_YAML="" if [ "${{ inputs.cluster_environment }}" = "openshift-baremetal" ]; then RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' @@ -1005,6 +1030,8 @@ jobs: provisionedCapacity: 98 ${BACKUP_SPEC} ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -1030,10 +1057,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/k8s-native-e2e-node-migration.yaml b/.github/workflows/k8s-native-e2e-node-migration.yaml index e46cf1baf..21eb1b344 100644 --- a/.github/workflows/k8s-native-e2e-node-migration.yaml +++ b/.github/workflows/k8s-native-e2e-node-migration.yaml @@ -48,7 +48,7 @@ on: ifc_names: type: string default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: type: string default: '30' ssh_user: @@ -147,7 +147,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -915,7 +915,7 @@ jobs: IFC_NAMES="${{ inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ inputs.max_subsys || '30' }}" SB_REPO="${{ inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ inputs.simplyblock_image }}" SPDK_IMAGE="${{ inputs.spdk_image }}" @@ -927,6 +927,31 @@ jobs: FORCE_FORMAT_4K="${{ (inputs.cluster_environment == 'aws-openshift' || inputs.cluster_environment == 'openshift-local' || inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${{ inputs.worker_nodes }}" + FIRST_WORKER="${FIRST_WORKER%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + RESERVED_CPU_YAML="" if [ "${{ inputs.cluster_environment }}" = "openshift-baremetal" ]; then RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' @@ -1001,6 +1026,8 @@ jobs: provisionedCapacity: 98 ${BACKUP_SPEC} ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -1026,10 +1053,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/k8s-native-e2e.yaml b/.github/workflows/k8s-native-e2e.yaml index d80ac4a87..c4337a415 100644 --- a/.github/workflows/k8s-native-e2e.yaml +++ b/.github/workflows/k8s-native-e2e.yaml @@ -44,8 +44,8 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: - description: 'Max logical volume count per storage node' + max_subsys: + description: 'Max subsystem count (cluster-level)' required: false default: '30' ssh_user: @@ -826,7 +826,7 @@ jobs: IFC_NAMES="${{ github.event.inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ github.event.inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ github.event.inputs.max_subsys || '30' }}" SB_REPO="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ github.event.inputs.simplyblock_image }}" SPDK_IMAGE="${{ github.event.inputs.spdk_image }}" @@ -838,6 +838,31 @@ jobs: FORCE_FORMAT_4K="${{ (github.event.inputs.cluster_environment == 'aws-openshift' || github.event.inputs.cluster_environment == 'openshift-local' || github.event.inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ github.event.inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${{ github.event.inputs.worker_nodes }}" + FIRST_WORKER="${FIRST_WORKER%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + RESERVED_CPU_YAML="" if [ "${{ github.event.inputs.cluster_environment }}" = "openshift-baremetal" ]; then RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' @@ -912,6 +937,8 @@ jobs: provisionedCapacity: 98 ${BACKUP_SPEC} ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -937,10 +964,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 @@ -976,6 +1001,8 @@ jobs: provisionedCapacity: 98 ${BACKUP_SPEC} ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -1001,10 +1028,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/k8s-native-stress.yaml b/.github/workflows/k8s-native-stress.yaml index d01a057d4..f6f58c944 100644 --- a/.github/workflows/k8s-native-stress.yaml +++ b/.github/workflows/k8s-native-stress.yaml @@ -44,7 +44,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -690,7 +690,7 @@ jobs: IFC_NAMES="${{ github.event.inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ github.event.inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ github.event.inputs.max_subsys || '30' }}" SB_REPO="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ github.event.inputs.simplyblock_image }}" SPDK_IMAGE="${{ github.event.inputs.spdk_image }}" @@ -702,6 +702,31 @@ jobs: FORCE_FORMAT_4K="${{ (github.event.inputs.cluster_environment == 'aws-openshift' || github.event.inputs.cluster_environment == 'openshift-local' || github.event.inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ github.event.inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${{ github.event.inputs.worker_nodes }}" + FIRST_WORKER="${FIRST_WORKER%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + RESERVED_CPU_YAML="" if [ "${{ github.event.inputs.cluster_environment }}" = "openshift-baremetal" ]; then RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' @@ -760,6 +785,8 @@ jobs: capacity: 96 provisionedCapacity: 98 ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -785,10 +812,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 @@ -823,6 +848,8 @@ jobs: capacity: 96 provisionedCapacity: 98 ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -848,10 +875,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/k8s-native-upgrade.yaml b/.github/workflows/k8s-native-upgrade.yaml index dcf21c647..139f6ce61 100644 --- a/.github/workflows/k8s-native-upgrade.yaml +++ b/.github/workflows/k8s-native-upgrade.yaml @@ -74,9 +74,9 @@ on: required: false default: 'br-ex:enp2s0f0' cluster_params: - description: 'Cluster config: ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_lvol=30' + description: 'Cluster config: ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_subsys=30' required: false - default: 'ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_lvol=30' + default: 'ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_subsys=30' cluster_environment: description: 'Target cluster environment' required: true @@ -239,7 +239,7 @@ jobs: # ── Parse cluster params ── - name: Parse cluster parameters run: | - PARAMS="${{ github.event.inputs.cluster_params || 'ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_lvol=30' }}" + PARAMS="${{ github.event.inputs.cluster_params || 'ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_subsys=30' }}" for kv in $(echo "$PARAMS" | tr ',' '\n'); do key=$(echo "$kv" | cut -d= -f1 | tr '[:lower:]' '[:upper:]') val=$(echo "$kv" | cut -d= -f2) @@ -505,7 +505,7 @@ jobs: SB_REPO="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ github.event.inputs.base_simplyblock_image }}" SPDK_IMAGE="${{ github.event.inputs.base_spdk_image }}" - MAX_LVOL="${MAX_LVOL:-30}" + MAX_SUBSYS="${MAX_SUBSYS:-30}" # Build worker YAML IFS=',' read -ra NODES <<< "${{ github.event.inputs.worker_nodes }}" @@ -541,6 +541,31 @@ jobs: PCIE_MODEL_YAML=' pcieModel: "SAMSUNG MZQLB1T9HAJR-00007"' fi + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${{ github.event.inputs.worker_nodes }}" + FIRST_WORKER="${FIRST_WORKER%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + # Build vault settings (conditional on TLS) VAULT_SETTINGS="" if [ "${{ github.event.inputs.tls_enabled }}" = "true" ]; then @@ -570,6 +595,8 @@ jobs: capacity: 96 provisionedCapacity: 98 ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -594,10 +621,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/monitoring-suite-docker.yaml b/.github/workflows/monitoring-suite-docker.yaml index 296ffe9e2..8c3f0a95a 100755 --- a/.github/workflows/monitoring-suite-docker.yaml +++ b/.github/workflows/monitoring-suite-docker.yaml @@ -72,7 +72,7 @@ on: # Bootstrap params # ========================= BOOTSTRAP_MAX_LVOL: - description: "bootstrap: --max-lvol" + description: "bootstrap: --max-subsys (cluster-wide max subsystems per node)" required: true default: "75" BOOTSTRAP_DATA_CHUNKS: @@ -546,7 +546,7 @@ jobs: set +e ./bootstrap-cluster.sh \ --sbcli-cmd "${SBCLI_CMD}" \ - --max-lvol "${BOOTSTRAP_MAX_LVOL}" \ + --max-subsys "${BOOTSTRAP_MAX_LVOL}" \ --data-chunks-per-stripe "${BOOTSTRAP_DATA_CHUNKS}" \ --parity-chunks-per-stripe "${BOOTSTRAP_PARITY_CHUNKS}" \ --journal-partition "${BOOTSTRAP_JOURNAL_PARTITION}" \ diff --git a/.github/workflows/monitoring-suite-k8s-native.yaml b/.github/workflows/monitoring-suite-k8s-native.yaml index 729f7b6a6..278bd310a 100644 --- a/.github/workflows/monitoring-suite-k8s-native.yaml +++ b/.github/workflows/monitoring-suite-k8s-native.yaml @@ -59,7 +59,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -616,7 +616,7 @@ jobs: IFC_NAMES="${{ github.event.inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ github.event.inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ github.event.inputs.max_subsys || '30' }}" SB_REPO="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ github.event.inputs.simplyblock_image }}" SPDK_IMAGE="${{ github.event.inputs.spdk_image }}" @@ -628,6 +628,31 @@ jobs: FORCE_FORMAT_4K="${{ (github.event.inputs.cluster_environment == 'aws-openshift' || github.event.inputs.cluster_environment == 'openshift-local' || github.event.inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ github.event.inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${{ github.event.inputs.worker_nodes }}" + FIRST_WORKER="${FIRST_WORKER%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + RESERVED_CPU_YAML="" if [ "${{ github.event.inputs.cluster_environment }}" = "openshift-baremetal" ]; then RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' @@ -683,6 +708,8 @@ jobs: capacity: 96 provisionedCapacity: 98 ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -708,10 +735,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/topology-suite-k8s-add-node.yml b/.github/workflows/topology-suite-k8s-add-node.yml index c5b32de30..7ae417f18 100755 --- a/.github/workflows/topology-suite-k8s-add-node.yml +++ b/.github/workflows/topology-suite-k8s-add-node.yml @@ -65,7 +65,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -212,7 +212,7 @@ jobs: worker_nodes: ${{ inputs.worker_nodes }} new_worker_nodes: ${{ inputs.new_worker_nodes }} ifc_names: ${{ inputs.ifc_names || 'br-ex:enp2s0f0' }} - max_lvol: ${{ inputs.max_lvol || '30' }} + max_subsys: ${{ inputs.max_subsys || '30' }} ssh_user: ${{ inputs.ssh_user || 'root' }} key_path: ${{ inputs.key_path }} send_slack_notification: false diff --git a/.github/workflows/topology-suite-k8s-migration.yml b/.github/workflows/topology-suite-k8s-migration.yml index 7df57b0a7..409608853 100755 --- a/.github/workflows/topology-suite-k8s-migration.yml +++ b/.github/workflows/topology-suite-k8s-migration.yml @@ -74,7 +74,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -214,7 +214,7 @@ jobs: new_ssd_pcie: ${{ inputs.new_ssd_pcie || '' }} reattach_volume: ${{ inputs.reattach_volume }} ifc_names: ${{ inputs.ifc_names || 'br-ex:enp2s0f0' }} - max_lvol: ${{ inputs.max_lvol || '30' }} + max_subsys: ${{ inputs.max_subsys || '30' }} ssh_user: ${{ inputs.ssh_user || 'root' }} key_path: ${{ inputs.key_path }} send_slack_notification: false From a06f806eb34e9f6b36a9591b68dce2340c7999d8 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Fri, 21 Aug 2026 17:12:00 +0530 Subject: [PATCH 06/12] Remove deprecated isSingleNode and strictNodeAntiAffinity from StorageCluster CRDs These fields were removed from the operator CRD in PR #440. Setting them causes strict decoding errors on latest operator. --- .github/workflows/k8s-native-cross-cluster-restore.yaml | 4 ---- .github/workflows/k8s-native-e2e-add-node.yaml | 2 -- .github/workflows/k8s-native-e2e-node-migration.yaml | 2 -- .github/workflows/k8s-native-e2e.yaml | 4 ---- .github/workflows/k8s-native-stress.yaml | 4 ---- .github/workflows/k8s-native-upgrade.yaml | 2 -- .github/workflows/monitoring-suite-k8s-native.yaml | 2 -- 7 files changed, 20 deletions(-) diff --git a/.github/workflows/k8s-native-cross-cluster-restore.yaml b/.github/workflows/k8s-native-cross-cluster-restore.yaml index 25bf0d994..b46245fb6 100644 --- a/.github/workflows/k8s-native-cross-cluster-restore.yaml +++ b/.github/workflows/k8s-native-cross-cluster-restore.yaml @@ -699,9 +699,7 @@ jobs: namespace: ${NS_C1} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} @@ -818,9 +816,7 @@ jobs: namespace: ${NS_C2} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} diff --git a/.github/workflows/k8s-native-e2e-add-node.yaml b/.github/workflows/k8s-native-e2e-add-node.yaml index 3c756e136..29c772824 100644 --- a/.github/workflows/k8s-native-e2e-add-node.yaml +++ b/.github/workflows/k8s-native-e2e-add-node.yaml @@ -1016,9 +1016,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} diff --git a/.github/workflows/k8s-native-e2e-node-migration.yaml b/.github/workflows/k8s-native-e2e-node-migration.yaml index 21eb1b344..9642adffa 100644 --- a/.github/workflows/k8s-native-e2e-node-migration.yaml +++ b/.github/workflows/k8s-native-e2e-node-migration.yaml @@ -1012,9 +1012,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} diff --git a/.github/workflows/k8s-native-e2e.yaml b/.github/workflows/k8s-native-e2e.yaml index c4337a415..ab3c39381 100644 --- a/.github/workflows/k8s-native-e2e.yaml +++ b/.github/workflows/k8s-native-e2e.yaml @@ -923,9 +923,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} @@ -987,9 +985,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} diff --git a/.github/workflows/k8s-native-stress.yaml b/.github/workflows/k8s-native-stress.yaml index f6f58c944..377eaf9bc 100644 --- a/.github/workflows/k8s-native-stress.yaml +++ b/.github/workflows/k8s-native-stress.yaml @@ -772,9 +772,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} @@ -835,9 +833,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} diff --git a/.github/workflows/k8s-native-upgrade.yaml b/.github/workflows/k8s-native-upgrade.yaml index 139f6ce61..022bddf2b 100644 --- a/.github/workflows/k8s-native-upgrade.yaml +++ b/.github/workflows/k8s-native-upgrade.yaml @@ -582,9 +582,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} diff --git a/.github/workflows/monitoring-suite-k8s-native.yaml b/.github/workflows/monitoring-suite-k8s-native.yaml index 278bd310a..137c3e41d 100644 --- a/.github/workflows/monitoring-suite-k8s-native.yaml +++ b/.github/workflows/monitoring-suite-k8s-native.yaml @@ -695,9 +695,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} From 635c358229942631f9003c52c359fb807cf6b0ce Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Fri, 21 Aug 2026 18:04:55 +0530 Subject: [PATCH 07/12] Add maxHugePagesSize to StorageCluster CRD templates hugepages_mem is now a required field in cluster create API. Default to 15G for all K8s native pipelines. --- .github/workflows/k8s-native-cross-cluster-restore.yaml | 2 ++ .github/workflows/k8s-native-e2e-add-node.yaml | 1 + .github/workflows/k8s-native-e2e-node-migration.yaml | 1 + .github/workflows/k8s-native-e2e.yaml | 2 ++ .github/workflows/k8s-native-stress.yaml | 2 ++ .github/workflows/k8s-native-upgrade.yaml | 1 + .github/workflows/monitoring-suite-k8s-native.yaml | 1 + 7 files changed, 10 insertions(+) diff --git a/.github/workflows/k8s-native-cross-cluster-restore.yaml b/.github/workflows/k8s-native-cross-cluster-restore.yaml index b46245fb6..3af92f053 100644 --- a/.github/workflows/k8s-native-cross-cluster-restore.yaml +++ b/.github/workflows/k8s-native-cross-cluster-restore.yaml @@ -712,6 +712,7 @@ jobs: ${BACKUP_SPEC} maxSubsystemCount: ${MAX_SUBSYS} vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -829,6 +830,7 @@ jobs: ${BACKUP_SPEC} maxSubsystemCount: ${MAX_SUBSYS} vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool diff --git a/.github/workflows/k8s-native-e2e-add-node.yaml b/.github/workflows/k8s-native-e2e-add-node.yaml index 29c772824..194d8833f 100644 --- a/.github/workflows/k8s-native-e2e-add-node.yaml +++ b/.github/workflows/k8s-native-e2e-add-node.yaml @@ -1030,6 +1030,7 @@ jobs: ${VAULT_SETTINGS} maxSubsystemCount: ${MAX_SUBSYS} vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool diff --git a/.github/workflows/k8s-native-e2e-node-migration.yaml b/.github/workflows/k8s-native-e2e-node-migration.yaml index 9642adffa..a27eb48a3 100644 --- a/.github/workflows/k8s-native-e2e-node-migration.yaml +++ b/.github/workflows/k8s-native-e2e-node-migration.yaml @@ -1026,6 +1026,7 @@ jobs: ${VAULT_SETTINGS} maxSubsystemCount: ${MAX_SUBSYS} vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool diff --git a/.github/workflows/k8s-native-e2e.yaml b/.github/workflows/k8s-native-e2e.yaml index ab3c39381..de43591bf 100644 --- a/.github/workflows/k8s-native-e2e.yaml +++ b/.github/workflows/k8s-native-e2e.yaml @@ -937,6 +937,7 @@ jobs: ${VAULT_SETTINGS} maxSubsystemCount: ${MAX_SUBSYS} vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -999,6 +1000,7 @@ jobs: ${VAULT_SETTINGS} maxSubsystemCount: ${MAX_SUBSYS} vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool diff --git a/.github/workflows/k8s-native-stress.yaml b/.github/workflows/k8s-native-stress.yaml index 377eaf9bc..83f8d60fc 100644 --- a/.github/workflows/k8s-native-stress.yaml +++ b/.github/workflows/k8s-native-stress.yaml @@ -785,6 +785,7 @@ jobs: ${VAULT_SETTINGS} maxSubsystemCount: ${MAX_SUBSYS} vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -846,6 +847,7 @@ jobs: ${VAULT_SETTINGS} maxSubsystemCount: ${MAX_SUBSYS} vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool diff --git a/.github/workflows/k8s-native-upgrade.yaml b/.github/workflows/k8s-native-upgrade.yaml index 022bddf2b..f4a4631d7 100644 --- a/.github/workflows/k8s-native-upgrade.yaml +++ b/.github/workflows/k8s-native-upgrade.yaml @@ -595,6 +595,7 @@ jobs: ${VAULT_SETTINGS} maxSubsystemCount: ${MAX_SUBSYS} vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool diff --git a/.github/workflows/monitoring-suite-k8s-native.yaml b/.github/workflows/monitoring-suite-k8s-native.yaml index 137c3e41d..5a55cd0a7 100644 --- a/.github/workflows/monitoring-suite-k8s-native.yaml +++ b/.github/workflows/monitoring-suite-k8s-native.yaml @@ -708,6 +708,7 @@ jobs: ${VAULT_SETTINGS} maxSubsystemCount: ${MAX_SUBSYS} vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool From 578a477bce57b9e797e0d41a52263beb1fb64e91 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Sat, 22 Aug 2026 15:30:33 +0530 Subject: [PATCH 08/12] Fix mass-create stress test: fail-fast on broken restarts and reduce cleanup timeouts Three changes to prevent the test from running 16+ hours when the cluster is broken (see run 32497309385): 1. _phase_rapid_restart_cycles: require >70% of iterations to detect node going offline, otherwise raise RuntimeError immediately. Previously all 30 iterations could silently time out as no-ops. 2. _wait_lvols_deleted: reduce stall_timeout from 1800 to 600s. The old value equalled the overall timeout so the stall check never fired independently. 3. _phase_cleanup: reduce CLEANUP_TIMEOUT from 1800 to 600s and pass stall_timeout=120s to delete_all_clones/delete_all_lvols. When the cluster is SUSPENDED/in_activation, cleanup now gives up after 2 min of no progress per step instead of 30 min. Co-Authored-By: Claude Opus 4.6 --- e2e/stress_test/mass_create_delete_stress.py | 42 +++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/e2e/stress_test/mass_create_delete_stress.py b/e2e/stress_test/mass_create_delete_stress.py index f01355319..7bd0cd101 100644 --- a/e2e/stress_test/mass_create_delete_stress.py +++ b/e2e/stress_test/mass_create_delete_stress.py @@ -1055,6 +1055,7 @@ def _phase_rapid_restart_cycles(self, label: str, ) results = [] + failures = 0 for i in range(1, iterations + 1): self.logger.info( f"[{label}] --- Iteration {i}/{iterations} ---" @@ -1075,14 +1076,17 @@ def _phase_rapid_restart_cycles(self, label: str, ) # Wait for node to go offline then back online + offline_ok = True try: self.sbcli_utils.wait_for_storage_node_status( node_uuid, ["offline", "unreachable"], timeout=600, ) except Exception as exc: + offline_ok = False + failures += 1 self.logger.warning( f"[{label}][{i}] Timed out waiting for " - f"offline/unreachable: {exc}" + f"offline/unreachable ({failures}/{i} failed): {exc}" ) self.sbcli_utils.wait_for_storage_node_status( @@ -1102,6 +1106,7 @@ def _phase_rapid_restart_cycles(self, label: str, "online_time_iso": online_ts.isoformat(), "stop_to_online_sec": stop_to_online, "cooldown_sec": cooldown, + "offline_detected": offline_ok, }) # Cooldown — no waiting for migration, just a short pause @@ -1109,8 +1114,23 @@ def _phase_rapid_restart_cycles(self, label: str, total_dur = sum(r["stop_to_online_sec"] for r in results) self._phase_durations[label] = round(total_dur, 1) + + # Require >70% of iterations to successfully detect node offline + successful = sum( + 1 for r in results if r.get("offline_detected", True) + ) + min_required = math.ceil(iterations * 0.7) + if successful < min_required: + raise RuntimeError( + f"[{label}] Only {successful}/{iterations} iterations " + f"detected node going offline (required {min_required}). " + f"{failures} iterations timed out — " + f"kill/restart mechanism is broken" + ) + self.logger.info( - f"[{label}] Completed {iterations} restart cycles, " + f"[{label}] Completed {iterations} restart cycles " + f"({successful}/{iterations} detected offline), " f"cumulative stop-to-online: {total_dur}s" ) return results @@ -3003,7 +3023,7 @@ def _fire_delete_lvol(self, lvol_name: str): def _wait_lvols_deleted( self, names: list, label: str, timeout: int = 1800, - stall_timeout: int = 1800, + stall_timeout: int = 600, ): """Wait for lvols/clones to disappear from the API. @@ -3158,7 +3178,7 @@ def _fire_delete_snapshot(self, snap_name: str): # Maximum wall-clock time for the entire cleanup phase. # Prevents stuck in_deletion lvols from blocking the test indefinitely. - CLEANUP_TIMEOUT = 1800 # 30 minutes + CLEANUP_TIMEOUT = 600 # 10 minutes — cleanup should not delay failure reporting def _phase_cleanup(self): timeout = self.CLEANUP_TIMEOUT @@ -3166,10 +3186,20 @@ def _phase_cleanup(self): def _run_cleanup(): deadline = time.time() + timeout + # Per-step stall timeout: give up quickly if cluster is + # unresponsive (e.g. SUSPENDED / in_activation). + stall = min(120, timeout // 4) + steps = [ - ("delete_all_clones", self.sbcli_utils.delete_all_clones), + ("delete_all_clones", lambda: self.sbcli_utils.delete_all_clones( + timeout=max(60, int(deadline - time.time())), + stall_timeout=stall, + )), ("delete_all_snapshots", self.sbcli_utils.delete_all_snapshots), - ("delete_all_lvols", self.sbcli_utils.delete_all_lvols), + ("delete_all_lvols", lambda: self.sbcli_utils.delete_all_lvols( + timeout=max(60, int(deadline - time.time())), + stall_timeout=stall, + )), ("delete_all_storage_pools", self.sbcli_utils.delete_all_storage_pools), ] for label, fn in steps: From 2f8d05aa3711e708366c24024fca764d14db5eda Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Sat, 22 Aug 2026 16:42:26 +0530 Subject: [PATCH 09/12] Fix graylog NFS copy: use sudo for mkdir under root-owned NFS path The NFS base directory is created with sudo (root-owned), so the subsequent mkdir for the timestamped subdirectory also needs sudo. Add chown after mkdir so the cp that follows works without sudo. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/collect-logs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/collect-logs.yml b/.github/workflows/collect-logs.yml index ffdd4891f..ad0a546a3 100755 --- a/.github/workflows/collect-logs.yml +++ b/.github/workflows/collect-logs.yml @@ -690,7 +690,8 @@ jobs: TIMESTAMP=$(date -u "+%Y%m%d-%H%M%S") NFS_DEST="${NFS_BASE}/graylog_collected-${TIMESTAMP}" - mkdir -p "${NFS_DEST}" + sudo mkdir -p "${NFS_DEST}" + sudo chown "$(id -u):$(id -g)" "${NFS_DEST}" echo "=== Copying collected logs to NFS ===" echo " Source: ${OUTPUT_DIR}" From 94b318d04fb058a6f51068e6ee651cfd205bbd74 Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Sun, 23 Aug 2026 17:00:54 +0530 Subject: [PATCH 10/12] Graylog export: collect in 1-hour reverse chunks with runner selection - Split time window into 1-hour chunks processed newest-first so progress is visible and high-volume containers (SPDK) don't lose data from scroll timeout expiry - Each chunk writes to a separate subfolder under OUTPUT_DIR (e.g. chunk_01_of_10_20260822_0900_to_1000/) - Increase OpenSearch scroll keepalive from 2m to 5m - Add RUNNER workflow input (vm22 / vm-runner-43) so the job always lands on a runner with NFS mounted - Fix job-summary total_lines to count logs in subdirectories Co-Authored-By: Claude Opus 4.6 --- .github/workflows/collect-logs.yml | 13 +- e2e/utils/test_graylog_export.py | 211 +++++++++++++++++++---------- 2 files changed, 150 insertions(+), 74 deletions(-) diff --git a/.github/workflows/collect-logs.yml b/.github/workflows/collect-logs.yml index ad0a546a3..c677721a3 100755 --- a/.github/workflows/collect-logs.yml +++ b/.github/workflows/collect-logs.yml @@ -104,6 +104,15 @@ on: # ========================= # Upload # ========================= + RUNNER: + description: "Runner to use (must have NFS mount at /mnt/nfs_share)" + required: false + default: "vm22" + type: choice + options: + - vm22 + - vm-runner-43 + UPLOAD_TO_MINIO: description: "Upload collected logs to MinIO" required: false @@ -117,7 +126,7 @@ concurrency: jobs: collect-logs: name: "Collect logs (${{ inputs.DEPLOY_MODE }}${{ inputs.DEPLOY_MODE == 'k8s-native' && format(' / {0}', inputs.cluster_environment) || '' }})" - runs-on: ${{ inputs.DEPLOY_MODE == 'k8s-native' && inputs.cluster_environment == 'aws-openshift' && 'vm-runner-43' || 'self-hosted' }} + runs-on: ${{ inputs.RUNNER || 'vm22' }} timeout-minutes: 720 env: @@ -717,7 +726,7 @@ jobs: gl_ip="${GRAYLOG_IP:-$MGMT_IP}" file_count="$(find "${OUTPUT_DIR}" -type f \( -name '*.log' -o -name '*.tar.gz' \) 2>/dev/null | wc -l || echo 0)" total_size="$(du -sh "${OUTPUT_DIR}" 2>/dev/null | awk '{print $1}' || echo 'unknown')" - total_lines="$(cat "${OUTPUT_DIR}"/*.log 2>/dev/null | wc -l || echo 0)" + total_lines="$(find "${OUTPUT_DIR}" -name '*.log' -exec cat {} + 2>/dev/null | wc -l || echo 0)" { echo "## Log Collection Summary" diff --git a/e2e/utils/test_graylog_export.py b/e2e/utils/test_graylog_export.py index a92fb1c2e..205471e0b 100755 --- a/e2e/utils/test_graylog_export.py +++ b/e2e/utils/test_graylog_export.py @@ -13,6 +13,12 @@ Fetch strategy: - Graylog REST API when reachable; OpenSearch scroll API otherwise. +Chunking: + - The total time window is split into 1-hour chunks. + - Chunks are processed in REVERSE order (newest first) so you can + verify collection is working while it runs. + - Each chunk writes to a subfolder: OUTPUT_DIR/chunk_NN_HH-MM_to_HH-MM/ + Environment variables: MGMT_IP Management node IP (required unless both dedicated IPs are set) CLUSTER_SECRET Graylog admin password / cluster secret (required) @@ -41,6 +47,7 @@ import os import sys +import math from datetime import datetime, timezone, timedelta try: @@ -105,6 +112,9 @@ PAGE_SIZE = 1000 MAX_RESULT_WINDOW = 100_000 +# Chunk size in minutes for splitting the time window +CHUNK_MINUTES = 60 + # --------------------------------------------------------------------------- # HTTP sessions # --------------------------------------------------------------------------- @@ -288,11 +298,10 @@ def os_discover_containers(): return [] -def os_fetch_container_logs(container_name, source, out_path, probe_cache=None): +def os_fetch_container_logs(container_name, source, out_path, + chunk_from_ms, chunk_to_ms, + probe_cache=None): """Fetch logs from OpenSearch using the scroll API. Returns line count.""" - from_ms = FROM_MS - to_ms = TO_MS - if probe_cache is None: probe_cache = {} if "index" not in probe_cache: @@ -306,7 +315,7 @@ def os_fetch_container_logs(container_name, source, out_path, probe_cache=None): esc = container_name.replace("/", "\\/").replace(":", "\\:") must_clauses = [ - {"range": {ts_f: {"gte": from_ms, "lte": to_ms, + {"range": {ts_f: {"gte": chunk_from_ms, "lte": chunk_to_ms, "format": "epoch_millis"}}}, {"query_string": {"default_field": cname_f, "query": f"*{esc}*", @@ -334,7 +343,7 @@ def _fmt(src): text = str(src.get("message", "")).replace("\n", "\\n") return f"{ts} src={s} ctr={cname} lvl={lvl} {text}" - init_url = f"{OPENSEARCH_BASE}/{index}/_search?scroll=2m" + init_url = f"{OPENSEARCH_BASE}/{index}/_search?scroll=5m" written = 0 try: @@ -371,7 +380,7 @@ def _fmt(src): try: sc_r = os_session.post( f"{OPENSEARCH_BASE}/_search/scroll", - json={"scroll": "2m", "scroll_id": scroll_id}, + json={"scroll": "5m", "scroll_id": scroll_id}, timeout=60, ) sc_r.raise_for_status() @@ -473,7 +482,8 @@ def gl_discover_containers(): return [] -def gl_fetch_container_logs(container_name, source, out_path): +def gl_fetch_container_logs(container_name, source, out_path, + chunk_from_iso, chunk_to_iso): """Fetch all logs for a container+source via Graylog. Returns line count.""" search_url = f"{GRAYLOG_BASE}/search/universal/absolute" # Use wildcard so partial names work (e.g. "spdk_8080" matches @@ -540,7 +550,7 @@ def _write_window(fh, q, f_iso, t_iso): return written # Probe total - msgs, total = _fetch_page(query, FROM_ISO, TO_ISO, 1, 0) + msgs, total = _fetch_page(query, chunk_from_iso, chunk_to_iso, 1, 0) if msgs is None: open(out_path, "w").close() return 0 @@ -548,11 +558,11 @@ def _write_window(fh, q, f_iso, t_iso): written = 0 with open(out_path, "w") as fh: if total <= MAX_RESULT_WINDOW: - written = _write_window(fh, query, FROM_ISO, TO_ISO) + written = _write_window(fh, query, chunk_from_iso, chunk_to_iso) else: # Split into 10-minute sub-windows - t = datetime.fromisoformat(FROM_ISO.replace("Z", "+00:00")) - t_end = datetime.fromisoformat(TO_ISO.replace("Z", "+00:00")) + t = datetime.fromisoformat(chunk_from_iso.replace("Z", "+00:00")) + t_end = datetime.fromisoformat(chunk_to_iso.replace("Z", "+00:00")) chunk = timedelta(minutes=10) while t < t_end: chunk_end = min(t + chunk, t_end) @@ -594,15 +604,96 @@ def discover_containers(graylog_ok): # Main # --------------------------------------------------------------------------- +def _safe(s): + return ( + s.replace("/", "_").replace("\\", "_") + .replace(":", "_").strip("_") + ) or "unnamed" + + +def _build_chunks(start, end, chunk_minutes=CHUNK_MINUTES): + """Build list of (chunk_start_dt, chunk_end_dt) in reverse order (newest first).""" + chunks = [] + t = start + while t < end: + c_end = min(t + timedelta(minutes=chunk_minutes), end) + chunks.append((t, c_end)) + t = c_end + chunks.reverse() # newest first + return chunks + + +def _fetch_chunk(pairs, chunk_start, chunk_end, chunk_dir, os_ok, + os_probe_cache, chunk_label): + """Fetch all container logs for a single time chunk. Returns total lines.""" + from concurrent.futures import ThreadPoolExecutor, as_completed + import threading + + c_from_ms = int(chunk_start.timestamp() * 1000) + c_to_ms = int(chunk_end.timestamp() * 1000) + c_from_iso = chunk_start.strftime("%Y-%m-%dT%H:%M:%S.000Z") + c_to_iso = chunk_end.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + os.makedirs(chunk_dir, exist_ok=True) + + max_workers = min(8, len(pairs)) + total_lines = 0 + lock = threading.Lock() + + def _fetch_one(container_name, source): + safe_cname = _safe(container_name) + if source: + safe_source = _safe(source) + fname = f"{safe_cname}__{safe_source}.log" + else: + fname = f"{safe_cname}.log" + out_path = os.path.join(chunk_dir, fname) + label = f"{container_name}@{source}" if source else container_name + + if os_ok: + n = os_fetch_container_logs( + container_name, source, out_path, + chunk_from_ms=c_from_ms, chunk_to_ms=c_to_ms, + probe_cache=os_probe_cache, + ) + else: + n = gl_fetch_container_logs( + container_name, source, out_path, + chunk_from_iso=c_from_iso, chunk_to_iso=c_to_iso, + ) + return label, n + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit(_fetch_one, cname, src): (cname, src) + for cname, src in sorted(pairs) + } + for future in as_completed(futures): + cname, src = futures[future] + label = f"{cname}@{src}" if src else cname + try: + label, n = future.result() + with lock: + total_lines += n + if n > 0: + print(f" {label:<55} {n:>8,} lines") + except Exception as exc: + print(f" {label:<55} FAILED: {exc}") + + return total_lines + + def main(): print("=" * 64) - print(" Graylog / OpenSearch Export Test") + print(" Graylog / OpenSearch Export (Chunked, Reverse Order)") print("=" * 64) print(f" Window : {FROM_ISO} -> {TO_ISO} ({DURATION_MINUTES} min)") + print(f" Chunk size : {CHUNK_MINUTES} min") print(f" Mode : {DEPLOY_MODE}") print(f" Field : {CNAME_FIELD}") print(f" Graylog : {GRAYLOG_BASE}") print(f" OpenSearch : {OPENSEARCH_BASE}") + print(f" Output : {OUTPUT_DIR}") print() # Check Graylog @@ -638,29 +729,13 @@ def main(): print("\nNeither Graylog nor OpenSearch is reachable. Exiting.") sys.exit(1) - # Discover (container, source) pairs + # Discover (container, source) pairs across the full window pairs = discover_containers(graylog_ok) if not pairs: print("\nNo containers found. Check your time window and log setup.") sys.exit(1) - # Create output directory - os.makedirs(OUTPUT_DIR, exist_ok=True) - - # Fetch strategy: prefer OpenSearch (scroll API handles large windows), - # fall back to Graylog only when OpenSearch is unavailable. - fetch_via = "OpenSearch" if os_ok else "Graylog" - print(f"\n[3] Fetching logs for {len(pairs)} (container, source) pairs " - f"via {fetch_via} -> {OUTPUT_DIR}") - print("-" * 64) - - def _safe(s): - return ( - s.replace("/", "_").replace("\\", "_") - .replace(":", "_").strip("_") - ) or "unnamed" - - # Pre-populate probe cache before parallel fetch + # Pre-populate probe cache once (shared across all chunks) os_probe_cache = {} if os_ok: try: @@ -669,52 +744,44 @@ def _safe(s): except Exception as exc: print(f" WARN: Failed to pre-populate probe cache: {exc}") - from concurrent.futures import ThreadPoolExecutor, as_completed - import threading - - max_workers = min(8, len(pairs)) - total_lines = 0 - lock = threading.Lock() + # Build 1-hour chunks in reverse order (newest first) + chunks = _build_chunks(start_dt, end_dt, CHUNK_MINUTES) + num_chunks = len(chunks) - def _fetch_one(container_name, source): - """Fetch a single container's logs. Returns (label, line_count).""" - safe_cname = _safe(container_name) - if source: - safe_source = _safe(source) - fname = f"{safe_cname}__{safe_source}.log" - else: - fname = f"{safe_cname}.log" - out_path = os.path.join(OUTPUT_DIR, fname) - label = f"{container_name}@{source}" if source else container_name + fetch_via = "OpenSearch" if os_ok else "Graylog" + print(f"\n[3] Fetching logs in {num_chunks} chunk(s) of {CHUNK_MINUTES} min " + f"(reverse order, newest first)") + print(f" {len(pairs)} (container, source) pairs via {fetch_via}") + print("=" * 64) - if os_ok: - n = os_fetch_container_logs( - container_name, source, out_path, - probe_cache=os_probe_cache, - ) - else: - n = gl_fetch_container_logs(container_name, source, out_path) - return label, n + os.makedirs(OUTPUT_DIR, exist_ok=True) + grand_total = 0 + + for idx, (c_start, c_end) in enumerate(chunks, 1): + c_start_label = c_start.strftime("%Y%m%d_%H%M") + c_end_label = c_end.strftime("%H%M") + chunk_dir_name = f"chunk_{idx:02d}_of_{num_chunks:02d}_{c_start_label}_to_{c_end_label}" + chunk_dir = os.path.join(OUTPUT_DIR, chunk_dir_name) + + c_start_pretty = c_start.strftime("%Y-%m-%d %H:%M") + c_end_pretty = c_end.strftime("%H:%M") + print(f"\n--- Chunk {idx}/{num_chunks}: {c_start_pretty} -> {c_end_pretty} " + f"-> {chunk_dir_name}/ ---") + + chunk_lines = _fetch_chunk( + pairs, c_start, c_end, chunk_dir, os_ok, + os_probe_cache, chunk_dir_name, + ) + grand_total += chunk_lines - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit(_fetch_one, cname, src): (cname, src) - for cname, src in sorted(pairs) - } - for future in as_completed(futures): - cname, src = futures[future] - label = f"{cname}@{src}" if src else cname - try: - label, n = future.result() - with lock: - total_lines += n - print(f" {label:<60} {n:>8,} lines") - except Exception as exc: - print(f" {label:<60} FAILED: {exc}") + print(f" Chunk {idx}/{num_chunks} done: {chunk_lines:,} lines") - print("-" * 64) - print(f" TOTAL: {total_lines:,} lines from {len(pairs)} (container, source) pairs") + print() + print("=" * 64) + print(f" TOTAL: {grand_total:,} lines from {len(pairs)} pairs " + f"across {num_chunks} chunk(s)") print(f" Output: {os.path.abspath(OUTPUT_DIR)}") + print("=" * 64) print() From cddd548361ffd21a5e7c454ebbd98e83a7bd780e Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Sun, 23 Aug 2026 17:40:16 +0530 Subject: [PATCH 11/12] Fix graylog export: sequential fetching + 429 retry to prevent circuit breaker OOM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parallel 8-thread fetching was creating 8 simultaneous OpenSearch scroll contexts, each consuming ~125MB heap. With 500+ containers, this triggered circuit_breaking_exception ("Data too large") at the 972MB heap limit. Changes (matching patterns from collect_logs.py in stress pipelines): - Sequential container fetching: one scroll context at a time instead of 8 parallel threads — prevents heap pressure - 429 retry with backoff: wait 10s/30s/60s on circuit_breaking_exception instead of silently dropping data - Graylog fallback: if OpenSearch returns 0 lines for a container, automatically retry via Graylog REST API - Scroll keepalive back to 2m: with sequential fetching, scroll contexts complete quickly and don't need 5m keepalive - Progress counter: [1/503] prefix shows sequential progress Co-Authored-By: Claude Opus 4.6 --- e2e/utils/test_graylog_export.py | 113 +++++++++++++++++++------------ 1 file changed, 71 insertions(+), 42 deletions(-) diff --git a/e2e/utils/test_graylog_export.py b/e2e/utils/test_graylog_export.py index 205471e0b..c5ce3f6f9 100755 --- a/e2e/utils/test_graylog_export.py +++ b/e2e/utils/test_graylog_export.py @@ -47,7 +47,7 @@ import os import sys -import math +import time from datetime import datetime, timezone, timedelta try: @@ -298,6 +298,31 @@ def os_discover_containers(): return [] +MAX_RETRIES = 3 +RETRY_BACKOFF = [10, 30, 60] # seconds between retries on 429 + + +def _os_request_with_retry(session, method, url, retries=MAX_RETRIES, **kwargs): + """Make an HTTP request with retry on 429 (circuit breaker) errors.""" + for attempt in range(retries + 1): + try: + r = session.request(method, url, **kwargs) + if r.status_code == 429 and attempt < retries: + wait = RETRY_BACKOFF[min(attempt, len(RETRY_BACKOFF) - 1)] + print(f" 429 circuit breaker hit, waiting {wait}s " + f"(attempt {attempt + 1}/{retries}) ...", file=sys.stderr) + time.sleep(wait) + continue + return r + except requests.RequestException: + if attempt < retries: + wait = RETRY_BACKOFF[min(attempt, len(RETRY_BACKOFF) - 1)] + time.sleep(wait) + continue + raise + return r # unreachable but keeps linters happy + + def os_fetch_container_logs(container_name, source, out_path, chunk_from_ms, chunk_to_ms, probe_cache=None): @@ -343,11 +368,12 @@ def _fmt(src): text = str(src.get("message", "")).replace("\n", "\\n") return f"{ts} src={s} ctr={cname} lvl={lvl} {text}" - init_url = f"{OPENSEARCH_BASE}/{index}/_search?scroll=5m" + init_url = f"{OPENSEARCH_BASE}/{index}/_search?scroll=2m" written = 0 try: - r = os_session.post(init_url, json=body, timeout=60) + r = _os_request_with_retry( + os_session, "POST", init_url, json=body, timeout=60) if not r.ok: print(f" WARN: OpenSearch scroll failed for {container_name}: " f"HTTP {r.status_code} {r.text[:300]}", file=sys.stderr) @@ -378,12 +404,17 @@ def _fmt(src): if len(hits) < PAGE_SIZE or not scroll_id: break try: - sc_r = os_session.post( + sc_r = _os_request_with_retry( + os_session, "POST", f"{OPENSEARCH_BASE}/_search/scroll", - json={"scroll": "5m", "scroll_id": scroll_id}, + json={"scroll": "2m", "scroll_id": scroll_id}, timeout=60, ) - sc_r.raise_for_status() + if not sc_r.ok: + print(f" WARN: scroll continuation failed for " + f"{container_name}: HTTP {sc_r.status_code}", + file=sys.stderr) + break sc_data = sc_r.json() scroll_id = sc_data.get("_scroll_id", scroll_id) hits = sc_data.get("hits", {}).get("hits", []) @@ -624,11 +655,14 @@ def _build_chunks(start, end, chunk_minutes=CHUNK_MINUTES): def _fetch_chunk(pairs, chunk_start, chunk_end, chunk_dir, os_ok, - os_probe_cache, chunk_label): - """Fetch all container logs for a single time chunk. Returns total lines.""" - from concurrent.futures import ThreadPoolExecutor, as_completed - import threading + graylog_ok, os_probe_cache, chunk_label): + """Fetch all container logs for a single time chunk. + Containers are fetched SEQUENTIALLY (one scroll context at a time) + to avoid OpenSearch circuit breaker / heap pressure. + On OpenSearch failure, falls back to Graylog for that container. + Returns total lines. + """ c_from_ms = int(chunk_start.timestamp() * 1000) c_to_ms = int(chunk_end.timestamp() * 1000) c_from_iso = chunk_start.strftime("%Y-%m-%dT%H:%M:%S.000Z") @@ -636,11 +670,9 @@ def _fetch_chunk(pairs, chunk_start, chunk_end, chunk_dir, os_ok, os.makedirs(chunk_dir, exist_ok=True) - max_workers = min(8, len(pairs)) total_lines = 0 - lock = threading.Lock() - def _fetch_one(container_name, source): + for i, (container_name, source) in enumerate(sorted(pairs), 1): safe_cname = _safe(container_name) if source: safe_source = _safe(source) @@ -650,35 +682,32 @@ def _fetch_one(container_name, source): out_path = os.path.join(chunk_dir, fname) label = f"{container_name}@{source}" if source else container_name - if os_ok: - n = os_fetch_container_logs( - container_name, source, out_path, - chunk_from_ms=c_from_ms, chunk_to_ms=c_to_ms, - probe_cache=os_probe_cache, - ) - else: - n = gl_fetch_container_logs( - container_name, source, out_path, - chunk_from_iso=c_from_iso, chunk_to_iso=c_to_iso, - ) - return label, n + n = 0 + try: + if os_ok: + n = os_fetch_container_logs( + container_name, source, out_path, + chunk_from_ms=c_from_ms, chunk_to_ms=c_to_ms, + probe_cache=os_probe_cache, + ) + # Fallback to Graylog if OpenSearch returned 0 lines + if n == 0 and graylog_ok: + n = gl_fetch_container_logs( + container_name, source, out_path, + chunk_from_iso=c_from_iso, chunk_to_iso=c_to_iso, + ) + else: + n = gl_fetch_container_logs( + container_name, source, out_path, + chunk_from_iso=c_from_iso, chunk_to_iso=c_to_iso, + ) + except Exception as exc: + print(f" [{i}/{len(pairs)}] {label:<50} FAILED: {exc}") + continue - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit(_fetch_one, cname, src): (cname, src) - for cname, src in sorted(pairs) - } - for future in as_completed(futures): - cname, src = futures[future] - label = f"{cname}@{src}" if src else cname - try: - label, n = future.result() - with lock: - total_lines += n - if n > 0: - print(f" {label:<55} {n:>8,} lines") - except Exception as exc: - print(f" {label:<55} FAILED: {exc}") + total_lines += n + if n > 0: + print(f" [{i}/{len(pairs)}] {label:<50} {n:>8,} lines") return total_lines @@ -770,7 +799,7 @@ def main(): chunk_lines = _fetch_chunk( pairs, c_start, c_end, chunk_dir, os_ok, - os_probe_cache, chunk_dir_name, + graylog_ok, os_probe_cache, chunk_dir_name, ) grand_total += chunk_lines From e47fa56b9b9019c4ed5980d5c81973cb95eda64e Mon Sep 17 00:00:00 2001 From: Raunak Jalan Date: Sun, 23 Aug 2026 21:35:23 +0530 Subject: [PATCH 12/12] Fix collect-logs summary exceeding 1024KB limit and remove artifact upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace per-file listing in job summary with per-chunk summary (chunk name, file count, size) to stay under GitHub's 1024KB limit - Remove total_lines count from summary (was cat-ing all logs just for a count) - Remove GitHub artifact upload step — logs go to NFS only Co-Authored-By: Claude Opus 4.6 --- .github/workflows/collect-logs.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/collect-logs.yml b/.github/workflows/collect-logs.yml index c677721a3..f1b98c3a1 100755 --- a/.github/workflows/collect-logs.yml +++ b/.github/workflows/collect-logs.yml @@ -726,7 +726,6 @@ jobs: gl_ip="${GRAYLOG_IP:-$MGMT_IP}" file_count="$(find "${OUTPUT_DIR}" -type f \( -name '*.log' -o -name '*.tar.gz' \) 2>/dev/null | wc -l || echo 0)" total_size="$(du -sh "${OUTPUT_DIR}" 2>/dev/null | awk '{print $1}' || echo 'unknown')" - total_lines="$(find "${OUTPUT_DIR}" -name '*.log' -exec cat {} + 2>/dev/null | wc -l || echo 0)" { echo "## Log Collection Summary" @@ -747,11 +746,20 @@ jobs: fi echo "| **Log Files** | ${file_count} |" echo "| **Total Size** | ${total_size} |" - echo "| **Total Lines** | ${total_lines} |" echo "" - echo "### Files Collected" + echo "### Chunks" echo '```' - find "${OUTPUT_DIR}" -type f \( -name '*.log' -o -name '*.tar.gz' \) -printf '%P (%s bytes)\n' 2>/dev/null | sort || echo "(none)" + for d in "${OUTPUT_DIR}"/chunk_*; do + [ -d "$d" ] || continue + chunk_name=$(basename "$d") + chunk_files=$(find "$d" -name '*.log' -size +0 2>/dev/null | wc -l) + chunk_size=$(du -sh "$d" 2>/dev/null | awk '{print $1}') + echo "${chunk_name} ${chunk_files} files ${chunk_size}" + done + # Fallback if no chunk dirs (flat layout) + if ! ls -d "${OUTPUT_DIR}"/chunk_* >/dev/null 2>&1; then + echo "${file_count} files ${total_size}" + fi echo '```' } >> "$GITHUB_STEP_SUMMARY" @@ -775,14 +783,6 @@ jobs: --source_dir "${OUTPUT_DIR}" \ --run_id "collected-logs-$(date +%Y%m%d_%H%M%S)" || true - - name: Upload as GitHub artifact - if: always() - uses: actions/upload-artifact@v4 - with: - name: collected-logs-${{ inputs.DEPLOY_MODE }}-${{ inputs.DEPLOY_MODE == 'k8s-native' && inputs.cluster_environment || inputs.MGMT_IP }}-${{ github.run_id }} - path: ${{ inputs.OUTPUT_DIR }}/** - if-no-files-found: warn - # ============================================================ # K8s-native: Cleanup kubeconfig # ============================================================