diff --git a/.github/workflows/bare-metal-deploy.yml b/.github/workflows/bare-metal-deploy.yml index 8814991e6..455b65ae5 100755 --- a/.github/workflows/bare-metal-deploy.yml +++ b/.github/workflows/bare-metal-deploy.yml @@ -192,8 +192,8 @@ jobs: # If the cluster is running on k8s, we need to set the environment variable DEPLOY_CMD="./bootstrap-cluster.sh \ - --max-lvol 10 --max-snap 10 - --max-size 400G --number-of-devices 1 + --max-subsys 10 --max-snap 10 + --max-size 400G --number-of-devices 1 --sbcli-cmd sbcli-dev --spdk-debug" if [ "${{ inputs.k8s_snode }}" == "true" ]; then diff --git a/.github/workflows/collect-logs.yml b/.github/workflows/collect-logs.yml index ffdd4891f..f1b98c3a1 100755 --- a/.github/workflows/collect-logs.yml +++ b/.github/workflows/collect-logs.yml @@ -104,6 +104,15 @@ on: # ========================= # Upload # ========================= + RUNNER: + description: "Runner to use (must have NFS mount at /mnt/nfs_share)" + required: false + default: "vm22" + type: choice + options: + - vm22 + - vm-runner-43 + UPLOAD_TO_MINIO: description: "Upload collected logs to MinIO" required: false @@ -117,7 +126,7 @@ concurrency: jobs: collect-logs: name: "Collect logs (${{ inputs.DEPLOY_MODE }}${{ inputs.DEPLOY_MODE == 'k8s-native' && format(' / {0}', inputs.cluster_environment) || '' }})" - runs-on: ${{ inputs.DEPLOY_MODE == 'k8s-native' && inputs.cluster_environment == 'aws-openshift' && 'vm-runner-43' || 'self-hosted' }} + runs-on: ${{ inputs.RUNNER || 'vm22' }} timeout-minutes: 720 env: @@ -690,7 +699,8 @@ jobs: TIMESTAMP=$(date -u "+%Y%m%d-%H%M%S") NFS_DEST="${NFS_BASE}/graylog_collected-${TIMESTAMP}" - mkdir -p "${NFS_DEST}" + sudo mkdir -p "${NFS_DEST}" + sudo chown "$(id -u):$(id -g)" "${NFS_DEST}" echo "=== Copying collected logs to NFS ===" echo " Source: ${OUTPUT_DIR}" @@ -716,7 +726,6 @@ jobs: gl_ip="${GRAYLOG_IP:-$MGMT_IP}" file_count="$(find "${OUTPUT_DIR}" -type f \( -name '*.log' -o -name '*.tar.gz' \) 2>/dev/null | wc -l || echo 0)" total_size="$(du -sh "${OUTPUT_DIR}" 2>/dev/null | awk '{print $1}' || echo 'unknown')" - total_lines="$(cat "${OUTPUT_DIR}"/*.log 2>/dev/null | wc -l || echo 0)" { echo "## Log Collection Summary" @@ -737,11 +746,20 @@ jobs: fi echo "| **Log Files** | ${file_count} |" echo "| **Total Size** | ${total_size} |" - echo "| **Total Lines** | ${total_lines} |" echo "" - echo "### Files Collected" + echo "### Chunks" echo '```' - find "${OUTPUT_DIR}" -type f \( -name '*.log' -o -name '*.tar.gz' \) -printf '%P (%s bytes)\n' 2>/dev/null | sort || echo "(none)" + for d in "${OUTPUT_DIR}"/chunk_*; do + [ -d "$d" ] || continue + chunk_name=$(basename "$d") + chunk_files=$(find "$d" -name '*.log' -size +0 2>/dev/null | wc -l) + chunk_size=$(du -sh "$d" 2>/dev/null | awk '{print $1}') + echo "${chunk_name} ${chunk_files} files ${chunk_size}" + done + # Fallback if no chunk dirs (flat layout) + if ! ls -d "${OUTPUT_DIR}"/chunk_* >/dev/null 2>&1; then + echo "${file_count} files ${total_size}" + fi echo '```' } >> "$GITHUB_STEP_SUMMARY" @@ -765,14 +783,6 @@ jobs: --source_dir "${OUTPUT_DIR}" \ --run_id "collected-logs-$(date +%Y%m%d_%H%M%S)" || true - - name: Upload as GitHub artifact - if: always() - uses: actions/upload-artifact@v4 - with: - name: collected-logs-${{ inputs.DEPLOY_MODE }}-${{ inputs.DEPLOY_MODE == 'k8s-native' && inputs.cluster_environment || inputs.MGMT_IP }}-${{ github.run_id }} - path: ${{ inputs.OUTPUT_DIR }}/** - if-no-files-found: warn - # ============================================================ # K8s-native: Cleanup kubeconfig # ============================================================ diff --git a/.github/workflows/k8s-e2e-ha.yaml b/.github/workflows/k8s-e2e-ha.yaml index 2190b2161..05b781d57 100755 --- a/.github/workflows/k8s-e2e-ha.yaml +++ b/.github/workflows/k8s-e2e-ha.yaml @@ -176,7 +176,7 @@ jobs: run: | cd $GITHUB_WORKSPACE/simplyBlockDeploy ./bootstrap-cluster.sh --sbcli-cmd "$SBCLI_CMD" \ - --max-lvol 10 --max-snap 10 --max-prov 500G --number-of-devices 1 \ + --max-subsys 10 --max-snap 10 --number-of-devices 1 \ --distr-ndcs $NDCS \ --distr-npcs $NPCS \ --distr-bs $BS \ diff --git a/.github/workflows/k8s-native-cross-cluster-restore.yaml b/.github/workflows/k8s-native-cross-cluster-restore.yaml index d7a24cb32..3af92f053 100644 --- a/.github/workflows/k8s-native-cross-cluster-restore.yaml +++ b/.github/workflows/k8s-native-cross-cluster-restore.yaml @@ -58,7 +58,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -640,7 +640,7 @@ jobs: IFC_NAMES="${{ github.event.inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ github.event.inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ github.event.inputs.max_subsys || '30' }}" SB_REPO="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ github.event.inputs.simplyblock_image }}" SPDK_IMAGE="${{ github.event.inputs.spdk_image }}" @@ -650,6 +650,30 @@ jobs: FORCE_FORMAT_4K="${{ (github.event.inputs.cluster_environment == 'aws-openshift' || github.event.inputs.cluster_environment == 'openshift-local' || github.event.inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ github.event.inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${C1_WORKERS%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + # Build C1 worker nodes YAML WORKER_YAML="" IFS=',' read -ra C1_NODES <<< "${C1_WORKERS}" @@ -675,9 +699,7 @@ jobs: namespace: ${NS_C1} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} @@ -688,6 +710,9 @@ jobs: capacity: 96 provisionedCapacity: 98 ${BACKUP_SPEC} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -711,9 +736,7 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 @@ -735,7 +758,7 @@ jobs: IFC_NAMES="${{ github.event.inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ github.event.inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ github.event.inputs.max_subsys || '30' }}" SB_REPO="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ github.event.inputs.simplyblock_image }}" SPDK_IMAGE="${{ github.event.inputs.spdk_image }}" @@ -745,6 +768,30 @@ jobs: FORCE_FORMAT_4K="${{ (github.event.inputs.cluster_environment == 'aws-openshift' || github.event.inputs.cluster_environment == 'openshift-local' || github.event.inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ github.event.inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${C2_WORKERS%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + # Build C2 worker nodes YAML WORKER_YAML="" IFS=',' read -ra C2_NODES <<< "${C2_WORKERS}" @@ -770,9 +817,7 @@ jobs: namespace: ${NS_C2} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} @@ -783,6 +828,9 @@ jobs: capacity: 96 provisionedCapacity: 98 ${BACKUP_SPEC} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -806,9 +854,7 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/k8s-native-e2e-add-node.yaml b/.github/workflows/k8s-native-e2e-add-node.yaml index b935ce2ff..194d8833f 100644 --- a/.github/workflows/k8s-native-e2e-add-node.yaml +++ b/.github/workflows/k8s-native-e2e-add-node.yaml @@ -42,7 +42,7 @@ on: ifc_names: type: string default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: type: string default: '30' ssh_user: @@ -132,7 +132,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -919,7 +919,7 @@ jobs: IFC_NAMES="${{ inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ inputs.max_subsys || '30' }}" SB_REPO="${{ inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ inputs.simplyblock_image }}" SPDK_IMAGE="${{ inputs.spdk_image }}" @@ -931,6 +931,31 @@ jobs: FORCE_FORMAT_4K="${{ (inputs.cluster_environment == 'aws-openshift' || inputs.cluster_environment == 'openshift-local' || inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${{ inputs.worker_nodes }}" + FIRST_WORKER="${FIRST_WORKER%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + RESERVED_CPU_YAML="" if [ "${{ inputs.cluster_environment }}" = "openshift-baremetal" ]; then RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' @@ -991,9 +1016,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} @@ -1005,6 +1028,9 @@ jobs: provisionedCapacity: 98 ${BACKUP_SPEC} ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -1030,10 +1056,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/k8s-native-e2e-node-migration.yaml b/.github/workflows/k8s-native-e2e-node-migration.yaml index e46cf1baf..a27eb48a3 100644 --- a/.github/workflows/k8s-native-e2e-node-migration.yaml +++ b/.github/workflows/k8s-native-e2e-node-migration.yaml @@ -48,7 +48,7 @@ on: ifc_names: type: string default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: type: string default: '30' ssh_user: @@ -147,7 +147,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -915,7 +915,7 @@ jobs: IFC_NAMES="${{ inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ inputs.max_subsys || '30' }}" SB_REPO="${{ inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ inputs.simplyblock_image }}" SPDK_IMAGE="${{ inputs.spdk_image }}" @@ -927,6 +927,31 @@ jobs: FORCE_FORMAT_4K="${{ (inputs.cluster_environment == 'aws-openshift' || inputs.cluster_environment == 'openshift-local' || inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${{ inputs.worker_nodes }}" + FIRST_WORKER="${FIRST_WORKER%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + RESERVED_CPU_YAML="" if [ "${{ inputs.cluster_environment }}" = "openshift-baremetal" ]; then RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' @@ -987,9 +1012,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} @@ -1001,6 +1024,9 @@ jobs: provisionedCapacity: 98 ${BACKUP_SPEC} ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -1026,10 +1052,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/k8s-native-e2e.yaml b/.github/workflows/k8s-native-e2e.yaml index d80ac4a87..de43591bf 100644 --- a/.github/workflows/k8s-native-e2e.yaml +++ b/.github/workflows/k8s-native-e2e.yaml @@ -44,8 +44,8 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: - description: 'Max logical volume count per storage node' + max_subsys: + description: 'Max subsystem count (cluster-level)' required: false default: '30' ssh_user: @@ -826,7 +826,7 @@ jobs: IFC_NAMES="${{ github.event.inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ github.event.inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ github.event.inputs.max_subsys || '30' }}" SB_REPO="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ github.event.inputs.simplyblock_image }}" SPDK_IMAGE="${{ github.event.inputs.spdk_image }}" @@ -838,6 +838,31 @@ jobs: FORCE_FORMAT_4K="${{ (github.event.inputs.cluster_environment == 'aws-openshift' || github.event.inputs.cluster_environment == 'openshift-local' || github.event.inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ github.event.inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${{ github.event.inputs.worker_nodes }}" + FIRST_WORKER="${FIRST_WORKER%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + RESERVED_CPU_YAML="" if [ "${{ github.event.inputs.cluster_environment }}" = "openshift-baremetal" ]; then RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' @@ -898,9 +923,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} @@ -912,6 +935,9 @@ jobs: provisionedCapacity: 98 ${BACKUP_SPEC} ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -937,10 +963,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 @@ -962,9 +986,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} @@ -976,6 +998,9 @@ jobs: provisionedCapacity: 98 ${BACKUP_SPEC} ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -1001,10 +1026,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/k8s-native-stress.yaml b/.github/workflows/k8s-native-stress.yaml index d01a057d4..83f8d60fc 100644 --- a/.github/workflows/k8s-native-stress.yaml +++ b/.github/workflows/k8s-native-stress.yaml @@ -44,7 +44,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -690,7 +690,7 @@ jobs: IFC_NAMES="${{ github.event.inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ github.event.inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ github.event.inputs.max_subsys || '30' }}" SB_REPO="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ github.event.inputs.simplyblock_image }}" SPDK_IMAGE="${{ github.event.inputs.spdk_image }}" @@ -702,6 +702,31 @@ jobs: FORCE_FORMAT_4K="${{ (github.event.inputs.cluster_environment == 'aws-openshift' || github.event.inputs.cluster_environment == 'openshift-local' || github.event.inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ github.event.inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${{ github.event.inputs.worker_nodes }}" + FIRST_WORKER="${FIRST_WORKER%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + RESERVED_CPU_YAML="" if [ "${{ github.event.inputs.cluster_environment }}" = "openshift-baremetal" ]; then RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' @@ -747,9 +772,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} @@ -760,6 +783,9 @@ jobs: capacity: 96 provisionedCapacity: 98 ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -785,10 +811,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 @@ -810,9 +834,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} @@ -823,6 +845,9 @@ jobs: capacity: 96 provisionedCapacity: 98 ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -848,10 +873,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/k8s-native-upgrade.yaml b/.github/workflows/k8s-native-upgrade.yaml index dcf21c647..f4a4631d7 100644 --- a/.github/workflows/k8s-native-upgrade.yaml +++ b/.github/workflows/k8s-native-upgrade.yaml @@ -74,9 +74,9 @@ on: required: false default: 'br-ex:enp2s0f0' cluster_params: - description: 'Cluster config: ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_lvol=30' + description: 'Cluster config: ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_subsys=30' required: false - default: 'ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_lvol=30' + default: 'ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_subsys=30' cluster_environment: description: 'Target cluster environment' required: true @@ -239,7 +239,7 @@ jobs: # ── Parse cluster params ── - name: Parse cluster parameters run: | - PARAMS="${{ github.event.inputs.cluster_params || 'ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_lvol=30' }}" + PARAMS="${{ github.event.inputs.cluster_params || 'ndcs=1,npcs=1,bs=4096,chunk_bs=4096,partitions=0,jm_count=3,max_subsys=30' }}" for kv in $(echo "$PARAMS" | tr ',' '\n'); do key=$(echo "$kv" | cut -d= -f1 | tr '[:lower:]' '[:upper:]') val=$(echo "$kv" | cut -d= -f2) @@ -505,7 +505,7 @@ jobs: SB_REPO="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ github.event.inputs.base_simplyblock_image }}" SPDK_IMAGE="${{ github.event.inputs.base_spdk_image }}" - MAX_LVOL="${MAX_LVOL:-30}" + MAX_SUBSYS="${MAX_SUBSYS:-30}" # Build worker YAML IFS=',' read -ra NODES <<< "${{ github.event.inputs.worker_nodes }}" @@ -541,6 +541,31 @@ jobs: PCIE_MODEL_YAML=' pcieModel: "SAMSUNG MZQLB1T9HAJR-00007"' fi + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${{ github.event.inputs.worker_nodes }}" + FIRST_WORKER="${FIRST_WORKER%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + # Build vault settings (conditional on TLS) VAULT_SETTINGS="" if [ "${{ github.event.inputs.tls_enabled }}" = "true" ]; then @@ -557,9 +582,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} @@ -570,6 +593,9 @@ jobs: capacity: 96 provisionedCapacity: 98 ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -594,10 +620,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/monitoring-suite-docker.yaml b/.github/workflows/monitoring-suite-docker.yaml index 296ffe9e2..8c3f0a95a 100755 --- a/.github/workflows/monitoring-suite-docker.yaml +++ b/.github/workflows/monitoring-suite-docker.yaml @@ -72,7 +72,7 @@ on: # Bootstrap params # ========================= BOOTSTRAP_MAX_LVOL: - description: "bootstrap: --max-lvol" + description: "bootstrap: --max-subsys (cluster-wide max subsystems per node)" required: true default: "75" BOOTSTRAP_DATA_CHUNKS: @@ -546,7 +546,7 @@ jobs: set +e ./bootstrap-cluster.sh \ --sbcli-cmd "${SBCLI_CMD}" \ - --max-lvol "${BOOTSTRAP_MAX_LVOL}" \ + --max-subsys "${BOOTSTRAP_MAX_LVOL}" \ --data-chunks-per-stripe "${BOOTSTRAP_DATA_CHUNKS}" \ --parity-chunks-per-stripe "${BOOTSTRAP_PARITY_CHUNKS}" \ --journal-partition "${BOOTSTRAP_JOURNAL_PARTITION}" \ diff --git a/.github/workflows/monitoring-suite-k8s-native.yaml b/.github/workflows/monitoring-suite-k8s-native.yaml index 729f7b6a6..5a55cd0a7 100644 --- a/.github/workflows/monitoring-suite-k8s-native.yaml +++ b/.github/workflows/monitoring-suite-k8s-native.yaml @@ -59,7 +59,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -616,7 +616,7 @@ jobs: IFC_NAMES="${{ github.event.inputs.ifc_names || 'br-ex:enp2s0f0' }}" MGMT_IFC="${IFC_NAMES%%:*}" DATA_NICS="${IFC_NAMES#*:}" - MAX_LVOL="${{ github.event.inputs.max_lvol || '30' }}" + MAX_SUBSYS="${{ github.event.inputs.max_subsys || '30' }}" SB_REPO="${{ github.event.inputs.simplyblock_repository || 'public.ecr.aws/simply-block/simplyblock' }}" SB_TAG="${{ github.event.inputs.simplyblock_image }}" SPDK_IMAGE="${{ github.event.inputs.spdk_image }}" @@ -628,6 +628,31 @@ jobs: FORCE_FORMAT_4K="${{ (github.event.inputs.cluster_environment == 'aws-openshift' || github.event.inputs.cluster_environment == 'openshift-local' || github.event.inputs.cluster_environment == 'openshift-baremetal') && 'true' || 'false' }}" DRIVE_SIZE_RANGE="${{ github.event.inputs.cluster_environment == 'openshift-baremetal' && '1500G-2000G' || '1.7T-2T' }}" + # Convert partitions to enableJournalDevice (inverted: 0→true, 1→false) + if [ "${PARTITIONS}" = "0" ]; then + ENABLE_JOURNAL_DEVICE="true" + else + ENABLE_JOURNAL_DEVICE="false" + fi + + # Calculate vcpuCount from CORE_PERCENTAGE by querying actual CPU count on a worker node + FIRST_WORKER="${{ github.event.inputs.worker_nodes }}" + FIRST_WORKER="${FIRST_WORKER%%,*}" + if [ "$OPENSHIFT_CLUSTER" = "true" ]; then + TOTAL_CPUS=$(oc debug node/"${FIRST_WORKER}" -- chroot /host nproc 2>/dev/null || echo "0") + else + TOTAL_CPUS=$(kubectl get node "${FIRST_WORKER}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "0") + fi + TOTAL_CPUS=$(echo "${TOTAL_CPUS}" | tr -d '[:space:]') + if [ "${TOTAL_CPUS}" -gt 0 ] 2>/dev/null; then + VCPU_COUNT=$(( TOTAL_CPUS * CORE_PERCENTAGE / 100 )) + [ "${VCPU_COUNT}" -lt 1 ] && VCPU_COUNT=1 + else + echo "WARNING: Could not determine CPU count, defaulting vcpuCount to 4" + VCPU_COUNT=4 + fi + echo "Calculated vcpuCount=${VCPU_COUNT} (${TOTAL_CPUS} CPUs * ${CORE_PERCENTAGE}%)" + RESERVED_CPU_YAML="" if [ "${{ github.event.inputs.cluster_environment }}" = "openshift-baremetal" ]; then RESERVED_CPU_YAML=' reservedSystemCPU: "0,1,10,11,16,17,26,27"' @@ -670,9 +695,7 @@ jobs: namespace: ${NAMESPACE} spec: fabricType: tcp - isSingleNode: false enableNodeAffinity: true - strictNodeAntiAffinity: false stripe: dataChunks: ${NDCS} parityChunks: ${NPCS} @@ -683,6 +706,9 @@ jobs: capacity: 96 provisionedCapacity: 98 ${VAULT_SETTINGS} + maxSubsystemCount: ${MAX_SUBSYS} + vcpuCount: ${VCPU_COUNT} + maxHugePagesSize: "17G" --- apiVersion: storage.simplyblock.io/v1alpha1 kind: StoragePool @@ -708,10 +734,8 @@ jobs: mgmtIfname: ${MGMT_IFC} dataIfname: - ${DATA_NICS} - maxSubsystemCount: ${MAX_LVOL} ${RESERVED_CPU_YAML} - partitions: ${PARTITIONS} - corePercentage: ${CORE_PERCENTAGE} + enableJournalDevice: ${ENABLE_JOURNAL_DEVICE} journalManager: count: ${JM_COUNT} percentPerDevice: 3 diff --git a/.github/workflows/topology-suite-k8s-add-node.yml b/.github/workflows/topology-suite-k8s-add-node.yml index c5b32de30..7ae417f18 100755 --- a/.github/workflows/topology-suite-k8s-add-node.yml +++ b/.github/workflows/topology-suite-k8s-add-node.yml @@ -65,7 +65,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -212,7 +212,7 @@ jobs: worker_nodes: ${{ inputs.worker_nodes }} new_worker_nodes: ${{ inputs.new_worker_nodes }} ifc_names: ${{ inputs.ifc_names || 'br-ex:enp2s0f0' }} - max_lvol: ${{ inputs.max_lvol || '30' }} + max_subsys: ${{ inputs.max_subsys || '30' }} ssh_user: ${{ inputs.ssh_user || 'root' }} key_path: ${{ inputs.key_path }} send_slack_notification: false diff --git a/.github/workflows/topology-suite-k8s-migration.yml b/.github/workflows/topology-suite-k8s-migration.yml index 7df57b0a7..409608853 100755 --- a/.github/workflows/topology-suite-k8s-migration.yml +++ b/.github/workflows/topology-suite-k8s-migration.yml @@ -74,7 +74,7 @@ on: description: 'Network interfaces (mgmt_ifc:data_nics)' required: false default: 'br-ex:enp2s0f0' - max_lvol: + max_subsys: description: 'Max logical volume count per storage node' required: false default: '30' @@ -214,7 +214,7 @@ jobs: new_ssd_pcie: ${{ inputs.new_ssd_pcie || '' }} reattach_volume: ${{ inputs.reattach_volume }} ifc_names: ${{ inputs.ifc_names || 'br-ex:enp2s0f0' }} - max_lvol: ${{ inputs.max_lvol || '30' }} + max_subsys: ${{ inputs.max_subsys || '30' }} ssh_user: ${{ inputs.ssh_user || 'root' }} key_path: ${{ inputs.key_path }} send_slack_notification: false diff --git a/e2e/e2e_tests/backup/test_backup_restore.py b/e2e/e2e_tests/backup/test_backup_restore.py index 623880776..c7ea24025 100644 --- a/e2e/e2e_tests/backup/test_backup_restore.py +++ b/e2e/e2e_tests/backup/test_backup_restore.py @@ -1057,10 +1057,10 @@ def _verify_lvol_crypto(self, lvol_id: str, label: str = ""): details = self.sbcli_utils.get_lvol_details(lvol_id=lvol_id) assert details, f"{label}: get_lvol_details returned empty for {lvol_id}" d = details[0] if isinstance(details, list) else details - crypto_val = d.get("crypto") or d.get("encryption") or d.get("Crypto") - self.logger.info(f"{label}: lvol {lvol_id} crypto={crypto_val}") + crypto_val = d.get("crypto_bdev") or d.get("crypto") or d.get("encryption") + self.logger.info(f"{label}: lvol {lvol_id} crypto_bdev={crypto_val}") assert crypto_val, ( - f"{label}: restored lvol {lvol_id} expected crypto=True, " + f"{label}: restored lvol {lvol_id} expected crypto_bdev to be set, " f"got {crypto_val!r}. Full details: {d}") return d @@ -2204,7 +2204,7 @@ def run(self): self.ssh_obj.exec_command( self.mgmt_nodes[0], f"echo '{{not valid json}}' > {bad_json}") - out, err = self._sbcli(f"backup import {bad_json}") + out, err = self._sbcli(f"backup import --from-file {bad_json}") assert err or "error" in out.lower(), \ "TC-BCK-035: expected error for malformed JSON import" self.logger.info("TC-BCK-035: got expected error ✓") @@ -2214,7 +2214,7 @@ def run(self): self.ssh_obj.exec_command( self.mgmt_nodes[0], f"echo '[]' > {good_json}") - out, err = self._sbcli(f"backup import {good_json}") + out, err = self._sbcli(f"backup import --from-file {good_json}") # Empty list → 0 imported; should not error assert "error" not in out.lower() or "0" in out, \ f"TC-BCK-036: unexpected error for empty-list import: {err}" @@ -3820,7 +3820,7 @@ def _run_cli_cross_cluster_restore(self, backup_id: str, # TC-BCK-073: import metadata on Cluster-2 self.logger.info(f"TC-BCK-073: Cluster-2 — backup import {meta_file}") out, err = self._sbcli_c2( - f"backup import {meta_file} --cluster-id {self._cluster2_id}") + f"backup import --from-file {meta_file} --cluster-id {self._cluster2_id}") assert not (err and "error" in err.lower()), \ f"TC-BCK-073: backup import on Cluster-2 failed: {err}" self.logger.info(f"TC-BCK-073: import result: {out.strip()}") @@ -3867,8 +3867,7 @@ def _run_cli_cross_cluster_restore(self, backup_id: str, f"backup restore {backup_id} " f"--lvol {restored_name} " f"--pool {self._cluster2_pool_name} " - f"--node {c2_node_id} " - f"--cluster-id {self._cluster2_id}") + f"--node {c2_node_id}") assert not (err3 and "error" in err3.lower()), \ f"TC-BCK-075: restore on Cluster-2 failed: {err3}" self.logger.info( diff --git a/e2e/stress_test/mass_create_delete_stress.py b/e2e/stress_test/mass_create_delete_stress.py index 005e3af75..7bd0cd101 100644 --- a/e2e/stress_test/mass_create_delete_stress.py +++ b/e2e/stress_test/mass_create_delete_stress.py @@ -1055,6 +1055,7 @@ def _phase_rapid_restart_cycles(self, label: str, ) results = [] + failures = 0 for i in range(1, iterations + 1): self.logger.info( f"[{label}] --- Iteration {i}/{iterations} ---" @@ -1075,14 +1076,17 @@ def _phase_rapid_restart_cycles(self, label: str, ) # Wait for node to go offline then back online + offline_ok = True try: self.sbcli_utils.wait_for_storage_node_status( node_uuid, ["offline", "unreachable"], timeout=600, ) except Exception as exc: + offline_ok = False + failures += 1 self.logger.warning( f"[{label}][{i}] Timed out waiting for " - f"offline/unreachable: {exc}" + f"offline/unreachable ({failures}/{i} failed): {exc}" ) self.sbcli_utils.wait_for_storage_node_status( @@ -1102,6 +1106,7 @@ def _phase_rapid_restart_cycles(self, label: str, "online_time_iso": online_ts.isoformat(), "stop_to_online_sec": stop_to_online, "cooldown_sec": cooldown, + "offline_detected": offline_ok, }) # Cooldown — no waiting for migration, just a short pause @@ -1109,8 +1114,23 @@ def _phase_rapid_restart_cycles(self, label: str, total_dur = sum(r["stop_to_online_sec"] for r in results) self._phase_durations[label] = round(total_dur, 1) + + # Require >70% of iterations to successfully detect node offline + successful = sum( + 1 for r in results if r.get("offline_detected", True) + ) + min_required = math.ceil(iterations * 0.7) + if successful < min_required: + raise RuntimeError( + f"[{label}] Only {successful}/{iterations} iterations " + f"detected node going offline (required {min_required}). " + f"{failures} iterations timed out — " + f"kill/restart mechanism is broken" + ) + self.logger.info( - f"[{label}] Completed {iterations} restart cycles, " + f"[{label}] Completed {iterations} restart cycles " + f"({successful}/{iterations} detected offline), " f"cumulative stop-to-online: {total_dur}s" ) return results @@ -3003,7 +3023,7 @@ def _fire_delete_lvol(self, lvol_name: str): def _wait_lvols_deleted( self, names: list, label: str, timeout: int = 1800, - stall_timeout: int = 1800, + stall_timeout: int = 600, ): """Wait for lvols/clones to disappear from the API. @@ -3158,7 +3178,7 @@ def _fire_delete_snapshot(self, snap_name: str): # Maximum wall-clock time for the entire cleanup phase. # Prevents stuck in_deletion lvols from blocking the test indefinitely. - CLEANUP_TIMEOUT = 1800 # 30 minutes + CLEANUP_TIMEOUT = 600 # 10 minutes — cleanup should not delay failure reporting def _phase_cleanup(self): timeout = self.CLEANUP_TIMEOUT @@ -3166,10 +3186,20 @@ def _phase_cleanup(self): def _run_cleanup(): deadline = time.time() + timeout + # Per-step stall timeout: give up quickly if cluster is + # unresponsive (e.g. SUSPENDED / in_activation). + stall = min(120, timeout // 4) + steps = [ - ("delete_all_clones", self.sbcli_utils.delete_all_clones), + ("delete_all_clones", lambda: self.sbcli_utils.delete_all_clones( + timeout=max(60, int(deadline - time.time())), + stall_timeout=stall, + )), ("delete_all_snapshots", self.sbcli_utils.delete_all_snapshots), - ("delete_all_lvols", self.sbcli_utils.delete_all_lvols), + ("delete_all_lvols", lambda: self.sbcli_utils.delete_all_lvols( + timeout=max(60, int(deadline - time.time())), + stall_timeout=stall, + )), ("delete_all_storage_pools", self.sbcli_utils.delete_all_storage_pools), ] for label, fn in steps: @@ -5145,8 +5175,8 @@ class MassCreateRapidRestart_6k_3Snap_Docker(_MassCreateDeleteDocker): 30 rapid container stop/restart cycles per phase.""" PERSISTENT_RETRY = True MAX_ENTITY_COUNT = 12000 - NUM_SUBSYSTEMS = 40 - NS_PER_SUBSYSTEM = 75 + NUM_SUBSYSTEMS = 60 + NS_PER_SUBSYSTEM = 50 SNAPSHOTS_PER_LVOL = 3 RAPID_RESTART_ITERATIONS = 30 RAPID_RESTART_COOLDOWN = 60 @@ -5171,8 +5201,8 @@ class MassCreateRapidRestart_6k_3Snap_K8s(_MassCreateDeleteK8s): 30 rapid pod delete/restart cycles per phase.""" PERSISTENT_RETRY = True MAX_ENTITY_COUNT = 12000 - NUM_SUBSYSTEMS = 40 - NS_PER_SUBSYSTEM = 75 + NUM_SUBSYSTEMS = 60 + NS_PER_SUBSYSTEM = 50 SNAPSHOTS_PER_LVOL = 3 RAPID_RESTART_ITERATIONS = 30 RAPID_RESTART_COOLDOWN = 60 diff --git a/e2e/utils/ssh_utils.py b/e2e/utils/ssh_utils.py index fed82725f..ae5cdde4f 100644 --- a/e2e/utils/ssh_utils.py +++ b/e2e/utils/ssh_utils.py @@ -1917,7 +1917,7 @@ def deploy_storage_node(self, node, max_lvol, max_prov_gb, ifname="eth0", branch time.sleep(10) - configure_cmd = f"{self.base_cmd} -d sn configure --max-subsys {max_lvol} --nodes-per-socket {nodes_per_socket}" + configure_cmd = f"{self.base_cmd} -d sn configure --nodes-per-socket {nodes_per_socket}" deploy_cmd = f"{self.base_cmd} sn deploy --ifname {ifname}" self.logger.info(f"Deploying storage node: {node}") diff --git a/e2e/utils/test_graylog_export.py b/e2e/utils/test_graylog_export.py index a92fb1c2e..c5ce3f6f9 100755 --- a/e2e/utils/test_graylog_export.py +++ b/e2e/utils/test_graylog_export.py @@ -13,6 +13,12 @@ Fetch strategy: - Graylog REST API when reachable; OpenSearch scroll API otherwise. +Chunking: + - The total time window is split into 1-hour chunks. + - Chunks are processed in REVERSE order (newest first) so you can + verify collection is working while it runs. + - Each chunk writes to a subfolder: OUTPUT_DIR/chunk_NN_HH-MM_to_HH-MM/ + Environment variables: MGMT_IP Management node IP (required unless both dedicated IPs are set) CLUSTER_SECRET Graylog admin password / cluster secret (required) @@ -41,6 +47,7 @@ import os import sys +import time from datetime import datetime, timezone, timedelta try: @@ -105,6 +112,9 @@ PAGE_SIZE = 1000 MAX_RESULT_WINDOW = 100_000 +# Chunk size in minutes for splitting the time window +CHUNK_MINUTES = 60 + # --------------------------------------------------------------------------- # HTTP sessions # --------------------------------------------------------------------------- @@ -288,11 +298,35 @@ def os_discover_containers(): return [] -def os_fetch_container_logs(container_name, source, out_path, probe_cache=None): - """Fetch logs from OpenSearch using the scroll API. Returns line count.""" - from_ms = FROM_MS - to_ms = TO_MS +MAX_RETRIES = 3 +RETRY_BACKOFF = [10, 30, 60] # seconds between retries on 429 + +def _os_request_with_retry(session, method, url, retries=MAX_RETRIES, **kwargs): + """Make an HTTP request with retry on 429 (circuit breaker) errors.""" + for attempt in range(retries + 1): + try: + r = session.request(method, url, **kwargs) + if r.status_code == 429 and attempt < retries: + wait = RETRY_BACKOFF[min(attempt, len(RETRY_BACKOFF) - 1)] + print(f" 429 circuit breaker hit, waiting {wait}s " + f"(attempt {attempt + 1}/{retries}) ...", file=sys.stderr) + time.sleep(wait) + continue + return r + except requests.RequestException: + if attempt < retries: + wait = RETRY_BACKOFF[min(attempt, len(RETRY_BACKOFF) - 1)] + time.sleep(wait) + continue + raise + return r # unreachable but keeps linters happy + + +def os_fetch_container_logs(container_name, source, out_path, + chunk_from_ms, chunk_to_ms, + probe_cache=None): + """Fetch logs from OpenSearch using the scroll API. Returns line count.""" if probe_cache is None: probe_cache = {} if "index" not in probe_cache: @@ -306,7 +340,7 @@ def os_fetch_container_logs(container_name, source, out_path, probe_cache=None): esc = container_name.replace("/", "\\/").replace(":", "\\:") must_clauses = [ - {"range": {ts_f: {"gte": from_ms, "lte": to_ms, + {"range": {ts_f: {"gte": chunk_from_ms, "lte": chunk_to_ms, "format": "epoch_millis"}}}, {"query_string": {"default_field": cname_f, "query": f"*{esc}*", @@ -338,7 +372,8 @@ def _fmt(src): written = 0 try: - r = os_session.post(init_url, json=body, timeout=60) + r = _os_request_with_retry( + os_session, "POST", init_url, json=body, timeout=60) if not r.ok: print(f" WARN: OpenSearch scroll failed for {container_name}: " f"HTTP {r.status_code} {r.text[:300]}", file=sys.stderr) @@ -369,12 +404,17 @@ def _fmt(src): if len(hits) < PAGE_SIZE or not scroll_id: break try: - sc_r = os_session.post( + sc_r = _os_request_with_retry( + os_session, "POST", f"{OPENSEARCH_BASE}/_search/scroll", json={"scroll": "2m", "scroll_id": scroll_id}, timeout=60, ) - sc_r.raise_for_status() + if not sc_r.ok: + print(f" WARN: scroll continuation failed for " + f"{container_name}: HTTP {sc_r.status_code}", + file=sys.stderr) + break sc_data = sc_r.json() scroll_id = sc_data.get("_scroll_id", scroll_id) hits = sc_data.get("hits", {}).get("hits", []) @@ -473,7 +513,8 @@ def gl_discover_containers(): return [] -def gl_fetch_container_logs(container_name, source, out_path): +def gl_fetch_container_logs(container_name, source, out_path, + chunk_from_iso, chunk_to_iso): """Fetch all logs for a container+source via Graylog. Returns line count.""" search_url = f"{GRAYLOG_BASE}/search/universal/absolute" # Use wildcard so partial names work (e.g. "spdk_8080" matches @@ -540,7 +581,7 @@ def _write_window(fh, q, f_iso, t_iso): return written # Probe total - msgs, total = _fetch_page(query, FROM_ISO, TO_ISO, 1, 0) + msgs, total = _fetch_page(query, chunk_from_iso, chunk_to_iso, 1, 0) if msgs is None: open(out_path, "w").close() return 0 @@ -548,11 +589,11 @@ def _write_window(fh, q, f_iso, t_iso): written = 0 with open(out_path, "w") as fh: if total <= MAX_RESULT_WINDOW: - written = _write_window(fh, query, FROM_ISO, TO_ISO) + written = _write_window(fh, query, chunk_from_iso, chunk_to_iso) else: # Split into 10-minute sub-windows - t = datetime.fromisoformat(FROM_ISO.replace("Z", "+00:00")) - t_end = datetime.fromisoformat(TO_ISO.replace("Z", "+00:00")) + t = datetime.fromisoformat(chunk_from_iso.replace("Z", "+00:00")) + t_end = datetime.fromisoformat(chunk_to_iso.replace("Z", "+00:00")) chunk = timedelta(minutes=10) while t < t_end: chunk_end = min(t + chunk, t_end) @@ -594,15 +635,94 @@ def discover_containers(graylog_ok): # Main # --------------------------------------------------------------------------- +def _safe(s): + return ( + s.replace("/", "_").replace("\\", "_") + .replace(":", "_").strip("_") + ) or "unnamed" + + +def _build_chunks(start, end, chunk_minutes=CHUNK_MINUTES): + """Build list of (chunk_start_dt, chunk_end_dt) in reverse order (newest first).""" + chunks = [] + t = start + while t < end: + c_end = min(t + timedelta(minutes=chunk_minutes), end) + chunks.append((t, c_end)) + t = c_end + chunks.reverse() # newest first + return chunks + + +def _fetch_chunk(pairs, chunk_start, chunk_end, chunk_dir, os_ok, + graylog_ok, os_probe_cache, chunk_label): + """Fetch all container logs for a single time chunk. + + Containers are fetched SEQUENTIALLY (one scroll context at a time) + to avoid OpenSearch circuit breaker / heap pressure. + On OpenSearch failure, falls back to Graylog for that container. + Returns total lines. + """ + c_from_ms = int(chunk_start.timestamp() * 1000) + c_to_ms = int(chunk_end.timestamp() * 1000) + c_from_iso = chunk_start.strftime("%Y-%m-%dT%H:%M:%S.000Z") + c_to_iso = chunk_end.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + os.makedirs(chunk_dir, exist_ok=True) + + total_lines = 0 + + for i, (container_name, source) in enumerate(sorted(pairs), 1): + safe_cname = _safe(container_name) + if source: + safe_source = _safe(source) + fname = f"{safe_cname}__{safe_source}.log" + else: + fname = f"{safe_cname}.log" + out_path = os.path.join(chunk_dir, fname) + label = f"{container_name}@{source}" if source else container_name + + n = 0 + try: + if os_ok: + n = os_fetch_container_logs( + container_name, source, out_path, + chunk_from_ms=c_from_ms, chunk_to_ms=c_to_ms, + probe_cache=os_probe_cache, + ) + # Fallback to Graylog if OpenSearch returned 0 lines + if n == 0 and graylog_ok: + n = gl_fetch_container_logs( + container_name, source, out_path, + chunk_from_iso=c_from_iso, chunk_to_iso=c_to_iso, + ) + else: + n = gl_fetch_container_logs( + container_name, source, out_path, + chunk_from_iso=c_from_iso, chunk_to_iso=c_to_iso, + ) + except Exception as exc: + print(f" [{i}/{len(pairs)}] {label:<50} FAILED: {exc}") + continue + + total_lines += n + if n > 0: + print(f" [{i}/{len(pairs)}] {label:<50} {n:>8,} lines") + + return total_lines + + def main(): print("=" * 64) - print(" Graylog / OpenSearch Export Test") + print(" Graylog / OpenSearch Export (Chunked, Reverse Order)") print("=" * 64) print(f" Window : {FROM_ISO} -> {TO_ISO} ({DURATION_MINUTES} min)") + print(f" Chunk size : {CHUNK_MINUTES} min") print(f" Mode : {DEPLOY_MODE}") print(f" Field : {CNAME_FIELD}") print(f" Graylog : {GRAYLOG_BASE}") print(f" OpenSearch : {OPENSEARCH_BASE}") + print(f" Output : {OUTPUT_DIR}") print() # Check Graylog @@ -638,29 +758,13 @@ def main(): print("\nNeither Graylog nor OpenSearch is reachable. Exiting.") sys.exit(1) - # Discover (container, source) pairs + # Discover (container, source) pairs across the full window pairs = discover_containers(graylog_ok) if not pairs: print("\nNo containers found. Check your time window and log setup.") sys.exit(1) - # Create output directory - os.makedirs(OUTPUT_DIR, exist_ok=True) - - # Fetch strategy: prefer OpenSearch (scroll API handles large windows), - # fall back to Graylog only when OpenSearch is unavailable. - fetch_via = "OpenSearch" if os_ok else "Graylog" - print(f"\n[3] Fetching logs for {len(pairs)} (container, source) pairs " - f"via {fetch_via} -> {OUTPUT_DIR}") - print("-" * 64) - - def _safe(s): - return ( - s.replace("/", "_").replace("\\", "_") - .replace(":", "_").strip("_") - ) or "unnamed" - - # Pre-populate probe cache before parallel fetch + # Pre-populate probe cache once (shared across all chunks) os_probe_cache = {} if os_ok: try: @@ -669,52 +773,44 @@ def _safe(s): except Exception as exc: print(f" WARN: Failed to pre-populate probe cache: {exc}") - from concurrent.futures import ThreadPoolExecutor, as_completed - import threading + # Build 1-hour chunks in reverse order (newest first) + chunks = _build_chunks(start_dt, end_dt, CHUNK_MINUTES) + num_chunks = len(chunks) - max_workers = min(8, len(pairs)) - total_lines = 0 - lock = threading.Lock() + fetch_via = "OpenSearch" if os_ok else "Graylog" + print(f"\n[3] Fetching logs in {num_chunks} chunk(s) of {CHUNK_MINUTES} min " + f"(reverse order, newest first)") + print(f" {len(pairs)} (container, source) pairs via {fetch_via}") + print("=" * 64) - def _fetch_one(container_name, source): - """Fetch a single container's logs. Returns (label, line_count).""" - safe_cname = _safe(container_name) - if source: - safe_source = _safe(source) - fname = f"{safe_cname}__{safe_source}.log" - else: - fname = f"{safe_cname}.log" - out_path = os.path.join(OUTPUT_DIR, fname) - label = f"{container_name}@{source}" if source else container_name + os.makedirs(OUTPUT_DIR, exist_ok=True) + grand_total = 0 + + for idx, (c_start, c_end) in enumerate(chunks, 1): + c_start_label = c_start.strftime("%Y%m%d_%H%M") + c_end_label = c_end.strftime("%H%M") + chunk_dir_name = f"chunk_{idx:02d}_of_{num_chunks:02d}_{c_start_label}_to_{c_end_label}" + chunk_dir = os.path.join(OUTPUT_DIR, chunk_dir_name) + + c_start_pretty = c_start.strftime("%Y-%m-%d %H:%M") + c_end_pretty = c_end.strftime("%H:%M") + print(f"\n--- Chunk {idx}/{num_chunks}: {c_start_pretty} -> {c_end_pretty} " + f"-> {chunk_dir_name}/ ---") + + chunk_lines = _fetch_chunk( + pairs, c_start, c_end, chunk_dir, os_ok, + graylog_ok, os_probe_cache, chunk_dir_name, + ) + grand_total += chunk_lines - if os_ok: - n = os_fetch_container_logs( - container_name, source, out_path, - probe_cache=os_probe_cache, - ) - else: - n = gl_fetch_container_logs(container_name, source, out_path) - return label, n + print(f" Chunk {idx}/{num_chunks} done: {chunk_lines:,} lines") - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit(_fetch_one, cname, src): (cname, src) - for cname, src in sorted(pairs) - } - for future in as_completed(futures): - cname, src = futures[future] - label = f"{cname}@{src}" if src else cname - try: - label, n = future.result() - with lock: - total_lines += n - print(f" {label:<60} {n:>8,} lines") - except Exception as exc: - print(f" {label:<60} FAILED: {exc}") - - print("-" * 64) - print(f" TOTAL: {total_lines:,} lines from {len(pairs)} (container, source) pairs") + print() + print("=" * 64) + print(f" TOTAL: {grand_total:,} lines from {len(pairs)} pairs " + f"across {num_chunks} chunk(s)") print(f" Output: {os.path.abspath(OUTPUT_DIR)}") + print("=" * 64) print()