From 970fd9e30e0e58fc83aef1e61bfd008da80fbc20 Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 4 Aug 2026 21:54:25 +0200 Subject: [PATCH 1/9] Add lblk cluster mode: Linux block devices as storage via SPDK AIO bdevs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New cluster-global device mode chosen at cluster create (--device-mode nvme|lblk, deploy-time only). In lblk mode, eligible Linux block devices (unmounted, unheld, unpartitioned whole disks; partitioned only with --force-format on add-node, which wipes them host-side) are wrapped in one SPDK AIO bdev per device. Everything from alceml upward is unchanged; nvme-tcp/rdma fabric is untouched. - Selection at `sn configure --lblk` (and the k8s node_configure twin) by device name include/exclude or serial number; the node config file carries an editable `lblk_devices` list ({name, serial, by_id, size, numa}) parallel to ssd_pcis, validated as exactly-one-device-source. - Identity is SERIAL-FIRST (lsblk SERIAL -> WWN -> stable synthetic id): add-node persists the selection on the node record; restart re-resolves serial -> current kernel name (stored name as fallback), so device renames across reboots cannot attach the wrong disk. AIO bdev names are derived from the serial (collision-safe) and stable across restarts. The serial-keyed restart reconcile works unchanged; missing device -> STATUS_REMOVED, new -> STATUS_NEW, same as nvme. - SPDK launch in lblk mode never passes an empty PCI allowlist (DPDK treats empty as allow-all; the k8s path passes PCI_ALLOWED="" today) — a host-bridge placeholder 0000:00:00.0 is used instead, and no vfio/uio binds ever happen, so SPDK cannot claim kernel disks. - Failure parity (control plane only): the distrib error_* event path is already bdev-generic; the hung-IO gap (AIO has no bdev_nvme timeout_us/ action_on_timeout) is closed by a device_monitor watchdog using queue-depth-sampled iostat — inflight IO with zero completion progress across 3 polls (30s) feeds io_error + UNAVAILABLE with a countable LOCAL_FAILURE cause into the existing flap/auto-restart/FAILED/ migration machinery; >=2 simultaneously stalled devices escalate to a node auto-restart; RPC failures freeze (never advance) the counters. Device disappearance from the host inventory drives device_remove (the SPDK_BDEV_EVENT_REMOVE treatment) after a 2-poll debounce. The late-event gate, reset (liveness probe — never delete/recreate the aio bdev in place), SMART info, restart_device and new_device_from_failed are mode-aware. - New snode endpoints: GET /blockdevices (whole-disk inventory with eligibility fields, by-id path, NUMA) and POST /wipe_block_device (re-validates busy state, wipefs partitions-then-disk), on both docker and k8s agents. - Phase-1 scope: lblk requires journal-on-device (GPT-partition JM mode is nvme-only); --ssd-pcie/--reattach-volume are rejected on lblk nodes. tests: 93 unit tests (eligibility/onboarding/watchdog/device_controller) + 10 FDB-backed integration tests (model round-trips, restart identity contract over renamed devices, watchdog -> real state machine, flap-limit force-FAILED + migration, disappearance -> device_remove, reset). Full unit tier 1005 green, ruff clean. External follow-ups: validate run_distr_with_ssd.sh tolerates the placeholder -A on a test node; confirm the fork's bdev_get_iostat carries qd-sampling fields; alceml over 512e aio bdevs (block_size currently omitted, 4096 fallback if needed). Co-Authored-By: Claude Fable 5 --- simplyblock_cli/cli-reference.yaml | 47 ++- simplyblock_cli/cli.py | 9 +- simplyblock_cli/clibase.py | 33 +- simplyblock_core/cluster_ops.py | 16 + simplyblock_core/constants.py | 25 ++ .../controllers/device_controller.py | 114 +++++- simplyblock_core/models/cluster.py | 7 + simplyblock_core/models/nvme_device.py | 13 + simplyblock_core/models/storage_node.py | 5 + simplyblock_core/rpc_client.py | 30 ++ simplyblock_core/services/device_monitor.py | 152 ++++++++ .../services/main_distr_event_collector.py | 11 +- simplyblock_core/snode_client.py | 10 + simplyblock_core/storage_node_ops.py | 207 ++++++++-- simplyblock_core/utils/__init__.py | 325 +++++++++++++++- .../api/internal/storage_node/docker.py | 30 ++ .../api/internal/storage_node/kubernetes.py | 30 ++ simplyblock_web/api/v2/_dtos.py | 6 + simplyblock_web/api/v2/cluster/__init__.py | 1 + .../api/v2/cluster/storage_node/__init__.py | 2 + simplyblock_web/node_configure.py | 56 ++- simplyblock_web/node_utils.py | 178 +++++++++ .../integration/test_lblk_device_lifecycle.py | 358 ++++++++++++++++++ tests/unit/test_api_dto_secrets.py | 1 + tests/unit/test_lblk_device_controller.py | 263 +++++++++++++ tests/unit/test_lblk_eligibility.py | 329 ++++++++++++++++ tests/unit/test_lblk_onboarding.py | 247 ++++++++++++ tests/unit/test_lblk_watchdog.py | 265 +++++++++++++ .../web/api/v2/test_storage_node_endpoints.py | 1 + 29 files changed, 2687 insertions(+), 84 deletions(-) create mode 100644 tests/integration/test_lblk_device_lifecycle.py create mode 100644 tests/unit/test_lblk_device_controller.py create mode 100644 tests/unit/test_lblk_eligibility.py create mode 100644 tests/unit/test_lblk_onboarding.py create mode 100644 tests/unit/test_lblk_watchdog.py diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 86b6366a70..ad3772f087 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -110,8 +110,31 @@ commands: required: false type: str default: "" + - name: "--lblk" + help: "Configure the node with Linux block devices (lblk cluster mode) instead of NVMe PCIe devices: eligible unmounted, unheld, unpartitioned whole disks are wrapped in SPDK AIO bdevs. Select devices with --blk-names, --blk-names-exclude or --blk-serials; without a selector, every eligible disk is used." + dest: lblk + type: bool + action: store_true + - name: "--blk-names" + help: "Comma separated list of block device names to use, like sdb,sdc (requires --lblk). Requested devices must be eligible; a busy device is an error." + dest: blk_names + required: false + type: str + default: "" + - name: "--blk-names-exclude" + help: "Comma separated list of block device names to exclude, like sda (requires --lblk). All other eligible disks are used." + dest: blk_names_exclude + required: false + type: str + default: "" + - name: "--blk-serials" + help: "Comma separated list of block device serial numbers (or WWNs) to use (requires --lblk)." + dest: blk_serials + required: false + type: str + default: "" - name: "--force" - help: "Force format detected or passed nvme pci address to 4K and clean partitions." + help: "Force format detected or passed nvme pci address to 4K and clean partitions. With --lblk: mark partitioned disks eligible; the partition wipe happens at add-node with --force-format." dest: force type: bool action: store_true @@ -186,6 +209,12 @@ commands: type: bool default: false action: store_true + - name: "--force-format" + help: "lblk cluster mode only: wipe partition tables and filesystem signatures from configured block devices that carry partitions (wipefs). Without this flag, partitioned devices are not eligible." + dest: force_format + type: bool + default: false + action: store_true - name: "--format-4k" help: "Force format nvme devices with 4K." dest: format_4k @@ -1024,6 +1053,14 @@ commands: dest: enable_failure_domain type: bool action: store_true + - name: "--device-mode" + help: "Storage-device mode for the whole cluster. 'nvme' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. 'lblk': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at 'sn configure' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected." + dest: device_mode + type: str + choices: + - nvme + - lblk + default: nvme - name: "--name" help: > Assigns a name to the newly created cluster. @@ -1206,6 +1243,14 @@ commands: dest: enable_failure_domain type: bool action: store_true + - name: "--device-mode" + help: "Storage-device mode for the whole cluster. 'nvme' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. 'lblk': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at 'sn configure' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected." + dest: device_mode + type: str + choices: + - nvme + - lblk + default: nvme - name: "--name" help: > Assigns a name to the newly created cluster. diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 16ab158860..66ae4efb3d 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -109,7 +109,11 @@ def init_storage_node__configure(self, subparser): subcommand.add_argument('--device-model', help='NVMe SSD model string, example: --model PM1628. Can be used alone to filter by model, or combined with --size-range to further filter by size.', type=str, default='', dest='device_model', required=False) subcommand.add_argument('--size-range', help='NVMe SSD device size range separated by -, can be X(m,g,t) or bytes as integer, example: --size-range 50G-1T or --size-range 1232345-67823987. Can be used alone to filter by size, or combined with --device-model to further filter by model.', type=str, default='', dest='size_range', required=False) subcommand.add_argument('--nvme-names', help='Comma separated list of nvme namespace names like nvme0n1,nvme1n1.', type=str, default='', dest='nvme_names', required=False) - subcommand.add_argument('--force', help='Force format detected or passed nvme pci address to 4K and clean partitions.', dest='force', action='store_true') + subcommand.add_argument('--lblk', help='Configure the node with Linux block devices (lblk cluster mode) instead of NVMe PCIe devices: eligible unmounted, unheld, unpartitioned whole disks are wrapped in SPDK AIO bdevs. Select devices with --blk-names, --blk-names-exclude or --blk-serials; without a selector, every eligible disk is used.', dest='lblk', action='store_true') + subcommand.add_argument('--blk-names', help='Comma separated list of block device names to use, like sdb,sdc (requires --lblk). Requested devices must be eligible; a busy device is an error.', type=str, default='', dest='blk_names', required=False) + subcommand.add_argument('--blk-names-exclude', help='Comma separated list of block device names to exclude, like sda (requires --lblk). All other eligible disks are used.', type=str, default='', dest='blk_names_exclude', required=False) + subcommand.add_argument('--blk-serials', help='Comma separated list of block device serial numbers (or WWNs) to use (requires --lblk).', type=str, default='', dest='blk_serials', required=False) + subcommand.add_argument('--force', help='Force format detected or passed nvme pci address to 4K and clean partitions. With --lblk: mark partitioned disks eligible; the partition wipe happens at add-node with --force-format.', dest='force', action='store_true') subcommand.add_argument('--calculate-hp-only', help='Calculate the minimum required huge pages, it depends on the following params: --cores-percentage, --sockets-to-use, --max-subsys, --nodes-per-socket, --number-of-devices.', dest='calculate_hp_only', action='store_true') subcommand.add_argument('--number-of-devices', help='Number of devices that will be used on this host. For calculating huge pages memory only.', type=int, dest='number_of_devices') @@ -131,6 +135,7 @@ def init_storage_node__add_node(self, subparser): subcommand.add_argument('ifname', help='The management interface name.', type=str) subcommand.add_argument('--journal-partition', help='**Deprecated since: 26.1** Replaced by: --enable-journal-device\n\n1: Auto-create small partitions for journal on nvme devices. 0: use a separate (the smallest) nvme device of the node for journal. The journal needs a maximum of 3 percent of total available raw disk space. Default: `1`.', type=int, dest='partitions', choices=[0,1,]) subcommand.add_argument('--enable-journal-device', help='Enables the use of a separate (the smallest) NVMe device of the node for the journal. Otherwise, the journal uses a maximum of 3%% of total available raw disk space across all NVMe devices.', default=False, dest='enable_journal_device', action='store_true') + subcommand.add_argument('--force-format', help='lblk cluster mode only: wipe partition tables and filesystem signatures from configured block devices that carry partitions (wipefs). Without this flag, partitioned devices are not eligible.', default=False, dest='force_format', action='store_true') subcommand.add_argument('--format-4k', help='Force format nvme devices with 4K.', dest='format_4k', action='store_true') if self.developer_mode: subcommand.add_argument('--jm-percent', help='Number in percent to use for JM from each device. Default: `3`.', type=int, default=3, dest='jm_percent') @@ -439,6 +444,7 @@ def init_cluster__create(self, subparser): subcommand.add_argument('--disable-monitoring', help='Disable monitoring stack, false by default. Default: `false`.', dest='disable_monitoring', action='store_true') subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster.', dest='strict_node_anti_affinity', action='store_true') subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true') + subcommand.add_argument('--device-mode', help='Storage-device mode for the whole cluster. \'nvme\' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. \'lblk\': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at \'sn configure\' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected.', type=str, default='nvme', dest='device_mode', choices=['nvme','lblk',]) subcommand.add_argument('--name', '-n', help='Assigns a name to the newly created cluster.', type=str, dest='name') subcommand.add_argument('--qpair-count', help='The NVMe/TCP transport qpair count per logical volume. Default: `32`.', type=range_type(0, 128), default=32, dest='qpair_count') subcommand.add_argument('--client-qpair-count', help='The default NVMe/TCP transport qpair count per logical volume for client. Default: `3`.', type=range_type(0, 128), default=3, dest='client_qpair_count') @@ -475,6 +481,7 @@ def init_cluster__add(self, subparser): subcommand.add_argument('--inflight-io-threshold', help='The number of inflight IOs allowed before the IO queuing starts. Default: `4`.', type=int, default=4, dest='inflight_io_threshold') subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster."', dest='strict_node_anti_affinity', action='store_true') subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true') + subcommand.add_argument('--device-mode', help='Storage-device mode for the whole cluster. \'nvme\' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. \'lblk\': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at \'sn configure\' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected.', type=str, default='nvme', dest='device_mode', choices=['nvme','lblk',]) subcommand.add_argument('--name', '-n', help='Assigns a name to the newly created cluster.', type=str, dest='name') subcommand.add_argument('--client-data-nic', help='Network interface name from client to use for logical volume connection.', type=str, dest='client_data_nic') subcommand.add_argument('--use-backup', help='The path to JSON file with S3/MinIO backup configuration.', type=str, dest='use_backup') diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 1cdd480d6d..bd1da74fb4 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -132,15 +132,34 @@ def storage_node__configure(self, sub_command, args): pci_blocked = [str(x) for x in args.pci_blocked.split(',')] if args.nvme_names: nvme_names = [str(x) for x in args.nvme_names.split(',')] + lblk = getattr(args, 'lblk', False) + blk_names = getattr(args, 'blk_names', None) + blk_names_exclude = getattr(args, 'blk_names_exclude', None) + blk_serials = getattr(args, 'blk_serials', None) use_pci_allowed = bool(args.pci_allowed) use_pci_blocked = bool(args.pci_blocked) use_model_range = bool(args.device_model or args.size_range) - if sum([use_pci_allowed, use_pci_blocked, use_model_range]) > 1: + use_lblk = bool(lblk or blk_names or blk_names_exclude or blk_serials) + if sum([use_pci_allowed, use_pci_blocked, use_model_range, use_lblk]) > 1: self.parser.error( - "Choose only one device selection method: --pci-allowed, --pci-blocked, or " + "Choose only one device selection method: --pci-allowed, --pci-blocked, " "--device-model/--size-range (--device-model and --size-range may be combined " - "with each other, but not with --pci-allowed or --pci-blocked)." + "with each other, but not with --pci-allowed or --pci-blocked), or --lblk with " + "its --blk-* selectors." ) + lblk_selection = None + if use_lblk: + if not lblk: + self.parser.error("--blk-names/--blk-names-exclude/--blk-serials require --lblk") + if sum([bool(blk_names), bool(blk_names_exclude), bool(blk_serials)]) > 1: + self.parser.error( + "Choose only one block-device selection method: --blk-names, " + "--blk-names-exclude, or --blk-serials.") + lblk_selection = { + "names": [str(x) for x in blk_names.split(',')] if blk_names else None, + "names_exclude": [str(x) for x in blk_names_exclude.split(',')] if blk_names_exclude else None, + "serials": [str(x) for x in blk_serials.split(',')] if blk_serials else None, + } cores_percentage = int(args.cores_percentage) if args.calculate_hp_only: if not args.number_of_devices: @@ -152,7 +171,8 @@ def storage_node__configure(self, sub_command, args): args.max_lvol, max_prov, sockets_to_use,args.nodes_per_socket, pci_allowed, pci_blocked, force=args.force, device_model=args.device_model, size_range=args.size_range, cores_percentage=cores_percentage, nvme_names=nvme_names, - calculate_hp_only=args.calculate_hp_only, number_of_devices=number_of_devices) + calculate_hp_only=args.calculate_hp_only, number_of_devices=number_of_devices, + lblk_selection=lblk_selection) def storage_node__deploy_cleaner(self, sub_command, args): storage_ops.deploy_cleaner() @@ -222,6 +242,7 @@ def storage_node__add_node(self, sub_command, args): spdk_sys_mem=spdk_sys_mem, expansion=expansion, failure_domain=failure_domain, + force_format=getattr(args, 'force_format', False), ) except Exception as e: print(e) @@ -1198,6 +1219,7 @@ def cluster_add(self, args): is_single_node = args.is_single_node client_data_nic = args.client_data_nic enable_failure_domain = getattr(args, 'enable_failure_domain', False) + device_mode = getattr(args, 'device_mode', 'nvme') max_fault_tolerance = min(distr_npcs, 2) if distr_npcs >= 1 else 1 @@ -1214,6 +1236,7 @@ def cluster_add(self, args): nvmf_base_port=args.nvmf_base_port, rpc_base_port=args.rpc_base_port, snode_api_port=args.snode_api_port, hashicorp_vault_settings=HashicorpVaultSettings({"base_url": args.hashicorp_vault_url}) if args.hashicorp_vault_url else None, enable_failure_domain=enable_failure_domain, + device_mode=device_mode, ) def cluster_create(self, args): @@ -1251,6 +1274,7 @@ def cluster_create(self, args): fabric = args.fabric client_data_nic = args.client_data_nic enable_failure_domain = getattr(args, 'enable_failure_domain', False) + device_mode = getattr(args, 'device_mode', 'nvme') # Private (developer-mode-only) arg: absent unless sbctl was run with --dev. enable_hang_device = getattr(args, "enable_hang_device", False) @@ -1273,6 +1297,7 @@ def cluster_create(self, args): nvmf_base_port=args.nvmf_base_port, rpc_base_port=args.rpc_base_port, snode_api_port=args.snode_api_port, hashicorp_vault_settings=HashicorpVaultSettings({"base_url": args.hashicorp_vault_url}) if args.hashicorp_vault_url else None, enable_failure_domain=enable_failure_domain, + device_mode=device_mode, enable_hang_device=enable_hang_device, ) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 7e3b91376d..aa53de7a9e 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -266,6 +266,16 @@ def parse_protocols(input_str: str): "rdma": "rdma" in parts, } +def _validated_device_mode(device_mode) -> str: + """Normalize/validate the cluster device mode ("nvme" | "lblk"). + Deploy-time only, like enable_failure_domain.""" + mode = (device_mode or constants.DEVICE_MODE_NVME).lower() + if mode not in (constants.DEVICE_MODE_NVME, constants.DEVICE_MODE_LBLK): + raise ValueError( + f"invalid device_mode {device_mode!r}; must be " + f"'{constants.DEVICE_MODE_NVME}' or '{constants.DEVICE_MODE_LBLK}'") + return mode + def create_cluster(blk_size, page_size_in_blocks, cli_pass, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, ifname, mgmt_ip, log_del_interval, metrics_retention_period, @@ -276,6 +286,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, nvmf_base_port=4420, rpc_base_port=8080, snode_api_port=50001, container_image_prefix=None, hashicorp_vault_settings : t.Optional[HashicorpVaultSettings] = None, enable_failure_domain=False, + device_mode=constants.DEVICE_MODE_NVME, enable_hang_device=False, ) -> str: if (distr_ndcs, distr_npcs) not in SUPPORTED_ERASURE_CODING_SCHEMES: @@ -409,6 +420,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, cluster.inflight_io_threshold = inflight_io_threshold cluster.strict_node_anti_affinity = strict_node_anti_affinity cluster.enable_failure_domain = enable_failure_domain + cluster.device_mode = _validated_device_mode(device_mode) cluster.contact_point = contact_point cluster.disable_monitoring = disable_monitoring cluster.mode = mode @@ -514,6 +526,7 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn nvmf_base_port=4420, rpc_base_port=8080, snode_api_port=50001, hashicorp_vault_settings : t.Optional[HashicorpVaultSettings] = None, enable_failure_domain=False, + device_mode=constants.DEVICE_MODE_NVME, ) -> str: """Thin wrapper around _add_cluster_impl() that serializes create calls for the same name behind a ClusterCreateLock. @@ -539,6 +552,7 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn client_data_nic=client_data_nic, max_fault_tolerance=max_fault_tolerance, backup_config=backup_config, nvmf_base_port=nvmf_base_port, rpc_base_port=rpc_base_port, snode_api_port=snode_api_port, hashicorp_vault_settings=hashicorp_vault_settings, enable_failure_domain=enable_failure_domain, + device_mode=device_mode, ) if not name: return _add_cluster_impl(**kwargs) @@ -562,6 +576,7 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca nvmf_base_port=4420, rpc_base_port=8080, snode_api_port=50001, hashicorp_vault_settings : t.Optional[HashicorpVaultSettings] = None, enable_failure_domain=False, + device_mode=constants.DEVICE_MODE_NVME, ) -> str: clusters = db_controller.get_clusters() @@ -600,6 +615,7 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca cluster.secret = SecretStr(utils.generate_string(20)) cluster.strict_node_anti_affinity = strict_node_anti_affinity cluster.enable_failure_domain = enable_failure_domain + cluster.device_mode = _validated_device_mode(device_mode) if clusters: cfg = db_controller.get_deploy_config() diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index bda9fba3ed..2c0ed037c1 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -70,6 +70,31 @@ def get_config_var(name, default=None): CACHED_LVOL_STAT_COLLECTOR_INTERVAL_SEC = 15 DEV_DISCOVERY_INTERVAL_SEC = 60 +# --- lblk cluster mode (Linux block devices via SPDK AIO bdevs) --- +DEVICE_MODE_NVME = "nvme" +DEVICE_MODE_LBLK = "lblk" +# DPDK PCI allowlist placeholder used when starting SPDK in lblk mode: an +# empty allowlist means "allow all" to DPDK (the k8s launch path passes an +# empty PCI_ALLOWED today), which would let SPDK's nvme driver claim +# kernel-owned NVMe disks. 0000:00:00.0 is the host bridge — syntactically a +# valid BDF, never a storage device, no DPDK driver binds it. +LBLK_PCI_ALLOWED_PLACEHOLDER = "0000:00:00.0" +# Queue-depth sampling period enabled on every AIO base bdev so +# bdev_get_iostat reports queue_depth (feeds the hung-IO watchdog). +AIO_QD_SAMPLING_PERIOD_US = 100000 # 100 ms +# Hung-IO watchdog: consecutive device_monitor polls (DEV_MONITOR_INTERVAL_SEC +# apart) with queue_depth > 0 and zero completion progress before the device +# is declared stalled (3 x 10s = 30s — deliberately above kernel SCSI/NVMe +# timeouts, which convert most stalls into EIO for us via the distrib +# error_* events; the watchdog only catches what the kernel never times out). +AIO_HUNG_IO_STALL_POLLS = 3 +# Consecutive polls a configured block device may be absent from the host's +# lsblk before it is treated as hot-removed (REMOVAL semantics). +AIO_DEVICE_ABSENT_POLLS = 2 +# Kernel block devices never eligible for lblk data placement. +LBLK_EXCLUDED_NAME_PREFIXES = ("ram", "loop", "sr", "fd", "zram", "nbd", + "md", "dm-", "drbd") + PMEM_DIR = '/tmp/pmem' NVME_PROGRAM_FAIL_COUNT = 50 diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index fc3871dbb6..3d218a7165 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -4,7 +4,7 @@ import logging import uuid -from simplyblock_core import distr_controller, utils, storage_node_ops +from simplyblock_core import constants, distr_controller, utils, storage_node_ops from simplyblock_core.controllers import device_events, tasks_controller from simplyblock_core.db_controller import DBController from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice @@ -563,7 +563,34 @@ def restart_device(device_id, force=False): except Exception as e: logger.error(f"Failed to log teardown-warning event for {device_id}: {e}") - if not snode.rpc_client().bdev_nvme_controller_list(device_obj.nvme_controller): + if device_obj.bdev_type == "aio": + # lblk mode: the base bdev is an AIO bdev over a kernel block device. + # Re-resolve serial-first (kernel names shift), recreate if gone. + if not snode.rpc_client().get_bdevs(device_obj.nvme_bdev): + try: + filename = device_obj.by_id_path or device_obj.device_path + try: + inventory, _ = snode.client(timeout=30, retry=1).get_blockdevices() + for blk in inventory or []: + if blk.get("serial") == device_obj.serial_number: + filename = blk.get("by_id_path") or blk.get("device_path") + device_obj.device_path = blk.get("device_path", device_obj.device_path) + device_obj.by_id_path = blk.get("by_id_path", device_obj.by_id_path) + break + except Exception as e: + logger.warning(f"blockdevices inventory failed, using stored path: {e}") + if not filename: + logger.error(f"No block device path known for {device_id}") + return False + snode.rpc_client().bdev_aio_create(device_obj.nvme_bdev, filename) + snode.rpc_client().bdev_examine(device_obj.nvme_bdev) + snode.rpc_client().bdev_wait_for_examine() + snode.rpc_client().bdev_set_qd_sampling_period( + device_obj.nvme_bdev, constants.AIO_QD_SAMPLING_PERIOD_US) + except Exception as e: + logger.error(e) + return False + elif not snode.rpc_client().bdev_nvme_controller_list(device_obj.nvme_controller): try: ret = snode.client(timeout=30, retry=1).bind_device_to_spdk(device_obj.pcie_address) logger.debug(ret) @@ -983,12 +1010,24 @@ def reset_storage_device(dev_id): logger.info("Resetting device") rpc_client = snode.rpc_client() - controller_name = device.nvme_controller - response = rpc_client.reset_device(controller_name) - if not response: - logger.error(f"Failed to reset NVMe BDev {controller_name}") - return False - time.sleep(3) + if device.bdev_type == "aio": + # No controller-reset primitive for AIO bdevs, and deleting/ + # recreating the bdev here would cascade a REMOVE through the + # alceml stack. Liveness-probe instead: bdev present => clear the + # error state below (device_set_online also forgives flaps); + # bdev gone => fail so the tasks framework escalates to + # restart_device, whose full stack rebuild is the real recovery. + if not rpc_client.get_bdevs(device.nvme_bdev): + logger.error(f"AIO bdev {device.nvme_bdev} is gone; reset cannot " + f"recover it — restart the device instead") + return False + else: + controller_name = device.nvme_controller + response = rpc_client.reset_device(controller_name) + if not response: + logger.error(f"Failed to reset NVMe BDev {controller_name}") + return False + time.sleep(3) # set io_error flag False device_set_io_error(dev_id, False) @@ -1333,18 +1372,47 @@ def new_device_from_failed(device_id): logger.error("Device is already added back from failed") return False - if not device_node.rpc_client().bdev_nvme_controller_list(device.nvme_controller): - try: - ret = device_node.client(timeout=30, retry=1).bind_device_to_spdk(device.pcie_address) - logger.debug(ret) - device_node.rpc_client().bdev_nvme_controller_attach(device.nvme_controller, device.pcie_address) - except Exception as e: - logger.error(e) + if device.bdev_type == "aio": + # lblk mode: ensure the AIO bdev exists again (serial-first + # re-resolution against the live host; stored path as fallback). + if not device_node.rpc_client().get_bdevs(device.nvme_bdev): + try: + filename = device.by_id_path or device.device_path + try: + inventory, _ = device_node.client(timeout=30, retry=1).get_blockdevices() + for blk in inventory or []: + if blk.get("serial") == device.serial_number: + filename = blk.get("by_id_path") or blk.get("device_path") + break + except Exception as e: + logger.warning(f"blockdevices inventory failed, using stored path: {e}") + if not filename: + logger.error(f"No block device path known for {device_id}") + return False + device_node.rpc_client().bdev_aio_create(device.nvme_bdev, filename) + device_node.rpc_client().bdev_examine(device.nvme_bdev) + device_node.rpc_client().bdev_wait_for_examine() + device_node.rpc_client().bdev_set_qd_sampling_period( + device.nvme_bdev, constants.AIO_QD_SAMPLING_PERIOD_US) + except Exception as e: + logger.error(e) + return False + if not device_node.rpc_client().get_bdevs(device.nvme_bdev): + logger.error(f"Failed to find AIO bdev {device.nvme_bdev}") return False + else: + if not device_node.rpc_client().bdev_nvme_controller_list(device.nvme_controller): + try: + ret = device_node.client(timeout=30, retry=1).bind_device_to_spdk(device.pcie_address) + logger.debug(ret) + device_node.rpc_client().bdev_nvme_controller_attach(device.nvme_controller, device.pcie_address) + except Exception as e: + logger.error(e) + return False - if not device_node.rpc_client().bdev_nvme_controller_list(device.nvme_controller): - logger.error(f"Failed to find device nvme controller {device.nvme_controller}") - return False + if not device_node.rpc_client().bdev_nvme_controller_list(device.nvme_controller): + logger.error(f"Failed to find device nvme controller {device.nvme_controller}") + return False new_device = NVMeDevice(device.to_dict()) new_device.uuid = str(uuid.uuid4()) @@ -1379,6 +1447,16 @@ def get_device_health_info(device_id): logger.error(e) return False + if device.bdev_type == "aio": + # SMART is not reachable through SPDK for AIO bdevs; host-side + # smartctl via the node agent is a possible follow-up. + return json.dumps({ + "bdev_type": "aio", + "device_path": device.device_path, + "smart": None, + "message": "SMART data is not available through SPDK for AIO devices", + }, indent=2) + rpc_client = snode.rpc_client() ret = rpc_client.bdev_nvme_get_controller_health_info(device.nvme_controller) return json.dumps(ret, indent=2) \ No newline at end of file diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index 82401eba93..4bb6143c65 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -184,6 +184,13 @@ def is_topology_owned(self) -> bool: # Deploy-time only — set at cluster create/add, never toggled at runtime; # an existing cluster must be redeployed to gain the feature. enable_failure_domain: bool = False + # Storage-device mode for the whole cluster. "nvme" (default): NVMe PCIe + # controllers auto-detected and attached through the SPDK nvme bdev. + # "lblk": arbitrary Linux block devices wrapped in SPDK AIO bdevs (one + # per device); everything from alceml upward is identical. Deploy-time + # only — set at cluster create/add, never toggled at runtime. Inter-node + # fabric (nvme-tcp/rdma) is unaffected by this mode. + device_mode: str = "nvme" snapshot_replication_target_cluster: str = "" snapshot_replication_target_pool: str = "" snapshot_replication_timeout: int = 60*10 diff --git a/simplyblock_core/models/nvme_device.py b/simplyblock_core/models/nvme_device.py index e36a80c700..fd6a8c0def 100644 --- a/simplyblock_core/models/nvme_device.py +++ b/simplyblock_core/models/nvme_device.py @@ -74,6 +74,19 @@ class NVMeDevice(BaseModel): # Passthrough bdev UUID for cross-node nvme bdev identification, # meaning that remote bdev to this bdev would share the same uuid. pt_bdev_uuid: str = "" + # Base-bdev type discriminator: "nvme" (SPDK nvme bdev over a PCIe + # controller) or "aio" (SPDK AIO bdev over a Linux block device, lblk + # cluster mode). For "aio" devices, pcie_address and nvme_controller stay + # empty and nvme_bdev holds the AIO bdev name; identity is serial_number + # (lsblk SERIAL/WWN or a synthetic stable id), with device_path / + # by_id_path re-resolved from the live host on every restart. + bdev_type: str = "nvme" + # Current kernel device path (e.g. /dev/sdb) — informational; re-learned + # each restart, never used as identity when a serial is available. + device_path: str = "" + # Stable /dev/disk/by-id/... symlink when the device has one; preferred + # as the AIO bdev filename so udev renames cannot bite mid-flight. + by_id_path: str = "" def __change_dev_connection_to(self, connecting_from_node): # Targeted single-record write. The previous implementation scanned diff --git a/simplyblock_core/models/storage_node.py b/simplyblock_core/models/storage_node.py index 52a68f808f..d48c3801ff 100644 --- a/simplyblock_core/models/storage_node.py +++ b/simplyblock_core/models/storage_node.py @@ -111,6 +111,11 @@ class StorageNode(BaseNodeObject): partitions_count: int = 0 # Unused poller_cpu_cores: List[int] = [] ssd_pcie: List = [] + # lblk cluster mode: the configured block-device selection for this node, + # entries {name, serial, by_id, size, numa}. Parallel to ssd_pcie (which + # stays empty in lblk mode). Persisted so restart re-resolves devices + # (serial-first) without depending on the host config file. + lblk_devices: List[dict] = [] pollers_mask: str = "" primary_ip: str = "" raid: str = "" diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 1bba9177e3..7ce0b493d2 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1261,6 +1261,36 @@ def bdev_examine(self, name): def bdev_wait_for_examine(self): return self._request("bdev_wait_for_examine") + def bdev_aio_create(self, name, filename, block_size=0): + """Create an SPDK AIO bdev over a Linux block device (lblk cluster + mode). ``filename`` is the device path — prefer the stable + /dev/disk/by-id symlink. ``block_size`` 0 lets SPDK use the device's + logical block size.""" + params = {"name": name, "filename": filename} + if block_size: + params["block_size"] = block_size + return self._request("bdev_aio_create", params) + + def bdev_aio_delete(self, name): + return self._request("bdev_aio_delete", {"name": name}) + + def bdev_aio_rescan(self, name): + """Re-read the backing device's size (device grow pickup).""" + return self._request("bdev_aio_rescan", {"name": name}) + + def bdev_set_qd_sampling_period(self, name, period_us): + """Enable queue-depth sampling on a bdev so bdev_get_iostat reports + queue_depth/io_time — the hung-IO watchdog's signal for AIO base + bdevs (period 0 disables).""" + params = {"name": name, "period": period_us} + return self._request("bdev_set_qd_sampling_period", params) + + def get_bdevs_2(self, name): + """(ret, err) probe variant of bdev_get_bdevs, mirroring + bdev_nvme_controller_list_2 — used where the caller must distinguish + 'bdev gone' from RPC failure without raising.""" + return self._request2("bdev_get_bdevs", {"name": name}) + def bdev_enable_histogram(self, name, enable=True, opc=None): # opc filters to a single I/O type (e.g. "read"/"write"); requires # SPDK >= 24.01. Toggling disable->enable clears the collected data, diff --git a/simplyblock_core/services/device_monitor.py b/simplyblock_core/services/device_monitor.py index e564cbf88c..0b5ed80c9d 100644 --- a/simplyblock_core/services/device_monitor.py +++ b/simplyblock_core/services/device_monitor.py @@ -3,6 +3,7 @@ from simplyblock_core import constants, db_controller, utils from simplyblock_core.controllers import tasks_controller, device_controller +from simplyblock_core.controllers.device_controller import CAUSE_LOCAL_FAILURE from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.nvme_device import NVMeDevice from simplyblock_core.models.storage_node import StorageNode @@ -15,6 +16,151 @@ db = db_controller.DBController() +# --- lblk (AIO) watchdog state ------------------------------------------- +# AIO bdevs have no bdev_nvme-style IO timeout (timeout_us + +# action_on_timeout=reset is what converts hung IO into failed IO — and +# thereby distrib error_* events — on the nvme path). For AIO devices the +# control plane compensates here: queue-depth sampling is enabled on every +# aio bdev at creation, so bdev_get_iostat reports queue_depth; a device +# with inflight IO and zero completion progress across +# AIO_HUNG_IO_STALL_POLLS consecutive sweeps is declared stalled and fed +# into the exact same machinery an erroring nvme device hits +# (io_error + UNAVAILABLE with a countable LOCAL_FAILURE cause → +# auto-restart budget → flap limit → FAILED → migration). +# +# _aio_progress: device_id -> (last_total_completed_ops, consecutive_stalls) +# _aio_absent: device_id -> consecutive polls missing from the host lsblk +_aio_progress: dict = {} +_aio_absent: dict = {} + + +def _aio_total_ops(stat: dict) -> int: + return (int(stat.get("num_read_ops") or 0) + + int(stat.get("num_write_ops") or 0) + + int(stat.get("num_unmap_ops") or 0)) + + +def _check_aio_hung_io(node, rpc_client) -> list: + """Return the node's ONLINE aio devices whose IO is stalled past the + threshold. An RPC failure or missing queue_depth counts as UNKNOWN — + the stall counter is frozen, not advanced: a wedged SPDK reactor slows + the RPC path itself, and mgmt-plane slowness must not be converted + into device failures (cf. constants NVME_TIMEOUT_US rationale).""" + stalled = [] + for dev in node.nvme_devices: + if dev.bdev_type != "aio" or dev.status != NVMeDevice.STATUS_ONLINE: + _aio_progress.pop(dev.get_id(), None) + continue + try: + ret = rpc_client.get_lvol_stats(dev.nvme_bdev) + except Exception as e: + logger.debug(f"iostat failed for {dev.nvme_bdev}: {e}") + continue # unknown — freeze + bdevs = (ret or {}).get("bdevs") or [] + if not bdevs: + continue # unknown — freeze + stat = bdevs[0] + queue_depth = stat.get("queue_depth") + if queue_depth is None: + # qd-sampling not active (fork without the fields, or sampling + # lost across an SPDK restart) — re-arm it and skip this poll. + try: + rpc_client.bdev_set_qd_sampling_period( + dev.nvme_bdev, constants.AIO_QD_SAMPLING_PERIOD_US) + except Exception: + pass + continue + total = _aio_total_ops(stat) + last_total, stalls = _aio_progress.get(dev.get_id(), (None, 0)) + # A stall tick requires inflight IO on THIS poll and zero completion + # progress since the previous one; any progress resets the window. + if last_total is not None and total == last_total and queue_depth > 0: + stalls += 1 + else: + stalls = 0 + _aio_progress[dev.get_id()] = (total, stalls) + if stalls >= constants.AIO_HUNG_IO_STALL_POLLS: + stalled.append(dev) + return stalled + + +def _check_aio_device_presence(node) -> list: + """Return the node's aio devices whose backing block device has been + absent from the host inventory for AIO_DEVICE_ABSENT_POLLS consecutive + sweeps (hot-removal). Inventory failure = unknown — counters freeze.""" + aio_devs = [dev for dev in node.nvme_devices + if dev.bdev_type == "aio" + and dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE, + NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE]] + if not aio_devs: + return [] + try: + inventory, _ = node.client(timeout=10, retry=1).get_blockdevices() + except Exception as e: + logger.debug(f"blockdevices inventory failed for node {node.get_id()}: {e}") + return [] + if not inventory: + return [] + serials = {d.get("serial") for d in inventory} + names = {d.get("name") for d in inventory} + gone = [] + for dev in aio_devs: + if dev.serial_number in serials or dev.device_name in names: + _aio_absent.pop(dev.get_id(), None) + continue + absent = _aio_absent.get(dev.get_id(), 0) + 1 + _aio_absent[dev.get_id()] = absent + if absent >= constants.AIO_DEVICE_ABSENT_POLLS: + gone.append(dev) + return gone + + +def _sweep_aio_devices(node) -> None: + """lblk failure parity: hot-removal → device_remove (the treatment + SPDK_BDEV_EVENT_REMOVE gets), hung IO → io_error + UNAVAILABLE with a + countable cause. Node-level pattern (>=2 devices stalled at once — + reactor stall, controller, expander) escalates to a node auto-restart + instead of failing devices one by one, mirroring the >=2 rule of the + io_error auto-restart path below.""" + gone = _check_aio_device_presence(node) + for dev in gone: + logger.warning(f"AIO device {dev.get_id()} ({dev.device_name}, serial " + f"{dev.serial_number}) disappeared from host inventory; removing") + _aio_absent.pop(dev.get_id(), None) + _aio_progress.pop(dev.get_id(), None) + try: + device_controller.device_remove(dev.get_id(), cause=CAUSE_LOCAL_FAILURE) + except Exception as e: + logger.error(f"device_remove failed for {dev.get_id()}: {e}") + + try: + rpc_client = node.rpc_client() + stalled = _check_aio_hung_io(node, rpc_client) + except Exception as e: + logger.debug(f"hung-IO sweep failed for node {node.get_id()}: {e}") + return + if not stalled: + return + for dev in stalled: + _aio_progress.pop(dev.get_id(), None) + if len(stalled) >= 2: + logger.warning(f"{len(stalled)} AIO devices stalled simultaneously on " + f"node {node.get_id()}; treating as node-level and " + f"queueing node auto-restart") + tasks_controller.add_node_to_auto_restart(node) + return + dev = stalled[0] + logger.warning(f"AIO device {dev.get_id()} ({dev.nvme_bdev}) has inflight IO " + f"with no completion progress for " + f"{constants.AIO_HUNG_IO_STALL_POLLS * constants.DEV_MONITOR_INTERVAL_SEC}s; " + f"marking unavailable") + try: + device_controller.device_set_io_error(dev.get_id(), True) + device_controller.device_set_unavailable(dev.get_id(), cause=CAUSE_LOCAL_FAILURE) + except Exception as e: + logger.error(f"failed to mark stalled device {dev.get_id()}: {e}") + + def main(): logger.info("Starting Device monitor...") while True: @@ -35,6 +181,12 @@ def main(): if node.status != StorageNode.STATUS_ONLINE: logger.warning(f"Node status is not online, id: {node.get_id()}, status: {node.status}") continue + + if cluster.device_mode == constants.DEVICE_MODE_LBLK: + _sweep_aio_devices(node) + # Re-read: the sweep may have changed device statuses. + node = db.get_storage_node_by_id(node.get_id()) + for dev in node.nvme_devices: if dev.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE, NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE]: diff --git a/simplyblock_core/services/main_distr_event_collector.py b/simplyblock_core/services/main_distr_event_collector.py index 2f2acf31fd..a8eb1851fd 100644 --- a/simplyblock_core/services/main_distr_event_collector.py +++ b/simplyblock_core/services/main_distr_event_collector.py @@ -236,8 +236,15 @@ def process_device_event(event, logger): logger.info(f"event was fired {time_delta.total_seconds()} seconds ago, target remote controller ok, skipping") event.status = f'skipping_late_by_{int(time_delta.total_seconds())}s_but_controller_ok' return - ret, err = event_node_obj.rpc_client().bdev_nvme_controller_list_2(device_obj.nvme_controller) - if err and err['code'] == 22: + if device_obj.bdev_type == "aio": + # AIO devices have no nvme controller — probe the base + # bdev instead: bdev gone => the late event is real. + ret, err = event_node_obj.rpc_client().get_bdevs_2(device_obj.nvme_bdev) + controller_missing = bool(err) or not ret + else: + ret, err = event_node_obj.rpc_client().bdev_nvme_controller_list_2(device_obj.nvme_controller) + controller_missing = bool(err) and err['code'] == 22 + if controller_missing: logger.info(f"event was fired {time_delta.total_seconds()} seconds ago, checking controller filed") event.status = f'late_by_{int(time_delta.total_seconds())}s' else: diff --git a/simplyblock_core/snode_client.py b/simplyblock_core/snode_client.py index 0872b0a6bc..247e0e6e61 100644 --- a/simplyblock_core/snode_client.py +++ b/simplyblock_core/snode_client.py @@ -216,6 +216,16 @@ def bind_device_to_spdk(self, device_pci): params = {"device_pci": device_pci} return self._request("POST", "bind_device_to_spdk", params) + def get_blockdevices(self): + """Whole-disk inventory for the lblk cluster mode.""" + return self._request("GET", "blockdevices") + + def wipe_block_device(self, device_name): + """--force-format for lblk add-node: wipe partition/FS signatures + from a whole disk (refused when busy).""" + return self._request("POST", "wipe_block_device", + {"device_name": device_name}) + def spdk_process_is_up(self, rpc_port, cluster_id): params = {"rpc_port": rpc_port, "cluster_id": cluster_id} return self._request("GET", "spdk_process_is_up", params) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 6c9657b520..5cb74ef8cd 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -2372,7 +2372,8 @@ def _cluster_add_lock_heartbeat(db_controller, cluster_id, owner, stop_event): return -def _classify_existing_endpoint_record(db_controller, cluster_id, node_addr, ssd_pcie): +def _classify_existing_endpoint_record(db_controller, cluster_id, node_addr, ssd_pcie, + lblk_serials=None): """Classify a pre-existing storage-node record for ``node_addr`` that owns one of the joining node's SSDs, before an add-node proceeds. @@ -2381,6 +2382,9 @@ def _classify_existing_endpoint_record(db_controller, cluster_id, node_addr, ssd their channels were reset mid-command); the caller's retry then finds the record already present and must not fail permanently on it. + Device ownership is tested by PCIe overlap (nvme mode) or by configured + block-device serial overlap (lblk mode, ``lblk_serials``). + Returns one of: - (None, None): no record for this endpoint owns any of these SSDs. - ("already_added", node): record is ONLINE — the earlier add completed; @@ -2400,10 +2404,18 @@ def _classify_existing_endpoint_record(db_controller, cluster_id, node_addr, ssd first: a cleanup is always needed when one exists, independent of whether another match is already online. """ + lblk_serials = set(lblk_serials or []) + + def _owns_devices(node): + pcie_overlap = any(ssd in node.ssd_pcie for ssd in ssd_pcie or []) + serial_overlap = bool(lblk_serials and lblk_serials.intersection( + e.get("serial") for e in (node.lblk_devices or []))) + return pcie_overlap or serial_overlap + matches = [ node for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id) - if node.api_endpoint == node_addr and any(ssd in node.ssd_pcie for ssd in ssd_pcie) + if node.api_endpoint == node_addr and _owns_devices(node) ] for node in matches: if node.status == StorageNode.STATUS_IN_CREATION: @@ -2422,7 +2434,8 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, num_partitions_per_dev=0, jm_percent=0, enable_test_device=False, namespace=None, enable_ha_jm=False, cr_name=None, cr_namespace=None, cr_plural=None, id_device_by_nqn=False, partition_size="", ha_jm_count=None, format_4k=False, - spdk_proxy_image=None, spdk_sys_mem=None, expansion=False, failure_domain=None): + spdk_proxy_image=None, spdk_sys_mem=None, expansion=False, failure_domain=None, + force_format=False): snode_api = SNodeClient(node_addr) node_info, _ = snode_api.info() if node_info.get("nodes_config") and node_info["nodes_config"].get("nodes"): @@ -2581,10 +2594,37 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, f"using {constants.MAX_SUBSYSTEMS_PER_NODE}") max_lvol = constants.MAX_SUBSYSTEMS_PER_NODE ssd_pcie = node_config.get("ssd_pcis") + lblk_configured = node_config.get("lblk_devices") or [] - if ssd_pcie: + lblk_mode = cluster.device_mode == constants.DEVICE_MODE_LBLK + if lblk_mode: + if not lblk_configured: + logger.error( + "This cluster runs in lblk device mode but the node config " + "carries no 'lblk_devices'; run 'sn configure --lblk ...' first.") + return False + if ssd_pcie: + logger.error("lblk device mode: the node config must not carry " + "'ssd_pcis' entries") + return False + # Phase 1: lblk requires journal-on-device (the GPT-partition JM + # mode detaches/re-attaches NVMe controllers to re-examine). + if num_partitions_per_dev != 0 and jm_percent != 0: + logger.error("lblk device mode requires --enable-journal-device " + "(journal on a dedicated device); partitioned " + "journal mode is not supported") + return False + elif lblk_configured and not ssd_pcie: + logger.error( + "The node config carries 'lblk_devices' but this cluster runs " + f"in {cluster.device_mode} device mode; re-run 'sn configure' " + "without --lblk or create the cluster with --device-mode lblk.") + return False + + if ssd_pcie or lblk_configured: action, existing = _classify_existing_endpoint_record( - db_controller, cluster_id, node_addr, ssd_pcie) + db_controller, cluster_id, node_addr, ssd_pcie, + lblk_serials=[e.get("serial") for e in lblk_configured]) if action == "cleanup": # Repeated partial attempts can leave several stale records # for the same endpoint; we clean one per task retry. @@ -2786,11 +2826,41 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, results = None l_cores = node_config.get("l-cores") spdk_cpu_mask = node_config.get("cpu_mask") - for ssd in ssd_pcie: - if format_4k: - snode_api.format_device_with_4k(ssd) + lblk_resolved = [] + if lblk_mode: + # No driver rebind in lblk mode — the AIO bdev needs the device + # on its kernel driver. Resolve the configured selection against + # the live host (serial-first), then --force-format wipes + # partitioned disks (re-validated host-side: busy => refused). + blk_inventory, err = snode_api.get_blockdevices() + if not blk_inventory: + logger.error(f"Failed to list block devices on {node_addr}: {err}") + return False + lblk_resolved, missing = utils.resolve_lblk_entries( + lblk_configured, blk_inventory) + if missing: + logger.error( + f"Configured block device(s) not found on host: " + f"{[(e.get('name'), e.get('serial')) for e in missing]}") + return False + for blk_info in lblk_resolved: + if blk_info.get("has_partitions"): + if not force_format: + logger.error( + f"Block device {blk_info['name']} is partitioned; " + f"pass --force-format to wipe it, or exclude it") + return False + ret, err = snode_api.wipe_block_device(blk_info["name"]) + if not ret: + logger.error(f"Failed to wipe block device " + f"{blk_info['name']}: {err}") + return False + else: + for ssd in ssd_pcie: + if format_4k: + snode_api.format_device_with_4k(ssd) + snode_api.bind_device_to_spdk(ssd) snode_api.bind_device_to_spdk(ssd) - snode_api.bind_device_to_spdk(ssd) if not spdk_proxy_image: spdk_proxy_image = cluster.container_image_prefix + constants.SIMPLY_BLOCK_DOCKER_IMAGE @@ -2806,7 +2876,14 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, namespace, mgmt_ip, rpc_port, rpc_user, rpc_pass, multi_threading_enabled=constants.SPDK_PROXY_MULTI_THREADING_ENABLED, timeout=constants.SPDK_PROXY_TIMEOUT, - ssd_pcie=ssd_pcie, total_mem=total_mem, system_mem=minimum_sys_memory, cluster_mode=cluster.mode, + # lblk mode: never pass an empty PCI allowlist — DPDK treats + # it as allow-all and SPDK's nvme driver could claim kernel + # NVMe disks (the k8s path passes PCI_ALLOWED="" today). The + # host-bridge placeholder is a valid BDF that never matches a + # storage device. + ssd_pcie=(ssd_pcie if not lblk_mode + else [constants.LBLK_PCI_ALLOWED_PLACEHOLDER]), + total_mem=total_mem, system_mem=minimum_sys_memory, cluster_mode=cluster.mode, socket=node_socket, cluster_id=cluster_id, spdk_proxy_image=spdk_proxy_image, mcp_max_unavailable=mcp_max_unavailable) time.sleep(5) @@ -2917,7 +2994,10 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, snode.cr_name = cr_name snode.cr_namespace = cr_namespace snode.cr_plural = cr_plural - snode.ssd_pcie = ssd_pcie + snode.ssd_pcie = ssd_pcie if not lblk_mode else [] + # lblk mode: persist the configured selection (with serial identity) + # so restart can re-resolve devices without the host config file. + snode.lblk_devices = lblk_configured if lblk_mode else [] snode.hostname = hostname snode.host_nqn = subsystem_nqn snode.subsystem = subsystem_nqn @@ -3153,13 +3233,16 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, # snode.ssd_pcie = node_info['spdk_pcie_list'] # snode.write_to_db() # discover devices - if not snode.ssd_pcie: - node_info, _ = snode_api.info() - ssds = node_info['spdk_pcie_list'] + if lblk_mode: + nvme_devs = utils.addAioDevices(rpc_client, snode, lblk_resolved) else: - ssds = snode.ssd_pcie + if not snode.ssd_pcie: + node_info, _ = snode_api.info() + ssds = node_info['spdk_pcie_list'] + else: + ssds = snode.ssd_pcie - nvme_devs = addNvmeDevices(rpc_client, snode, ssds) + nvme_devs = addNvmeDevices(rpc_client, snode, ssds) if nvme_devs: for nvme in nvme_devs: @@ -3915,6 +3998,8 @@ def _finalize_node_removal(removed_node: StorageNode): snode_api.leave_swarm() pci_address = [] for dev in removed_node.nvme_devices: + if dev.bdev_type == "aio": + continue # no PCIe identity; lblk devices are wiped via wipe_block_device if dev.pcie_address not in pci_address: ret = snode_api.delete_dev_gpt_partitions(dev.pcie_address) logger.debug(ret) @@ -4351,6 +4436,11 @@ def _restart_storage_node_impl( snode.hostname = node_info['hostname'] if snode.num_partitions_per_dev == 0 and reattach_volume: + if snode.lblk_devices: + # EBS re-homing + PCI rebinding is nvme-mode machinery; + # lblk devices re-resolve by serial at discovery below. + logger.error("--reattach-volume is not supported on lblk-mode nodes") + return False new_cloud_instance_id = node_info['cloud_instance']['id'] detached_volumes = node_utils.detach_ebs_volumes(snode.cloud_instance_id) if not detached_volumes: @@ -4498,9 +4588,16 @@ def _restart_storage_node_impl( if not snode.spdk_proxy_image: snode.spdk_proxy_image = cluster.container_image_prefix + constants.SIMPLY_BLOCK_DOCKER_IMAGE + lblk_mode = cluster.device_mode == constants.DEVICE_MODE_LBLK + results = None try: if new_ssd_pcie and type(new_ssd_pcie) is list: + if lblk_mode: + # Phase 1: growing an lblk node's device set goes through + # `sn configure --lblk` + re-add, not restart-time PCI binds. + logger.error("--ssd-pcie is not supported on lblk-mode clusters") + return False for new_ssd in new_ssd_pcie: if new_ssd not in snode.ssd_pcie: try: @@ -4517,7 +4614,11 @@ def _restart_storage_node_impl( snode.l_cores, snode.spdk_mem, snode.spdk_image, spdk_debug, cluster_ip, fdb_connection, snode.namespace, snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, multi_threading_enabled=constants.SPDK_PROXY_MULTI_THREADING_ENABLED, timeout=constants.SPDK_PROXY_TIMEOUT, - ssd_pcie=snode.ssd_pcie, total_mem=total_mem, system_mem=minimum_sys_memory, cluster_mode=cluster.mode, + # lblk: placeholder allowlist — an empty list means allow-all to + # DPDK and SPDK's nvme driver could claim kernel NVMe disks. + ssd_pcie=(snode.ssd_pcie if not lblk_mode + else [constants.LBLK_PCI_ALLOWED_PLACEHOLDER]), + total_mem=total_mem, system_mem=minimum_sys_memory, cluster_mode=cluster.mode, socket=snode.socket, cluster_id=snode.cluster_id, spdk_proxy_image=snode.spdk_proxy_image) @@ -4666,18 +4767,38 @@ def _restart_storage_node_impl( return False node_info, _ = snode_api.info() - if not snode.ssd_pcie: - ssds = node_info['spdk_pcie_list'] + if lblk_mode: + # Re-resolve the persisted selection against the live host, + # SERIAL-FIRST (kernel names shift across reboots; the stored name + # is only the fallback for serial-less devices), then rebuild the + # AIO bdevs. A missing device degrades to STATUS_REMOVED in the + # reconcile below — same semantics as a missing NVMe controller. + blk_inventory, blk_err = snode_api.get_blockdevices() + if not blk_inventory: + logger.error(f"Failed to list block devices: {blk_err}") + return False + lblk_resolved, lblk_missing = utils.resolve_lblk_entries( + snode.lblk_devices, blk_inventory) + for entry in lblk_missing: + logger.warning(f"Configured block device {entry.get('name')} " + f"(serial {entry.get('serial')}) not found on host") + nvme_devs = utils.addAioDevices(rpc_client, snode, lblk_resolved) + if not nvme_devs: + logger.error("No eligible block devices were found!") + return False else: - ssds = [] - for ssd in snode.ssd_pcie: - if ssd in node_info['spdk_pcie_list']: - ssds.append(ssd) - - nvme_devs = addNvmeDevices(rpc_client, snode, ssds) - if not nvme_devs: - logger.error("No NVMe devices was found!") - return False + if not snode.ssd_pcie: + ssds = node_info['spdk_pcie_list'] + else: + ssds = [] + for ssd in snode.ssd_pcie: + if ssd in node_info['spdk_pcie_list']: + ssds.append(ssd) + + nvme_devs = addNvmeDevices(rpc_client, snode, ssds) + if not nvme_devs: + logger.error("No NVMe devices was found!") + return False logger.info(f"Devices found: {len(nvme_devs)}") logger.debug(nvme_devs) @@ -4702,8 +4823,15 @@ def _restart_storage_node_impl( if not db_dev.is_partition and not found_dev.is_partition: db_dev.device_name = found_dev.device_name db_dev.nvme_bdev = found_dev.nvme_bdev - db_dev.nvme_controller = found_dev.nvme_controller - db_dev.pcie_address = found_dev.pcie_address + if found_dev.bdev_type == "aio": + # AIO devices have no controller/PCIe identity; refresh + # the re-resolved kernel path instead. + db_dev.bdev_type = "aio" + db_dev.device_path = found_dev.device_path + db_dev.by_id_path = found_dev.by_id_path + else: + db_dev.nvme_controller = found_dev.nvme_controller + db_dev.pcie_address = found_dev.pcie_address # if db_dev.status in [ NVMeDevice.STATUS_ONLINE]: # db_dev.status = NVMeDevice.STATUS_UNAVAILABLE @@ -5135,7 +5263,9 @@ def list_storage_devices(node_id): "Name": device.alceml_name, "Size": utils.humanbytes(device.size), "Serial Number": device.serial_number, - "PCIe": device.pcie_address, + # lblk (aio) devices have no PCIe identity — show the kernel path. + "PCIe": (device.pcie_address if device.bdev_type != "aio" + else device.device_path), "Status": device.status, "IO Err": device.io_error, # Device health is only meaningful when its node is ONLINE/DOWN. @@ -5769,6 +5899,8 @@ def shutdown_storage_node(node_id, force=False, keep_auto_restart=False, return False pci_address = [] for dev in snode.nvme_devices: + if dev.bdev_type == "aio": + continue # lblk devices never left their kernel driver if dev.pcie_address not in pci_address: try: ret = snode.client(timeout=30, retry=1).bind_device_to_nvme(dev.pcie_address) @@ -6072,7 +6204,7 @@ def upgrade_automated_deployment_config(): def generate_automated_deployment_config(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_allowed, pci_blocked, cores_percentage=0, force=False, device_model="", size_range="", nvme_names=None, k8s=False, - calculate_hp_only=False, number_of_devices=0): + calculate_hp_only=False, number_of_devices=0, lblk_selection=None): # Reject an over-cap max_lvol here rather than only in the CLI: this is the # single entry point shared by `sn configure` and the k8s node-configure # job, and the value it writes into NODES_CONFIG_FILE becomes the node's @@ -6094,13 +6226,16 @@ def generate_automated_deployment_config(max_lvol, max_prov, sockets_to_use, nod if total_cores < 6: raise ValueError("Error: Not enough CPU cores to deploy storage node. Minimum 6 cores required.") - # load vfio_pci and uio_pci_generic - utils.load_kernel_module("vfio_pci") - utils.load_kernel_module("uio_pci_generic") + if lblk_selection is None: + # load vfio_pci and uio_pci_generic (nvme mode only — lblk keeps + # devices on their kernel driver, SPDK accesses them via AIO) + utils.load_kernel_module("vfio_pci") + utils.load_kernel_module("uio_pci_generic") nodes_config, system_info = utils.generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_allowed, pci_blocked, cores_percentage, force=force, - device_model=device_model, size_range=size_range, nvme_names=nvme_names) + device_model=device_model, size_range=size_range, nvme_names=nvme_names, + lblk_selection=lblk_selection) if not nodes_config or not nodes_config.get("nodes"): return False utils.store_config_file(nodes_config, constants.NODES_CONFIG_FILE, create_read_only_file=True) diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 4d8430ec56..bfa6797fbf 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -1,5 +1,6 @@ # coding=utf-8 import glob +import hashlib import json import logging import math @@ -1409,6 +1410,107 @@ def addNvmeDevices(rpc_client, snode, devs): return devices +def aio_bdev_name_for_serial(serial: str) -> str: + """Stable AIO bdev name derived from the device's serial identity — the + lblk analogue of the PCI-derived nvme controller name. Survives kernel + device renames across reboots. Whenever sanitization loses information + (special chars replaced, or truncation), a short hash of the ORIGINAL + serial is appended so distinct serials can never collide.""" + sanitized = re.sub(r"[^A-Za-z0-9_]", "_", serial) + if sanitized != serial or len(sanitized) > 40: + digest = hashlib.sha1(serial.encode()).hexdigest()[:6] + sanitized = f"{sanitized[:40]}_{digest}" + return f"aio_{sanitized}" + + +def resolve_lblk_entries(configured_entries, host_devices): + """Match the node's configured lblk devices against the live host + inventory, SERIAL-FIRST: kernel names shift across reboots, so the stored + name is only a fallback for devices without a resolvable serial. Returns + ``(resolved, missing)`` where resolved entries carry the CURRENT + name/path/by-id.""" + by_serial = {d["serial"]: d for d in host_devices} + by_name = {d["name"]: d for d in host_devices} + resolved, missing = [], [] + for entry in configured_entries: + live = by_serial.get(entry.get("serial")) or by_name.get(entry.get("name")) + if live is None: + missing.append(entry) + continue + resolved.append({ + "name": live["name"], + "current_path": live["device_path"], + "serial": entry.get("serial") or live["serial"], + "by_id": live.get("by_id_path") or entry.get("by_id", ""), + "size": int(live.get("size") or entry.get("size") or 0), + "numa": int(live.get("numa_node", entry.get("numa", -1))), + "model": live.get("model", ""), + "has_partitions": bool(live.get("has_partitions")), + }) + return resolved, missing + + +def addAioDevices(rpc_client, snode, blk_entries): + """lblk-mode sibling of addNvmeDevices: create one SPDK AIO bdev per + resolved block device and model it as an NVMeDevice with + bdev_type="aio". Idempotent — an already-present bdev (restart path) is + reused. Everything above the base bdev (alceml, PT, subsystems) is + built by the same code as for nvme devices.""" + devices = [] + next_physical_label = snode.physical_label + for entry in blk_entries: + bdev_name = aio_bdev_name_for_serial(entry["serial"]) + ret = rpc_client.get_bdevs(bdev_name) + if not ret: + # Prefer the by-id path as the filename so a udev rename between + # resolution and create cannot swap devices under us. + filename = entry.get("by_id") or entry["current_path"] + ret = rpc_client.bdev_aio_create(bdev_name, filename) + if not ret: + raise Exception( + f"bdev_aio_create failed for {bdev_name} ({filename}) " + f"on {rpc_client.host}") + rpc_client.bdev_examine(bdev_name) + rpc_client.bdev_wait_for_examine() + + ret = rpc_client.get_bdevs(bdev_name) + if not ret: + raise Exception(f"AIO bdev {bdev_name} not found after create on {rpc_client.host}") + bdev = ret[0] + total_size = bdev['block_size'] * bdev['num_blocks'] + if total_size == 0: + logger.warning(f"Skipping zero-size block device {entry['name']} ({bdev_name})") + continue + + # Queue-depth sampling feeds the control-plane hung-IO watchdog + # (AIO has no bdev_nvme-style timeout/action_on_timeout). + try: + rpc_client.bdev_set_qd_sampling_period( + bdev_name, constants.AIO_QD_SAMPLING_PERIOD_US) + except Exception as e: + logger.warning(f"qd-sampling enable failed on {bdev_name}: {e}") + + devices.append( + NVMeDevice({ + 'uuid': str(uuid.uuid4()), + 'device_name': entry["name"], + 'size': total_size, + 'physical_label': next_physical_label, + 'pcie_address': "", + 'model_id': entry.get("model", ""), + 'serial_number': entry["serial"], + 'nvme_bdev': bdev_name, + 'nvme_controller': "", + 'bdev_type': "aio", + 'device_path': entry["current_path"], + 'by_id_path': entry.get("by_id", ""), + 'node_id': snode.get_id(), + 'cluster_id': snode.cluster_id, + 'status': NVMeDevice.STATUS_ONLINE + })) + return devices + + def get_random_snapshot_vuid(all_lvols=None, all_snapshots=None): # Monotonic allocation via DBController.next_vuid — shares the single vuid # sequence with lvols/clones (one numeric space, so no cross-collision). @@ -1580,6 +1682,130 @@ def detect_nvmes(pci_allowed, pci_blocked, device_model, size_range, nvme_names) return nvmes +def filter_eligible_block_devices(devices, include_names=None, exclude_names=None, + include_serials=None, force_format=False): + """Eligibility filter for the lblk cluster mode (pure — unit-testable). + + ``devices`` is the list produced by node_utils.get_block_devices_info(). + A device is eligible iff it is a whole disk, not a special device + (LBLK_EXCLUDED_NAME_PREFIXES), carries no mountpoint anywhere in its + subtree, has no holders (LVM/md/dm-crypt), does not back the root + filesystem, is not read-only, has a non-zero size, and is unpartitioned + unless ``force_format`` (the actual wipe happens at add-node). + + Selection is one of: ``include_names`` (explicitly requested names must + exist AND be eligible — a busy requested device is a hard error), + ``exclude_names`` (all eligible minus these), ``include_serials`` + (matched against the serial/WWN identity). Without a selection, every + eligible disk is taken. + + Returns ``(eligible_devices, rejected)`` where rejected is a list of + ``(device_dict, reason)``. Raises ValueError on a requested-but- + ineligible name/serial or on duplicate serials among the selection. + """ + include_names = set(include_names or []) + exclude_names = set(exclude_names or []) + include_serials = set(include_serials or []) + + def _ineligible_reason(dev): + if dev.get("type") != "disk": + return "not a whole disk" + if dev["name"].startswith(constants.LBLK_EXCLUDED_NAME_PREFIXES): + return "special device type" + if dev.get("mounted_in_subtree"): + return "mounted (busy)" + if dev.get("holders"): + return f"held by {dev['holders']} (busy)" + if dev.get("is_root_disk"): + return "backs the root filesystem" + if dev.get("ro"): + return "read-only" + if not dev.get("size"): + return "zero size" + if dev.get("has_partitions") and not force_format: + return "partitioned (pass --force to format at add-node)" + return None + + eligible, rejected = [], [] + for dev in devices: + reason = _ineligible_reason(dev) + if reason: + rejected.append((dev, reason)) + else: + eligible.append(dev) + + by_name = {d["name"]: d for d in eligible} + rejected_by_name = {d["name"]: r for d, r in rejected} + if include_names: + missing = include_names - set(by_name) + if missing: + details = {n: rejected_by_name.get(n, "not present") for n in sorted(missing)} + raise ValueError(f"requested block devices are not eligible: {details}") + selected = [by_name[n] for n in sorted(include_names)] + elif include_serials: + by_serial = {d["serial"]: d for d in eligible} + missing_serials = include_serials - set(by_serial) + if missing_serials: + raise ValueError( + f"no eligible block device found for serial(s): {sorted(missing_serials)}") + selected = [by_serial[s] for s in sorted(include_serials)] + else: + selected = [d for d in eligible if d["name"] not in exclude_names] + + serials = [d["serial"] for d in selected] + dupes = {s for s in serials if serials.count(s) > 1} + if dupes: + raise ValueError( + f"duplicate serial number(s) among selected block devices: {sorted(dupes)}; " + f"device identity requires unique serials per node") + return selected, rejected + + +def detect_lblk_devices(include_names=None, exclude_names=None, + include_serials=None, force_format=False): + """Local-host block-device detection for `sn configure --lblk`. + Returns ``{name: config_entry}`` where config_entry is the shape stored + in the node config file's ``lblk_devices`` list.""" + devices = node_utils.get_block_devices_info() + selected, rejected = filter_eligible_block_devices( + devices, include_names=include_names, exclude_names=exclude_names, + include_serials=include_serials, force_format=force_format) + for dev, reason in rejected: + logger.debug(f"block device {dev['name']} skipped: {reason}") + result = {} + for dev in selected: + if dev.get("serial_synthetic"): + logger.warning( + f"block device {dev['name']} has no hardware serial/WWN; using " + f"synthetic identity {dev['serial']} (stable across reboots " + f"only while size and by-id path are unchanged)") + result[dev["name"]] = { + "name": dev["name"], + "serial": dev["serial"], + "by_id": dev.get("by_id_path", ""), + "size": int(dev["size"]), + "numa": int(dev.get("numa_node", -1)), + } + return result + + +def node_config_device_count(node) -> int: + """Number of storage devices a node-config entry carries — ssd_pcis for + nvme mode, lblk_devices for lblk mode.""" + return len(node.get("lblk_devices") or []) or len(node.get("ssd_pcis") or []) + + +def node_config_min_sys_memory(node) -> int: + """Minimum system memory for a node-config entry: 2 GiB + total device + capacity. lblk entries carry their sizes; nvme goes through nvme-cli.""" + lblk = node.get("lblk_devices") or [] + if lblk: + total = 2147483648 + sum(int(e.get("size") or 0) for e in lblk) + logger.debug(f"Minimum system memory is {humanbytes(total)}") + return int(total) + return calculate_minimum_sys_memory(node.get("ssd_pcis") or []) + + def get_total_capacity_of_nvme_devices(pci_lst): json_string = get_nvme_list_verbose() data = json.loads(json_string) @@ -1805,9 +2031,9 @@ def regenerate_config(new_config, old_config, force=False): if old_config["nodes"][i]["socket"] != new_config["nodes"][i]["socket"]: logger.error("The socket is changed, please rerun sbcli configure without upgrade firstly") return False - number_of_alcemls = len(new_config["nodes"][i]["ssd_pcis"]) + number_of_alcemls = node_config_device_count(new_config["nodes"][i]) if (old_config["nodes"][i]["cpu_mask"] != new_config["nodes"][i]["cpu_mask"] or - len(old_config["nodes"][i]["ssd_pcis"]) != len(new_config["nodes"][i]["ssd_pcis"]) or force): + node_config_device_count(old_config["nodes"][i]) != number_of_alcemls or force): try: isolated_cores = hexa_to_cpu_list(new_config["nodes"][i]["cpu_mask"]) except ValueError: @@ -1840,6 +2066,8 @@ def regenerate_config(new_config, old_config, force=False): number_of_distribs = 12 old_config["nodes"][i]["number_of_distribs"] = number_of_distribs old_config["nodes"][i]["ssd_pcis"] = new_config["nodes"][i]["ssd_pcis"] + if new_config["nodes"][i].get("lblk_devices") is not None: + old_config["nodes"][i]["lblk_devices"] = new_config["nodes"][i]["lblk_devices"] old_config["nodes"][i]["nic_ports"] = new_config["nodes"][i]["nic_ports"] for nic in old_config["nodes"][i]["nic_ports"]: if nic not in all_nics: @@ -1857,7 +2085,7 @@ def regenerate_config(new_config, old_config, force=False): old_config["nodes"][i]["small_pool_count"] = small_pool_count old_config["nodes"][i]["large_pool_count"] = large_pool_count old_config["nodes"][i]["huge_page_memory"] = minimum_hp_memory - minimum_sys_memory = calculate_minimum_sys_memory(old_config["nodes"][i]["ssd_pcis"]) + minimum_sys_memory = node_config_min_sys_memory(old_config["nodes"][i]) old_config["nodes"][i]["sys_memory"] = minimum_sys_memory memory_details = node_utils.get_memory_details() @@ -1867,7 +2095,7 @@ def regenerate_config(new_config, old_config, force=False): total_required_memory = 0 all_isolated_cores = set() for node in old_config["nodes"]: - if len(node["ssd_pcis"]) == 0: + if node_config_device_count(node) == 0: logger.error(f"There are no enough SSD devices on numa node {node['socket']}") return False total_required_memory += node["huge_page_memory"] + node["sys_memory"] @@ -1882,7 +2110,8 @@ def regenerate_config(new_config, old_config, force=False): def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_allowed, pci_blocked, - cores_percentage=0, force=False, device_model="", size_range="", nvme_names=None): + cores_percentage=0, force=False, device_model="", size_range="", nvme_names=None, + lblk_selection=None): system_info = {} nodes_config: dict = {"nodes": []} @@ -1890,12 +2119,36 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a validate_sockets(sockets_to_use, cores_by_numa) logger.debug(f"Cores by numa {cores_by_numa}") nics = detect_nics() - nvmes = detect_nvmes(pci_allowed, pci_blocked, device_model, size_range, nvme_names) - if not nvmes: - logger.error( - "There are no enough SSD devices on system, you may run 'sbctl sn clean-devices', to clean devices stored in /etc/simplyblock/sn_config_file") - return False, False - if force: + lblk_mode = lblk_selection is not None + lblk_entries: dict = {} + if lblk_mode: + # lblk cluster mode: eligible Linux block devices instead of NVMe + # PCIe controllers. No driver unbind, no formatting here (--force + # only marks partitioned disks eligible; the wipe happens at + # add-node). Reuse the NVMe NUMA-distribution scaffolding by + # presenting the same {name: {"numa_node": ...}} shape. + try: + lblk_entries = detect_lblk_devices( + include_names=lblk_selection.get("names"), + exclude_names=lblk_selection.get("names_exclude"), + include_serials=lblk_selection.get("serials"), + force_format=force) + except ValueError as e: + logger.error(str(e)) + return False, False + nvmes = {name: {"pci_address": "", "numa_node": entry["numa"]} + for name, entry in lblk_entries.items()} + if not nvmes: + logger.error("No eligible Linux block devices found on this system " + "(devices must be unmounted, unheld, unpartitioned whole disks)") + return False, False + else: + nvmes = detect_nvmes(pci_allowed, pci_blocked, device_model, size_range, nvme_names) + if not nvmes: + logger.error( + "There are no enough SSD devices on system, you may run 'sbctl sn clean-devices', to clean devices stored in /etc/simplyblock/sn_config_file") + return False, False + if force and not lblk_mode: nvme_devices = " ".join([f"/dev/{d}n1" for d in nvmes.keys()]) logger.warning(f"Formating Nvme devices {nvme_devices}") answer = input("Type YES/Y to continue: ").strip().lower() @@ -1927,11 +2180,15 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a for nvme, val in nvmes.items(): pci = val["pci_address"] numa = int(val["numa_node"]) - pci_utils.unbind_driver(pci) + if not lblk_mode: + # lblk keeps the kernel driver — the AIO bdev needs the block + # device usable by the kernel, the exact opposite of DPDK claim. + pci_utils.unbind_driver(pci) + dev_ref = pci if not lblk_mode else nvme if numa in sockets_to_use: - system_info[numa]["nvmes"].append(pci) + system_info[numa]["nvmes"].append(dev_ref) else: - system_info.setdefault(numa, {"cores": [], "nics": [], "nvmes": []})["nvmes"].append(pci) + system_info.setdefault(numa, {"cores": [], "nics": [], "nvmes": []})["nvmes"].append(dev_ref) nvme_by_numa: dict = {nid: [] for nid in sockets_to_use} nvme_numa_neg1 = [] @@ -1998,11 +2255,18 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a node_info["number_of_distribs"] = number_of_distribs nvme_neg1_list = all_nvmes_neg1_per_node[node_index] - for nvme_name in nvme_neg1_list: - node_info["ssd_pcis"].append(nvmes[nvme_name]["pci_address"]) - for nvme_name in nvme_per_core_group[idx]: - node_info["ssd_pcis"].append(nvmes[nvme_name]["pci_address"]) - number_of_alcemls = len(node_info["ssd_pcis"]) + if lblk_mode: + node_info["lblk_devices"] = [] + for dev_name in nvme_neg1_list: + node_info["lblk_devices"].append(lblk_entries[dev_name]) + for dev_name in nvme_per_core_group[idx]: + node_info["lblk_devices"].append(lblk_entries[dev_name]) + else: + for nvme_name in nvme_neg1_list: + node_info["ssd_pcis"].append(nvmes[nvme_name]["pci_address"]) + for nvme_name in nvme_per_core_group[idx]: + node_info["ssd_pcis"].append(nvmes[nvme_name]["pci_address"]) + number_of_alcemls = node_config_device_count(node_info) node_info["number_of_alcemls"] = number_of_alcemls small_pool_count, large_pool_count = calculate_pool_count(number_of_alcemls, 2 * number_of_distribs, len(core_group["isolated"]), @@ -2016,7 +2280,7 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a node_info["max_lvol"] = max_lvol node_info["max_size"] = max_prov node_info["huge_page_memory"] = max(minimum_hp_memory, max_prov) - minimum_sys_memory = calculate_minimum_sys_memory(node_info["ssd_pcis"]) + minimum_sys_memory = node_config_min_sys_memory(node_info) node_info["sys_memory"] = minimum_sys_memory all_nodes.append(node_info) node_index += 1 @@ -2027,7 +2291,7 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a total_required_memory = 0 all_isolated_cores = set() for node in all_nodes: - if len(node["ssd_pcis"]) == 0: + if node_config_device_count(node) == 0: logger.error(f"There are no enough SSD devices on numa node {node['socket']}") return False, False total_required_memory += node["huge_page_memory"] + node["sys_memory"] @@ -2265,12 +2529,31 @@ def validate_node_config(node): logger.error(f"Missing required distribution field '{field}' in node: {node.get('socket')}") return False + # Exactly one device source: PCIe SSDs (nvme mode) or Linux block + # devices (lblk mode). Both empty, or both populated, is a broken config. + lblk_devices = node.get("lblk_devices") or [] + if bool(node["ssd_pcis"]) == bool(lblk_devices): + logger.error( + f"Node config must carry exactly one non-empty device list of " + f"'ssd_pcis' / 'lblk_devices' in node: {node.get('socket')}") + return False + # Check ssd_pcis fields for ssd in node["ssd_pcis"]: if not is_valid_pci_address(ssd): logger.error(f"Missing required SSD field '{ssd}' in node: {node.get('socket')}") return False + # Check lblk_devices entries (manually editable — validate shape). + for entry in lblk_devices: + if not isinstance(entry, dict) or not entry.get("name") or not entry.get("serial"): + logger.error(f"lblk_devices entry missing 'name'/'serial' in node: {node.get('socket')}") + return False + if not isinstance(entry.get("size"), int) or entry["size"] <= 0: + logger.error(f"lblk_devices entry '{entry.get('name')}' needs a positive integer " + f"'size' in node: {node.get('socket')}") + return False + if not node["isolated"]: logger.error(f"'isolated' list is empty in node: {node.get('socket')}") return False diff --git a/simplyblock_web/api/internal/storage_node/docker.py b/simplyblock_web/api/internal/storage_node/docker.py index a67468c9b6..ee5f62c325 100644 --- a/simplyblock_web/api/internal/storage_node/docker.py +++ b/simplyblock_web/api/internal/storage_node/docker.py @@ -472,6 +472,36 @@ def get_node_lsblk(): return data +@api.get('/blockdevices', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'array', + 'items': {'type': 'object', 'additionalProperties': True}, + })}}}, +}) +def get_blockdevices(): + """Whole-disk inventory for the lblk cluster mode (eligibility fields, + serial/WWN identity, by-id path, NUMA).""" + return utils.get_response(node_utils.get_block_devices_info()) + + +class _WipeBlockDeviceParams(BaseModel): + device_name: str + + +@api.post('/wipe_block_device', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'boolean' + })}}}, +}) +def wipe_block_device(body: _WipeBlockDeviceParams): + """--force-format for lblk add-node: wipe partition/FS signatures from a + whole disk. Refuses busy devices (mounts/holders/root disk).""" + ok, reason = node_utils.wipe_block_device_signatures(body.device_name) + if not ok: + return utils.get_response(None, reason) + return utils.get_response(True) + + def get_nodes_config(): logger.debug("function:get_nodes_config start") file_path = constants.NODES_CONFIG_FILE diff --git a/simplyblock_web/api/internal/storage_node/kubernetes.py b/simplyblock_web/api/internal/storage_node/kubernetes.py index e558048ee4..6642db0831 100644 --- a/simplyblock_web/api/internal/storage_node/kubernetes.py +++ b/simplyblock_web/api/internal/storage_node/kubernetes.py @@ -138,6 +138,36 @@ def get_info(): }) +@api.get('/blockdevices', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'array', + 'items': {'type': 'object', 'additionalProperties': True}, + })}}}, +}) +def get_blockdevices(): + """Whole-disk inventory for the lblk cluster mode (eligibility fields, + serial/WWN identity, by-id path, NUMA).""" + return utils.get_response(node_utils.get_block_devices_info()) + + +class _WipeBlockDeviceParams(BaseModel): + device_name: str + + +@api.post('/wipe_block_device', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'boolean' + })}}}, +}) +def wipe_block_device(body: _WipeBlockDeviceParams): + """--force-format for lblk add-node: wipe partition/FS signatures from a + whole disk. Refuses busy devices (mounts/holders/root disk).""" + ok, reason = node_utils.wipe_block_device_signatures(body.device_name) + if not ok: + return utils.get_response(None, reason) + return utils.get_response(True) + + @api.post('/join_swarm', responses={ 200: {'content': {'application/json': {'schema': utils.response_schema({ 'type': 'boolean' diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index 0e40fbf1d4..5c1d663c4d 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -112,6 +112,7 @@ class ClusterDTO(BaseModel): node_affinity: bool anti_affinity: bool enable_failure_domain: bool + device_mode: str secret: SecretStr tls_enabled: bool max_fault_tolerance: int @@ -141,6 +142,7 @@ def from_model(model: Cluster, stat_obj: Optional[StatsObject] = None): node_affinity=model.enable_node_affinity, anti_affinity=model.strict_node_anti_affinity, enable_failure_domain=model.enable_failure_domain, + device_mode=model.device_mode, secret=model.secret, tls_enabled=model.tls, max_fault_tolerance=model.max_fault_tolerance, @@ -159,6 +161,8 @@ class DeviceDTO(BaseModel): serial_number: str nvme_controller: str pcie_address: str + bdev_type: str = "nvme" + device_path: str = "" status: str # None => health check not applicable (owning node not ONLINE/DOWN) health_check: Optional[bool] @@ -182,6 +186,8 @@ def from_model(model: NVMeDevice, storage_node_id: str, stat_obj: Optional[Stats serial_number=model.serial_number, nvme_controller=model.nvme_controller, pcie_address=model.pcie_address, + bdev_type=model.bdev_type, + device_path=model.device_path, status=model.status, health_check=model.health_check, retries_exhausted=model.retries_exhausted, diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index f94cf367e9..5be860320f 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -85,6 +85,7 @@ class ClusterParams(BaseModel): backup_config: Optional[BackupConfigParams] = None hashicorp_vault_settings: Optional[HashicorpVaultSettings] = None enable_failure_domain: bool = False + device_mode: Literal["nvme", "lblk"] = "nvme" @model_validator(mode="after") def validate_erasure_coding_scheme(self): diff --git a/simplyblock_web/api/v2/cluster/storage_node/__init__.py b/simplyblock_web/api/v2/cluster/storage_node/__init__.py index 1f68785187..3448b73518 100644 --- a/simplyblock_web/api/v2/cluster/storage_node/__init__.py +++ b/simplyblock_web/api/v2/cluster/storage_node/__init__.py @@ -55,6 +55,7 @@ class StorageNodeParams(BaseModel): spdk_sys_mem: Optional[str] = None failure_domain: Optional[int] = None expand: bool = False + force_format: bool = False @api.post('/', name='clusters:storage-nodes:create', status_code=201, responses={201: {"content": None}}) @@ -86,6 +87,7 @@ def add(request: Request, cluster: Cluster, parameters: StorageNodeParams, respo "spdk_sys_mem": parameters.spdk_sys_mem, "failure_domain": parameters.failure_domain, "expansion": parameters.expand, + "force_format": parameters.force_format, } ) if not task_id_or_false: diff --git a/simplyblock_web/node_configure.py b/simplyblock_web/node_configure.py index 84aeae4295..76c8488dfc 100755 --- a/simplyblock_web/node_configure.py +++ b/simplyblock_web/node_configure.py @@ -158,6 +158,39 @@ def parse_arguments() -> argparse.Namespace: dest='nvme_names', required=False ) + parser.add_argument( + '--lblk', + help='Configure the node with Linux block devices (lblk cluster mode) instead of ' + 'NVMe PCIe devices: eligible unmounted, unheld, unpartitioned whole disks are ' + 'wrapped in SPDK AIO bdevs', + action='store_true', + dest='lblk', + required=False + ) + parser.add_argument( + '--blk-names', + help='Comma separated list of block device names to use, like sdb,sdc (requires --lblk)', + type=str, + default='', + dest='blk_names', + required=False + ) + parser.add_argument( + '--blk-names-exclude', + help='Comma separated list of block device names to exclude, like sda (requires --lblk)', + type=str, + default='', + dest='blk_names_exclude', + required=False + ) + parser.add_argument( + '--blk-serials', + help='Comma separated list of block device serial numbers (or WWNs) to use (requires --lblk)', + type=str, + default='', + dest='blk_serials', + required=False + ) return parser.parse_args() @@ -198,6 +231,18 @@ def validate_arguments(args: argparse.Namespace) -> None: "pci-allowed and pci-blocked cannot be both specified" ) + use_lblk = bool(args.lblk or args.blk_names or args.blk_names_exclude or args.blk_serials) + if use_lblk and not args.lblk: + raise argparse.ArgumentError( + None, "--blk-names/--blk-names-exclude/--blk-serials require --lblk") + if use_lblk and (args.pci_allowed or args.pci_blocked or args.device_model + or args.size_range or args.nvme_names): + raise argparse.ArgumentError( + None, "--lblk cannot be combined with NVMe device selection options") + if sum([bool(args.blk_names), bool(args.blk_names_exclude), bool(args.blk_serials)]) > 1: + raise argparse.ArgumentError( + None, "Choose only one of --blk-names, --blk-names-exclude, --blk-serials") + max_prov = utils.parse_size(args.max_prov, assume_unit='G') if max_prov < 0: raise argparse.ArgumentError( @@ -258,6 +303,14 @@ def main() -> None: if args.nvme_names: nvme_names = [nvme_name.strip() for nvme_name in args.nvme_names.split(',') if nvme_name.strip()] + lblk_selection = None + if args.lblk: + lblk_selection = { + "names": [x.strip() for x in args.blk_names.split(',') if x.strip()] or None, + "names_exclude": [x.strip() for x in args.blk_names_exclude.split(',') if x.strip()] or None, + "serials": [x.strip() for x in args.blk_serials.split(',') if x.strip()] or None, + } + # Generate the deployment configuration generate_automated_deployment_config( max_lvol=int(args.max_lvol), @@ -271,7 +324,8 @@ def main() -> None: device_model=args.device_model, size_range=args.size_range, nvme_names=nvme_names, - k8s=True + k8s=True, + lblk_selection=lblk_selection ) except argparse.ArgumentError as e: diff --git a/simplyblock_web/node_utils.py b/simplyblock_web/node_utils.py index f902285983..9a4ebd40ae 100644 --- a/simplyblock_web/node_utils.py +++ b/simplyblock_web/node_utils.py @@ -148,6 +148,184 @@ def get_spdk_devices(): return [] +def _read_sysfs(path: str) -> str: + try: + with open(path, "r") as f: + return f.read().strip() + except OSError: + return "" + + +def _disk_holders(name: str) -> List[str]: + """Union of /sys/block//holders and every partition's holders — + catches LVM PVs, md members and dm-crypt without a mountpoint.""" + import os + holders: List[str] = [] + base = f"/sys/block/{name}" + try: + holders.extend(os.listdir(f"{base}/holders")) + except OSError: + pass + try: + for entry in os.listdir(base): + if entry.startswith(name): + try: + holders.extend(os.listdir(f"{base}/{entry}/holders")) + except OSError: + pass + except OSError: + pass + return sorted(set(holders)) + + +def _disk_by_id_path(name: str) -> str: + """Preferred stable /dev/disk/by-id symlink for a whole disk: wwn-* first, + then any other non-partition link. Empty when none exists.""" + import os + by_id_dir = "/dev/disk/by-id" + target = f"/dev/{name}" + candidates: List[str] = [] + try: + for entry in os.listdir(by_id_dir): + if "-part" in entry: + continue + path = os.path.join(by_id_dir, entry) + try: + if os.path.realpath(path) == target: + candidates.append(path) + except OSError: + continue + except OSError: + return "" + if not candidates: + return "" + candidates.sort(key=lambda p: (0 if "/wwn-" in p.replace("\\", "/") else 1, p)) + return candidates[0] + + +def _root_disk_names() -> List[str]: + """Kernel names of the disk(s) backing the root filesystem.""" + out, _, rc = shell_utils.run_command("findmnt -no SOURCE /") + if rc != 0 or not out.strip(): + return [] + source = out.strip().splitlines()[0] + # Walk PKNAME upwards (handles /dev/sda2, dm/LVM roots, etc.). + out, _, rc = shell_utils.run_command(f"lsblk -no PKNAME,NAME {source}") + names = set() + if rc == 0: + for line in out.splitlines(): + for token in line.split(): + names.add(token.strip()) + if source.startswith("/dev/"): + names.add(source[len("/dev/"):]) + return sorted(n for n in names if n) + + +def _subtree_mounted(dev: dict) -> bool: + if dev.get("mountpoint"): + return True + return any(_subtree_mounted(child) for child in dev.get("children") or []) + + +def get_block_devices_info() -> List[dict]: + """Inventory of whole-disk block devices for the lblk cluster mode. + + One dict per lsblk TYPE=disk entry, carrying everything the control + plane needs for eligibility filtering, identity (serial-first) and AIO + bdev creation. Sizes are bytes (lsblk -b). Serial falls back to WWN; + devices with neither get a synthetic-stable id derived from + hostname|by-id-or-name|size so identity survives reboots. + """ + import hashlib + import socket + + logger.debug("function:get_block_devices_info start") + out, err, rc = shell_utils.run_command( + "lsblk -J -b -o NAME,TYPE,SIZE,SERIAL,WWN,MOUNTPOINT,MODEL,ROTA,RO,VENDOR,PKNAME") + if rc != 0: + logger.error("Error running lsblk: %s", err) + return [] + try: + data = json.loads(out) + except json.JSONDecodeError as e: + logger.error("Failed to parse lsblk output: %s", e) + return [] + + root_disks = _root_disk_names() + hostname = socket.gethostname() + devices: List[dict] = [] + for dev in data.get("blockdevices", []): + if dev.get("type") != "disk": + continue + name = dev.get("name", "") + children = dev.get("children") or [] + by_id_path = _disk_by_id_path(name) + serial = (dev.get("serial") or "").strip() + wwn = (dev.get("wwn") or "").strip() + if not serial: + serial = wwn + synthetic = False + if not serial: + seed = f"{hostname}|{by_id_path or name}|{dev.get('size') or 0}" + serial = "SYN-" + hashlib.sha1(seed.encode()).hexdigest()[:16] + synthetic = True + devices.append({ + "name": name, + "device_path": f"/dev/{name}", + "type": dev.get("type"), + "size": int(dev.get("size") or 0), + "serial": serial, + "serial_synthetic": synthetic, + "wwn": wwn, + "model": (dev.get("model") or "").strip(), + "vendor": (dev.get("vendor") or "").strip(), + "rota": bool(dev.get("rota")), + "ro": bool(dev.get("ro")), + "has_partitions": any(c.get("type") == "part" for c in children), + "mounted_in_subtree": _subtree_mounted(dev), + "holders": _disk_holders(name), + "is_root_disk": name in root_disks, + "by_id_path": by_id_path, + "numa_node": int(_read_sysfs(f"/sys/block/{name}/device/numa_node") or -1), + }) + logger.debug("function:get_block_devices_info end") + return devices + + +def wipe_block_device_signatures(device_name: str) -> Tuple[bool, str]: + """Wipe partition-table / filesystem signatures from a whole disk + (`--force-format` on lblk add-node). Re-validates that the device is not + busy before touching it: any mountpoint in the subtree or any holder + refuses the wipe. Wipes partitions first, then the disk itself.""" + import re as _re + if not _re.match(r"^[a-zA-Z0-9_\-]+$", device_name): + return False, f"invalid device name {device_name!r}" + for dev in get_block_devices_info(): + if dev["name"] == device_name: + if dev["mounted_in_subtree"]: + return False, f"device {device_name} has mounted filesystems" + if dev["holders"]: + return False, (f"device {device_name} is held by " + f"{dev['holders']}") + if dev["is_root_disk"]: + return False, f"device {device_name} backs the root filesystem" + break + else: + return False, f"device {device_name} not found" + + out, _, rc = shell_utils.run_command( + f"lsblk -nro NAME -x NAME /dev/{device_name}") + if rc != 0: + return False, f"lsblk failed for {device_name}" + # Children (partitions) first, whole disk last. + names = [n for n in out.split() if n and n != device_name] + for name in names + [device_name]: + _, err, rc = shell_utils.run_command(f"wipefs -a /dev/{name}") + if rc != 0: + return False, f"wipefs /dev/{name} failed: {err}" + return True, "" + + def _get_mem_info(): logger.debug("function:_get_mem_info start") out, err, rc = shell_utils.run_command("cat /proc/meminfo") diff --git a/tests/integration/test_lblk_device_lifecycle.py b/tests/integration/test_lblk_device_lifecycle.py new file mode 100644 index 0000000000..725e2e789b --- /dev/null +++ b/tests/integration/test_lblk_device_lifecycle.py @@ -0,0 +1,358 @@ +# coding=utf-8 +"""Integration tests for the lblk (Linux block device / SPDK AIO) device +mode against a real FoundationDB (testcontainer via tests/integration/ +conftest.py). + +What runs REAL here: the FDB persistence layer (model round-trips), the +device_controller state machine (device_set_state flap accounting, forced +FAILED, device_remove) and the device_monitor watchdog logic. What is +faked: SPDK RPC (per-call mocks), the node agent (blockdevices inventory) +and the distr/event fan-out (patched at the consuming module). + +Scenarios: + 1. Model round-trip — cluster.device_mode, node.lblk_devices and the + per-device aio identity fields survive FDB serialization. + 2. Restart identity contract — resolve_lblk_entries + addAioDevices + against a renamed-device inventory produce records whose serials match + the DB reconcile keys (serial-first restart survival). + 3. Watchdog stall — real device_set_unavailable/io_error transitions in + FDB after the hung-IO threshold, with a countable flap. + 4. Flap limit — repeated LOCAL_FAILURE transitions force STATUS_FAILED + and queue failed-device migration. + 5. Disappearance — the presence sweep drives the real device_remove to + STATUS_REMOVED. + 6. reset_storage_device — aio liveness probe against a real DB record. +""" + +import uuid as uuid_mod +from unittest.mock import MagicMock, patch + +import pytest + +from simplyblock_core import constants, utils +from simplyblock_core.controllers import device_controller +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import device_monitor + + +CLUSTER_ID = "11111111-1111-1111-1111-111111111111" + + +def _seed_cluster(db, device_mode="lblk", status=Cluster.STATUS_ACTIVE): + cluster = Cluster() + cluster.uuid = CLUSTER_ID + cluster.status = status + cluster.device_mode = device_mode + cluster.ha_type = "ha" + cluster.write_to_db(db.kv_store) + return cluster + + +def _aio_device(serial="S1", name="sdb", status=NVMeDevice.STATUS_ONLINE): + dev = NVMeDevice() + dev.uuid = str(uuid_mod.uuid4()) + dev.cluster_id = CLUSTER_ID + dev.status = status + dev.bdev_type = "aio" + dev.serial_number = serial + dev.device_name = name + dev.device_path = f"/dev/{name}" + dev.by_id_path = f"/dev/disk/by-id/wwn-{serial}" + dev.nvme_bdev = utils.aio_bdev_name_for_serial(serial) + dev.size = 100 << 30 + dev.cluster_device_order = 0 + return dev + + +def _seed_node(db, devices, node_id=None, status=StorageNode.STATUS_ONLINE): + node = StorageNode() + node.uuid = node_id or str(uuid_mod.uuid4()) + node.cluster_id = CLUSTER_ID + node.status = status + node.mgmt_ip = "10.0.0.1" + node.api_endpoint = "10.0.0.1:5000" + node.lblk_devices = [ + {"name": d.device_name, "serial": d.serial_number, + "by_id": d.by_id_path, "size": d.size, "numa": 0} + for d in devices + ] + for d in devices: + d.node_id = node.uuid + node.nvme_devices = devices + node.write_to_db(db.kv_store) + return node + + +@pytest.fixture() +def db(): + return DBController() + + +# --------------------------------------------------------------------------- +# 1. Model round-trips +# --------------------------------------------------------------------------- + +class TestModelRoundTrip: + + def test_cluster_device_mode_persists(self, db): + _seed_cluster(db, device_mode="lblk") + read = db.get_cluster_by_id(CLUSTER_ID) + assert read.device_mode == "lblk" + + def test_cluster_device_mode_defaults_nvme(self, db): + cluster = Cluster() + cluster.uuid = CLUSTER_ID + cluster.status = Cluster.STATUS_ACTIVE + cluster.write_to_db(db.kv_store) + assert db.get_cluster_by_id(CLUSTER_ID).device_mode == "nvme" + + def test_node_and_device_fields_persist(self, db): + _seed_cluster(db) + dev = _aio_device(serial="S3Z8NX0M600123", name="sdb") + node = _seed_node(db, [dev]) + + read_node = db.get_storage_node_by_id(node.get_id()) + assert read_node.lblk_devices == [{ + "name": "sdb", "serial": "S3Z8NX0M600123", + "by_id": "/dev/disk/by-id/wwn-S3Z8NX0M600123", + "size": 100 << 30, "numa": 0, + }] + read_dev = read_node.nvme_devices[0] + assert read_dev.bdev_type == "aio" + assert read_dev.device_path == "/dev/sdb" + assert read_dev.by_id_path == "/dev/disk/by-id/wwn-S3Z8NX0M600123" + assert read_dev.nvme_bdev == utils.aio_bdev_name_for_serial("S3Z8NX0M600123") + assert read_dev.pcie_address == "" + assert read_dev.nvme_controller == "" + + def test_nvme_device_records_unaffected(self, db): + _seed_cluster(db, device_mode="nvme") + dev = NVMeDevice() + dev.uuid = str(uuid_mod.uuid4()) + dev.cluster_id = CLUSTER_ID + dev.status = NVMeDevice.STATUS_ONLINE + dev.pcie_address = "0000:00:1e.0" + dev.nvme_controller = "nvme_1e" + node = _seed_node(db, [dev]) + read_dev = db.get_storage_node_by_id(node.get_id()).nvme_devices[0] + assert read_dev.bdev_type == "nvme" + assert read_dev.pcie_address == "0000:00:1e.0" + + +# --------------------------------------------------------------------------- +# 2. Restart identity contract (serial-first over renamed devices) +# --------------------------------------------------------------------------- + +class TestRestartIdentityContract: + + def test_renamed_devices_resolve_to_same_reconcile_keys(self, db): + _seed_cluster(db) + d1, d2 = _aio_device("S1", "sdb"), _aio_device("S2", "sdc") + node = _seed_node(db, [d1, d2]) + node = db.get_storage_node_by_id(node.get_id()) + + # Reboot renamed sdb->sdd and sdc->sdb (a swap-adjacent shuffle). + live_inventory = [ + {"name": "sdd", "device_path": "/dev/sdd", "serial": "S1", + "by_id_path": "/dev/disk/by-id/wwn-S1", "size": 100 << 30, + "numa_node": 0, "model": "M"}, + {"name": "sdb", "device_path": "/dev/sdb", "serial": "S2", + "by_id_path": "/dev/disk/by-id/wwn-S2", "size": 100 << 30, + "numa_node": 0, "model": "M"}, + ] + resolved, missing = utils.resolve_lblk_entries(node.lblk_devices, live_inventory) + assert missing == [] + + rpc = MagicMock() + rpc.host = "t" + rpc.get_bdevs.return_value = None + created = {} + + def _create(name, filename, block_size=0): + created[name] = filename + rpc.get_bdevs.return_value = [ + {"name": name, "block_size": 4096, "num_blocks": 100}] + return name + + rpc.bdev_aio_create.side_effect = _create + discovered = utils.addAioDevices(rpc, node, resolved) + + # The reconcile at restart keys on serial_number: every discovered + # serial must match a DB record, with the CURRENT (renamed) path. + db_by_serial = {d.serial_number: d for d in node.nvme_devices} + for found in discovered: + assert found.serial_number in db_by_serial + by_serial = {d.serial_number: d for d in discovered} + assert by_serial["S1"].device_name == "sdd" + assert by_serial["S2"].device_name == "sdb" + # Stable bdev names: identical to what add-node created. + assert by_serial["S1"].nvme_bdev == db_by_serial["S1"].nvme_bdev + # AIO filename used the stable by-id path, not the volatile name. + assert created[by_serial["S1"].nvme_bdev] == "/dev/disk/by-id/wwn-S1" + + def test_missing_device_flagged_for_removal_semantics(self, db): + _seed_cluster(db) + node = _seed_node(db, [_aio_device("S1", "sdb"), _aio_device("S2", "sdc")]) + node = db.get_storage_node_by_id(node.get_id()) + live_inventory = [ + {"name": "sdb", "device_path": "/dev/sdb", "serial": "S1", + "by_id_path": "", "size": 1, "numa_node": 0, "model": "M"}, + ] + resolved, missing = utils.resolve_lblk_entries(node.lblk_devices, live_inventory) + assert [e["serial"] for e in resolved] == ["S1"] + assert [e["serial"] for e in missing] == ["S2"] + + +# --------------------------------------------------------------------------- +# 3-5. Watchdog + real device_controller state machine +# --------------------------------------------------------------------------- + +def _patched_fanout(): + """Patch the SPDK/event fan-out that device_set_state / device_remove + perform, leaving the FDB state machine real.""" + return [ + patch.object(device_controller, "distr_controller", MagicMock()), + patch.object(device_controller, "device_events", MagicMock()), + patch.object(StorageNode, "rpc_client", + lambda self, **kw: MagicMock()), + ] + + +class TestWatchdogAgainstRealStateMachine: + + def setup_method(self, _method): + device_monitor._aio_progress.clear() + device_monitor._aio_absent.clear() + + def test_stall_marks_device_unavailable_with_flap(self, db): + _seed_cluster(db) + dev = _aio_device("S1", "sdb") + node = _seed_node(db, [dev]) + node = db.get_storage_node_by_id(node.get_id()) + + stall_rpc = MagicMock() + stall_rpc.get_lvol_stats.return_value = {"bdevs": [{ + "num_read_ops": 100, "num_write_ops": 0, "num_unmap_ops": 0, + "queue_depth": 4}]} + inventory = [{"name": "sdb", "serial": "S1"}] + agent = MagicMock() + agent.get_blockdevices.return_value = (inventory, None) + + patches = _patched_fanout() + [ + patch.object(StorageNode, "client", lambda self, **kw: agent), + ] + for p in patches: + p.start() + try: + with patch.object(StorageNode, "rpc_client", + lambda self, **kw: stall_rpc): + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS + 1): + node = db.get_storage_node_by_id(node.get_id()) + device_monitor._sweep_aio_devices(node) + finally: + for p in patches: + p.stop() + + read = db.get_storage_device_by_id(dev.get_id()) + assert read.status == NVMeDevice.STATUS_UNAVAILABLE + assert read.io_error is True + assert read.flap_count == 1 # ONLINE -> UNAVAILABLE, LOCAL_FAILURE, node ONLINE + + def test_flap_limit_forces_failed_and_queues_migration(self, db): + _seed_cluster(db) + dev = _aio_device("S1", "sdb") + _seed_node(db, [dev]) + + patches = _patched_fanout() + [ + patch.object(device_controller, "DEVICE_FLAP_DEBOUNCE_SEC", 0.0), + # re-online between flaps queues FN_DEV_MIG — irrelevant noise here + patch.object(device_controller.tasks_controller, + "add_device_mig_task_for_node", return_value=None), + patch.object(device_controller.tasks_controller, + "add_device_failed_mig_task"), + ] + started = [p.start() for p in patches] + mig_task = started[-1] + try: + for _ in range(device_controller.DEVICE_FLAP_LIMIT + 1): + device_controller.device_set_unavailable( + dev.get_id(), cause=device_controller.CAUSE_LOCAL_FAILURE) + read = db.get_storage_device_by_id(dev.get_id()) + if read.status == NVMeDevice.STATUS_FAILED: + break + device_controller.device_set_online(dev.get_id()) + finally: + for p in patches: + p.stop() + + read = db.get_storage_device_by_id(dev.get_id()) + assert read.status == NVMeDevice.STATUS_FAILED + mig_task.assert_called_once_with(dev.get_id()) + + def test_disappearance_drives_real_device_remove(self, db): + _seed_cluster(db) + dev = _aio_device("S1", "sdb") + node = _seed_node(db, [dev]) + + agent = MagicMock() + agent.get_blockdevices.return_value = ( + [{"name": "other", "serial": "ZZZ"}], None) + idle_rpc = MagicMock() + idle_rpc.get_lvol_stats.return_value = {"bdevs": [{ + "num_read_ops": 0, "num_write_ops": 0, "num_unmap_ops": 0, + "queue_depth": 0}]} + + patches = _patched_fanout() + [ + patch.object(StorageNode, "client", lambda self, **kw: agent), + ] + for p in patches: + p.start() + try: + with patch.object(StorageNode, "rpc_client", + lambda self, **kw: idle_rpc): + for _ in range(constants.AIO_DEVICE_ABSENT_POLLS): + fresh = db.get_storage_node_by_id(node.get_id()) + device_monitor._sweep_aio_devices(fresh) + finally: + for p in patches: + p.stop() + + read = db.get_storage_device_by_id(dev.get_id()) + assert read.status == NVMeDevice.STATUS_REMOVED + + +# --------------------------------------------------------------------------- +# 6. reset_storage_device against a real DB record +# --------------------------------------------------------------------------- + +class TestResetAgainstDb: + + def test_reset_aio_liveness_probe_recovers_unavailable_device(self, db): + _seed_cluster(db) + dev = _aio_device("S1", "sdb", status=NVMeDevice.STATUS_UNAVAILABLE) + _seed_node(db, [dev]) + + rpc = MagicMock() + rpc.get_bdevs.return_value = [{"name": dev.nvme_bdev}] + + patches = _patched_fanout() + for p in patches: + p.start() + try: + with patch.object(StorageNode, "rpc_client", lambda self, **kw: rpc), \ + patch.object(device_controller.tasks_controller, + "get_active_dev_restart_task", return_value=None), \ + patch.object(device_controller.tasks_controller, + "add_device_mig_task_for_node", return_value=None): + assert device_controller.reset_storage_device(dev.get_id()) + finally: + for p in patches: + p.stop() + + read = db.get_storage_device_by_id(dev.get_id()) + assert read.status == NVMeDevice.STATUS_ONLINE + assert read.io_error is False + rpc.reset_device.assert_not_called() diff --git a/tests/unit/test_api_dto_secrets.py b/tests/unit/test_api_dto_secrets.py index 95e7685f1a..be07e95e33 100644 --- a/tests/unit/test_api_dto_secrets.py +++ b/tests/unit/test_api_dto_secrets.py @@ -52,6 +52,7 @@ def _build_cluster_dto(): node_affinity=False, anti_affinity=False, enable_failure_domain=False, + device_mode="nvme", secret=SecretStr("CLUSTER-SECRET"), tls_enabled=False, max_fault_tolerance=1, diff --git a/tests/unit/test_lblk_device_controller.py b/tests/unit/test_lblk_device_controller.py new file mode 100644 index 0000000000..50b0da2fd2 --- /dev/null +++ b/tests/unit/test_lblk_device_controller.py @@ -0,0 +1,263 @@ +# coding=utf-8 +"""Unit tests for the lblk (aio) branches in controllers/device_controller.py +and the mode-aware late-event gate in services/main_distr_event_collector.py. + +Covered: + - reset_storage_device: aio liveness-probe semantics — bdev present clears + the error state (no nvme controller reset issued); bdev gone returns + False so the tasks framework escalates to restart_device. + - get_device_health_info: aio SMART stub (never calls the nvme RPC). + - new_device_from_failed: aio path recreates the AIO bdev serial-first + from the live inventory instead of bind_device_to_spdk + controller + attach. + - restart_device: aio path recreates the missing AIO bdev (with qd + sampling re-armed) instead of the PCIe attach sequence. + - late-event gate: for aio devices the "controller gone?" probe is + get_bdevs_2 on the base bdev; a present bdev skips the late event. +""" + +import json +import unittest +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch + +from simplyblock_core.controllers import device_controller +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.services import main_distr_event_collector as collector + + +def _aio_dev(uid="dev-1", status=NVMeDevice.STATUS_ONLINE): + d = NVMeDevice() + d.uuid = uid + d.node_id = "node-1" + d.cluster_id = "cluster-1" + d.status = status + d.bdev_type = "aio" + d.serial_number = "S1" + d.nvme_bdev = "aio_S1" + d.device_path = "/dev/sdb" + d.by_id_path = "/dev/disk/by-id/wwn-1" + return d + + +class TestResetStorageDeviceAio(unittest.TestCase): + + def _run(self, bdev_present): + device = _aio_dev(status=NVMeDevice.STATUS_UNAVAILABLE) + snode = MagicMock() + snode.cluster_id = "cluster-1" + rpc = MagicMock() + rpc.get_bdevs.return_value = [{"name": "aio_S1"}] if bdev_present else None + snode.rpc_client.return_value = rpc + + db = MagicMock() + db.get_storage_device_by_id.return_value = device + db.get_storage_node_by_id.return_value = snode + + with patch.object(device_controller, "DBController", return_value=db), \ + patch.object(device_controller.tasks_controller, + "get_active_dev_restart_task", return_value=None), \ + patch.object(device_controller, "device_set_unavailable") as set_unavail, \ + patch.object(device_controller, "device_set_io_error") as set_io_err, \ + patch.object(device_controller, "device_set_retries_exhausted") as set_retries, \ + patch.object(device_controller, "device_set_online") as set_online, \ + patch.object(device_controller, "device_events"): + result = device_controller.reset_storage_device("dev-1") + return result, rpc, set_unavail, set_io_err, set_retries, set_online + + def test_bdev_present_clears_error_state(self): + result, rpc, _, set_io_err, set_retries, set_online = self._run(True) + self.assertTrue(result) + set_io_err.assert_called_once_with("dev-1", False) + set_retries.assert_called_once_with("dev-1", False) + set_online.assert_called_once() + rpc.reset_device.assert_not_called() + + def test_bdev_gone_fails_for_escalation(self): + result, rpc, _, set_io_err, _, set_online = self._run(False) + self.assertFalse(result) + set_io_err.assert_not_called() + set_online.assert_not_called() + rpc.reset_device.assert_not_called() + + +class TestHealthInfoAio(unittest.TestCase): + + def test_aio_returns_stub_without_nvme_rpc(self): + device = _aio_dev() + snode = MagicMock() + db = MagicMock() + db.get_storage_device_by_id.return_value = device + db.get_storage_node_by_id.return_value = snode + with patch.object(device_controller, "DBController", return_value=db): + ret = device_controller.get_device_health_info("dev-1") + data = json.loads(ret) + self.assertEqual(data["bdev_type"], "aio") + self.assertIsNone(data["smart"]) + snode.rpc_client.assert_not_called() + + +class TestNewDeviceFromFailedAio(unittest.TestCase): + + def _run(self, bdev_present_initially, inventory=None, create_ok=True): + device = _aio_dev(status=NVMeDevice.STATUS_FAILED_AND_MIGRATED) + node = MagicMock() + node.get_id.return_value = "node-1" + node.nvme_devices = [device] + + rpc = MagicMock() + state = {"present": bdev_present_initially} + + def _get_bdevs(name): + return [{"name": name}] if state["present"] else None + + def _aio_create(name, filename, block_size=0): + if create_ok: + state["present"] = True + return name + return None + + rpc.get_bdevs.side_effect = _get_bdevs + rpc.bdev_aio_create.side_effect = _aio_create + node.rpc_client.return_value = rpc + + client = MagicMock() + client.get_blockdevices.return_value = (inventory or [], None) + node.client.return_value = client + + db = MagicMock() + db.get_storage_nodes.return_value = [node] + with patch.object(device_controller, "DBController", return_value=db): + result = device_controller.new_device_from_failed("dev-1") + return result, rpc, db + + def test_bdev_already_present_no_create(self): + result, rpc, db = self._run(True) + self.assertTrue(result) + rpc.bdev_aio_create.assert_not_called() + db.atomic_update.assert_called_once() + + def test_recreates_bdev_serial_first_from_inventory(self): + inventory = [{"name": "sdx", "serial": "S1", + "device_path": "/dev/sdx", + "by_id_path": "/dev/disk/by-id/wwn-NEW"}] + result, rpc, _ = self._run(False, inventory=inventory) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-NEW") + rpc.bdev_set_qd_sampling_period.assert_called_once() + + def test_falls_back_to_stored_path_when_inventory_empty(self): + result, rpc, _ = self._run(False, inventory=[]) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1") + + def test_create_failure_returns_false(self): + result, _, db = self._run(False, inventory=[], create_ok=False) + self.assertFalse(result) + db.atomic_update.assert_not_called() + + +class TestRestartDeviceAio(unittest.TestCase): + + def _run(self, bdev_present): + device = _aio_dev(status=NVMeDevice.STATUS_REMOVED) + device.nvmf_nqn = "" + device.alceml_bdev = "" + snode = MagicMock() + snode.cluster_id = "cluster-1" + snode.nvme_devices = [device] + snode.jm_device = None + + rpc = MagicMock() + state = {"present": bdev_present} + rpc.get_bdevs.side_effect = ( + lambda name: [{"name": name}] if state["present"] else None) + + def _aio_create(name, filename, block_size=0): + state["present"] = True + return name + + rpc.bdev_aio_create.side_effect = _aio_create + snode.rpc_client.return_value = rpc + + client = MagicMock() + client.get_blockdevices.return_value = ([], None) + snode.client.return_value = client + + db = MagicMock() + db.get_storage_device_by_id.return_value = device + db.get_storage_node_by_id.return_value = snode + + with patch.object(device_controller, "DBController", return_value=db), \ + patch.object(device_controller.tasks_controller, + "get_active_dev_restart_task", return_value=None), \ + patch.object(device_controller, "device_set_retries_exhausted"), \ + patch.object(device_controller, "device_set_unavailable"), \ + patch.object(device_controller, "_def_create_device_stack", + return_value=True) as create_stack, \ + patch.object(device_controller, "device_set_io_error") as set_io_err, \ + patch.object(device_controller, "device_set_online") as set_online, \ + patch.object(device_controller, "device_events"): + result = device_controller.restart_device("dev-1") + return result, rpc, create_stack, set_io_err, set_online + + def test_missing_aio_bdev_recreated_before_stack(self): + result, rpc, create_stack, set_io_err, set_online = self._run(False) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1") + rpc.bdev_set_qd_sampling_period.assert_called_once() + create_stack.assert_called_once() + set_io_err.assert_called_once_with("dev-1", False) + set_online.assert_called_once() + # never the nvme path + rpc.bdev_nvme_controller_attach.assert_not_called() + + def test_present_aio_bdev_not_recreated(self): + result, rpc, create_stack, _, _ = self._run(True) + self.assertTrue(result) + rpc.bdev_aio_create.assert_not_called() + create_stack.assert_called_once() + + +class TestLateEventGateAio(unittest.TestCase): + + def test_present_aio_bdev_skips_late_event(self): + device = _aio_dev() + device.cluster_device_order = 7 + + home_node = MagicMock() + home_node.get_id.return_value = "node-1" + home_node.nvme_devices = [device] + + event_node = MagicMock() + event_node.get_id.return_value = "node-2" + rpc = MagicMock() + rpc.get_bdevs_2.return_value = ([{"name": "aio_S1"}], None) + event_node.rpc_client.return_value = rpc + + event = MagicMock() + event.message = "error_read" + event.node_id = "node-2" + event.storage_id = 7 + stale = datetime.now() - timedelta(seconds=30) + event.object_dict = {"timestamp": stale.strftime('%Y-%m-%dT%H:%M:%S.%fZ')} + + db = MagicMock() + db.get_storage_node_by_id.return_value = event_node + db.get_storage_nodes.return_value = [home_node] + + with patch.object(collector, "db", db), \ + patch.object(collector, "_is_target_remote_controller_healthy", + return_value=False): + collector.process_device_event(event, collector.logger) + + rpc.get_bdevs_2.assert_called_once_with("aio_S1") + rpc.bdev_nvme_controller_list_2.assert_not_called() + self.assertIn("skipping", event.status) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lblk_eligibility.py b/tests/unit/test_lblk_eligibility.py new file mode 100644 index 0000000000..6b6a8fe723 --- /dev/null +++ b/tests/unit/test_lblk_eligibility.py @@ -0,0 +1,329 @@ +# coding=utf-8 +"""Unit tests for lblk-mode device eligibility, detection, identity and +node-config schema (pure helpers in simplyblock_core.utils). + +Covered: + - filter_eligible_block_devices: every rejection reason, all three + selection methods (names / names-exclude / serials), hard errors on + requested-but-ineligible devices and duplicate serials, force_format. + - detect_lblk_devices: config-entry mapping + synthetic-serial warning. + - aio_bdev_name_for_serial: stability, sanitization, collision-freedom. + - resolve_lblk_entries: serial-first resolution (rename survival), stored + name fallback, missing devices, field refresh. + - node_config_device_count / node_config_min_sys_memory. + - validate_node_config: exactly-one-device-source rule + lblk entry shape. +""" + +import unittest +from unittest.mock import patch + +from simplyblock_core import utils + + +def _blk(name, serial="", size=100 << 30, mounted=False, holders=None, + root=False, ro=False, parts=False, dtype="disk", by_id="", + numa=0, synthetic=False, model="MODEL-X", wwn=""): + return { + "name": name, + "device_path": f"/dev/{name}", + "type": dtype, + "size": size, + "serial": serial or f"SER-{name}", + "serial_synthetic": synthetic, + "wwn": wwn, + "model": model, + "vendor": "ACME", + "rota": False, + "ro": ro, + "has_partitions": parts, + "mounted_in_subtree": mounted, + "holders": holders or [], + "is_root_disk": root, + "by_id_path": by_id, + "numa_node": numa, + } + + +class TestEligibility(unittest.TestCase): + + def _reasons(self, devs, **kwargs): + _, rejected = utils.filter_eligible_block_devices(devs, **kwargs) + return {d["name"]: r for d, r in rejected} + + def test_clean_disk_is_eligible(self): + sel, rej = utils.filter_eligible_block_devices([_blk("sdb")]) + self.assertEqual([d["name"] for d in sel], ["sdb"]) + self.assertEqual(rej, []) + + def test_partition_type_rejected(self): + reasons = self._reasons([_blk("sdb1", dtype="part")]) + self.assertIn("not a whole disk", reasons["sdb1"]) + + def test_special_prefixes_rejected(self): + for name in ("ram0", "loop3", "sr0", "zram1", "nbd0", "md127", "dm-0", "drbd0", "fd0"): + reasons = self._reasons([_blk(name, serial=f"S-{name}")]) + self.assertIn("special", reasons[name], name) + + def test_mounted_subtree_rejected(self): + reasons = self._reasons([_blk("sdb", mounted=True)]) + self.assertIn("busy", reasons["sdb"]) + + def test_holders_rejected(self): + reasons = self._reasons([_blk("sdb", holders=["dm-0"])]) + self.assertIn("held by", reasons["sdb"]) + + def test_root_disk_rejected(self): + reasons = self._reasons([_blk("sda", root=True)]) + self.assertIn("root", reasons["sda"]) + + def test_read_only_rejected(self): + reasons = self._reasons([_blk("sdb", ro=True)]) + self.assertIn("read-only", reasons["sdb"]) + + def test_zero_size_rejected(self): + reasons = self._reasons([_blk("sdb", size=0)]) + self.assertIn("zero size", reasons["sdb"]) + + def test_partitioned_rejected_without_force(self): + reasons = self._reasons([_blk("sdb", parts=True)]) + self.assertIn("partitioned", reasons["sdb"]) + + def test_partitioned_eligible_with_force(self): + sel, _ = utils.filter_eligible_block_devices( + [_blk("sdb", parts=True)], force_format=True) + self.assertEqual([d["name"] for d in sel], ["sdb"]) + + def test_nvme_kernel_devices_remain_eligible(self): + # "arbitrary Linux block devices" includes kernel-driver NVMe disks + sel, _ = utils.filter_eligible_block_devices([_blk("nvme0n1")]) + self.assertEqual([d["name"] for d in sel], ["nvme0n1"]) + + # --- selection methods ------------------------------------------------ + + def test_include_names_selects_only_requested(self): + devs = [_blk("sdb"), _blk("sdc"), _blk("sdd")] + sel, _ = utils.filter_eligible_block_devices(devs, include_names=["sdb", "sdd"]) + self.assertEqual(sorted(d["name"] for d in sel), ["sdb", "sdd"]) + + def test_include_names_busy_device_is_hard_error(self): + devs = [_blk("sdb", mounted=True)] + with self.assertRaises(ValueError) as ctx: + utils.filter_eligible_block_devices(devs, include_names=["sdb"]) + self.assertIn("busy", str(ctx.exception)) + + def test_include_names_absent_device_is_hard_error(self): + with self.assertRaises(ValueError) as ctx: + utils.filter_eligible_block_devices([_blk("sdb")], include_names=["sdz"]) + self.assertIn("not present", str(ctx.exception)) + + def test_exclude_names(self): + devs = [_blk("sdb"), _blk("sdc")] + sel, _ = utils.filter_eligible_block_devices(devs, exclude_names=["sdb"]) + self.assertEqual([d["name"] for d in sel], ["sdc"]) + + def test_include_serials(self): + devs = [_blk("sdb", serial="S1"), _blk("sdc", serial="S2")] + sel, _ = utils.filter_eligible_block_devices(devs, include_serials=["S2"]) + self.assertEqual([d["name"] for d in sel], ["sdc"]) + + def test_include_serials_missing_is_hard_error(self): + with self.assertRaises(ValueError) as ctx: + utils.filter_eligible_block_devices( + [_blk("sdb", serial="S1")], include_serials=["S9"]) + self.assertIn("S9", str(ctx.exception)) + + def test_duplicate_serials_hard_error(self): + devs = [_blk("sdb", serial="DUP"), _blk("sdc", serial="DUP")] + with self.assertRaises(ValueError) as ctx: + utils.filter_eligible_block_devices(devs) + self.assertIn("DUP", str(ctx.exception)) + + def test_no_selection_takes_all_eligible(self): + devs = [_blk("sdb"), _blk("sda", root=True, mounted=True), _blk("sdc")] + sel, _ = utils.filter_eligible_block_devices(devs) + self.assertEqual(sorted(d["name"] for d in sel), ["sdb", "sdc"]) + + +class TestDetectLblkDevices(unittest.TestCase): + + def test_maps_config_entry_shape(self): + devs = [_blk("sdb", serial="S1", by_id="/dev/disk/by-id/wwn-0x1", + size=42, numa=1)] + with patch.object(utils.node_utils, "get_block_devices_info", return_value=devs): + result = utils.detect_lblk_devices() + self.assertEqual(result, { + "sdb": {"name": "sdb", "serial": "S1", + "by_id": "/dev/disk/by-id/wwn-0x1", "size": 42, "numa": 1}, + }) + + def test_synthetic_serial_warns_but_passes(self): + devs = [_blk("sdb", serial="SYN-abc123", synthetic=True)] + with patch.object(utils.node_utils, "get_block_devices_info", return_value=devs), \ + patch.object(utils, "logger") as mock_logger: + result = utils.detect_lblk_devices() + self.assertIn("sdb", result) + self.assertTrue(mock_logger.warning.called) + + +class TestAioBdevName(unittest.TestCase): + + def test_plain_serial(self): + self.assertEqual(utils.aio_bdev_name_for_serial("S3Z8NX0M600123"), + "aio_S3Z8NX0M600123") + + def test_stable(self): + self.assertEqual(utils.aio_bdev_name_for_serial("ABC_1"), + utils.aio_bdev_name_for_serial("ABC_1")) + + def test_special_chars_never_collide(self): + a = utils.aio_bdev_name_for_serial("S1:A") + b = utils.aio_bdev_name_for_serial("S1;A") + self.assertNotEqual(a, b) + for name in (a, b): + self.assertRegex(name, r"^aio_[A-Za-z0-9_]+$") + + def test_long_serial_truncated_with_hash(self): + serial = "X" * 100 + name = utils.aio_bdev_name_for_serial(serial) + self.assertLessEqual(len(name), len("aio_") + 40 + 7) + self.assertNotEqual(name, utils.aio_bdev_name_for_serial("X" * 99)) + + +class TestResolveLblkEntries(unittest.TestCase): + + CONFIGURED = [ + {"name": "sdb", "serial": "S1", "by_id": "/dev/disk/by-id/wwn-1", + "size": 100, "numa": 0}, + {"name": "sdc", "serial": "S2", "by_id": "", "size": 200, "numa": 1}, + ] + + def test_serial_first_survives_rename(self): + # After reboot S1 moved sdb->sdx; stored name must NOT win. + host = [_blk("sdx", serial="S1", by_id="/dev/disk/by-id/wwn-1"), + _blk("sdc", serial="S2")] + resolved, missing = utils.resolve_lblk_entries(self.CONFIGURED, host) + self.assertEqual(missing, []) + by_serial = {e["serial"]: e for e in resolved} + self.assertEqual(by_serial["S1"]["name"], "sdx") + self.assertEqual(by_serial["S1"]["current_path"], "/dev/sdx") + + def test_name_fallback_when_serial_unknown(self): + # Host reports a different serial for sdb (e.g. synthetic drift); + # the stored name is the last-resort match. + host = [_blk("sdb", serial="OTHER"), _blk("sdc", serial="S2")] + resolved, missing = utils.resolve_lblk_entries(self.CONFIGURED, host) + self.assertEqual(missing, []) + names = {e["name"] for e in resolved} + self.assertEqual(names, {"sdb", "sdc"}) + + def test_missing_device_reported(self): + host = [_blk("sdc", serial="S2")] + resolved, missing = utils.resolve_lblk_entries(self.CONFIGURED, host) + self.assertEqual(len(resolved), 1) + self.assertEqual(missing[0]["serial"], "S1") + + def test_live_fields_refresh(self): + host = [_blk("sdb", serial="S1", by_id="/dev/disk/by-id/wwn-NEW", + size=999, numa=1, parts=True), + _blk("sdc", serial="S2")] + resolved, _ = utils.resolve_lblk_entries(self.CONFIGURED, host) + entry = next(e for e in resolved if e["serial"] == "S1") + self.assertEqual(entry["by_id"], "/dev/disk/by-id/wwn-NEW") + self.assertEqual(entry["size"], 999) + self.assertEqual(entry["numa"], 1) + self.assertTrue(entry["has_partitions"]) + + +class TestNodeConfigHelpers(unittest.TestCase): + + def test_device_count_lblk(self): + node = {"ssd_pcis": [], "lblk_devices": [{"name": "sdb"}, {"name": "sdc"}]} + self.assertEqual(utils.node_config_device_count(node), 2) + + def test_device_count_nvme(self): + node = {"ssd_pcis": ["0000:00:1e.0"], "lblk_devices": []} + self.assertEqual(utils.node_config_device_count(node), 1) + + def test_device_count_missing_keys(self): + self.assertEqual(utils.node_config_device_count({}), 0) + + def test_min_sys_memory_lblk_sums_sizes(self): + node = {"lblk_devices": [{"size": 10}, {"size": 32}]} + self.assertEqual(utils.node_config_min_sys_memory(node), 2147483648 + 42) + + def test_min_sys_memory_nvme_delegates(self): + node = {"ssd_pcis": ["0000:00:1e.0"], "lblk_devices": []} + with patch.object(utils, "calculate_minimum_sys_memory", return_value=7) as m: + self.assertEqual(utils.node_config_min_sys_memory(node), 7) + m.assert_called_once_with(["0000:00:1e.0"]) + + +class TestValidateNodeConfig(unittest.TestCase): + + def _node(self, ssd_pcis=None, lblk_devices=None): + return { + "socket": 0, + "cpu_mask": "0x3", + "isolated": [0, 1], + "l-cores": "0@0,1@1", + "number_of_alcemls": 1, + "distribution": { + "app_thread_core": [0], "jm_cpu_core": [0], + "poller_cpu_cores": [1], "alceml_cpu_cores": [1], + "distrib_cpu_cores": [1], "jc_singleton_core": [0], + }, + "ssd_pcis": ssd_pcis if ssd_pcis is not None else [], + "lblk_devices": lblk_devices if lblk_devices is not None else [], + "nic_ports": ["eth0"], + "number_of_distribs": 2, + "small_pool_count": 1, + "large_pool_count": 1, + "max_lvol": 10, + "max_size": 1 << 30, + "huge_page_memory": 1 << 30, + "sys_memory": 1 << 31, + } + + def test_valid_nvme_config(self): + self.assertTrue(utils.validate_node_config(self._node(ssd_pcis=["0000:00:1e.0"]))) + + def test_valid_lblk_config(self): + node = self._node(lblk_devices=[{"name": "sdb", "serial": "S1", "size": 100}]) + self.assertTrue(utils.validate_node_config(node)) + + def test_nvme_config_without_lblk_key_still_valid(self): + node = self._node(ssd_pcis=["0000:00:1e.0"]) + del node["lblk_devices"] + self.assertTrue(utils.validate_node_config(node)) + + def test_both_sources_rejected(self): + node = self._node(ssd_pcis=["0000:00:1e.0"], + lblk_devices=[{"name": "sdb", "serial": "S1", "size": 1}]) + self.assertFalse(utils.validate_node_config(node)) + + def test_neither_source_rejected(self): + self.assertFalse(utils.validate_node_config(self._node())) + + def test_lblk_entry_missing_serial_rejected(self): + node = self._node(lblk_devices=[{"name": "sdb", "size": 100}]) + self.assertFalse(utils.validate_node_config(node)) + + def test_lblk_entry_missing_name_rejected(self): + node = self._node(lblk_devices=[{"serial": "S1", "size": 100}]) + self.assertFalse(utils.validate_node_config(node)) + + def test_lblk_entry_bad_size_rejected(self): + for size in (0, -5, "100", None): + node = self._node(lblk_devices=[{"name": "sdb", "serial": "S1", "size": size}]) + self.assertFalse(utils.validate_node_config(node), f"size={size!r}") + + def test_lblk_entry_not_a_dict_rejected(self): + node = self._node(lblk_devices=["sdb"]) + self.assertFalse(utils.validate_node_config(node)) + + def test_invalid_pci_still_rejected(self): + self.assertFalse(utils.validate_node_config(self._node(ssd_pcis=["/dev/sdb"]))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lblk_onboarding.py b/tests/unit/test_lblk_onboarding.py new file mode 100644 index 0000000000..407ca1d449 --- /dev/null +++ b/tests/unit/test_lblk_onboarding.py @@ -0,0 +1,247 @@ +# coding=utf-8 +"""Unit tests for lblk-mode device onboarding. + +Covered: + - utils.addAioDevices: fresh-create vs reuse (restart idempotency), + by-id-preferred filename, examine + qd-sampling wiring, zero-size skip, + create-failure raise, full NVMeDevice field population. + - storage_node_ops._classify_existing_endpoint_record: serial-based + overlap detection for lblk nodes (add-node idempotency). + - cluster_ops._validated_device_mode. + - constants sanity (placeholder BDF shape, excluded prefixes are a tuple + usable with str.startswith). +""" + +import re +import unittest +from unittest.mock import MagicMock + +from simplyblock_core import cluster_ops, constants, utils +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.storage_node_ops import _classify_existing_endpoint_record + + +def _entry(name="sdb", serial="S1", by_id="/dev/disk/by-id/wwn-1", + size=100 << 30, numa=0, model="MODEL-X"): + return {"name": name, "serial": serial, "by_id": by_id, "size": size, + "numa": numa, "model": model, "current_path": f"/dev/{name}", + "has_partitions": False} + + +def _snode(node_id="node-1", cluster_id="cluster-1", physical_label=3): + n = StorageNode() + n.uuid = node_id + n.cluster_id = cluster_id + n.physical_label = physical_label + return n + + +class _FakeRpc: + """Minimal SPDK RPC fake for addAioDevices: get_bdevs answers from an + internal registry; bdev_aio_create registers; every call is recorded.""" + + def __init__(self, existing=None, create_ok=True, block_size=4096, + num_blocks=1000): + self.host = "test-host" + self.bdevs = dict(existing or {}) + self.create_ok = create_ok + self.block_size = block_size + self.num_blocks = num_blocks + self.calls = [] + + def get_bdevs(self, name): + self.calls.append(("get_bdevs", name)) + if name in self.bdevs: + return [self.bdevs[name]] + return None + + def bdev_aio_create(self, name, filename, block_size=0): + self.calls.append(("bdev_aio_create", name, filename)) + if not self.create_ok: + return None + self.bdevs[name] = {"name": name, "block_size": self.block_size, + "num_blocks": self.num_blocks} + return name + + def bdev_examine(self, name): + self.calls.append(("bdev_examine", name)) + return True + + def bdev_wait_for_examine(self): + self.calls.append(("bdev_wait_for_examine",)) + return True + + def bdev_set_qd_sampling_period(self, name, period): + self.calls.append(("qd_sampling", name, period)) + return True + + def _called(self, method): + return [c for c in self.calls if c[0] == method] + + +class TestAddAioDevices(unittest.TestCase): + + def test_fresh_create_full_field_population(self): + rpc = _FakeRpc() + snode = _snode() + devs = utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(len(devs), 1) + dev = devs[0] + self.assertIsInstance(dev, NVMeDevice) + self.assertEqual(dev.bdev_type, "aio") + self.assertEqual(dev.nvme_bdev, utils.aio_bdev_name_for_serial("S1")) + self.assertEqual(dev.serial_number, "S1") + self.assertEqual(dev.device_name, "sdb") + self.assertEqual(dev.device_path, "/dev/sdb") + self.assertEqual(dev.by_id_path, "/dev/disk/by-id/wwn-1") + self.assertEqual(dev.pcie_address, "") + self.assertEqual(dev.nvme_controller, "") + self.assertEqual(dev.model_id, "MODEL-X") + self.assertEqual(dev.size, 4096 * 1000) + self.assertEqual(dev.physical_label, 3) + self.assertEqual(dev.node_id, "node-1") + self.assertEqual(dev.cluster_id, "cluster-1") + self.assertEqual(dev.status, NVMeDevice.STATUS_ONLINE) + + def test_filename_prefers_by_id(self): + rpc = _FakeRpc() + utils.addAioDevices(rpc, _snode(), [_entry()]) + create = rpc._called("bdev_aio_create")[0] + self.assertEqual(create[2], "/dev/disk/by-id/wwn-1") + + def test_filename_falls_back_to_current_path(self): + rpc = _FakeRpc() + utils.addAioDevices(rpc, _snode(), [_entry(by_id="")]) + create = rpc._called("bdev_aio_create")[0] + self.assertEqual(create[2], "/dev/sdb") + + def test_reuse_existing_bdev_no_create(self): + name = utils.aio_bdev_name_for_serial("S1") + rpc = _FakeRpc(existing={name: {"name": name, "block_size": 4096, + "num_blocks": 10}}) + devs = utils.addAioDevices(rpc, _snode(), [_entry()]) + self.assertEqual(len(devs), 1) + self.assertEqual(rpc._called("bdev_aio_create"), []) + + def test_examine_and_qd_sampling_wired(self): + rpc = _FakeRpc() + utils.addAioDevices(rpc, _snode(), [_entry()]) + self.assertTrue(rpc._called("bdev_examine")) + self.assertTrue(rpc._called("bdev_wait_for_examine")) + qd = rpc._called("qd_sampling")[0] + self.assertEqual(qd[2], constants.AIO_QD_SAMPLING_PERIOD_US) + + def test_zero_size_skipped(self): + rpc = _FakeRpc(num_blocks=0) + devs = utils.addAioDevices(rpc, _snode(), [_entry()]) + self.assertEqual(devs, []) + + def test_create_failure_raises(self): + rpc = _FakeRpc(create_ok=False) + with self.assertRaises(Exception): + utils.addAioDevices(rpc, _snode(), [_entry()]) + + def test_multiple_devices(self): + rpc = _FakeRpc() + devs = utils.addAioDevices(rpc, _snode(), [ + _entry(name="sdb", serial="S1"), _entry(name="sdc", serial="S2")]) + self.assertEqual([d.serial_number for d in devs], ["S1", "S2"]) + self.assertEqual(len({d.nvme_bdev for d in devs}), 2) + + +class TestClassifyEndpointRecordLblk(unittest.TestCase): + + def _db_with(self, node): + db = MagicMock() + db.get_storage_nodes_by_cluster_id.return_value = [node] + return db + + def _lblk_node(self, status, serials=("S1",)): + n = StorageNode() + n.uuid = "existing" + n.api_endpoint = "1.2.3.4:5000" + n.status = status + n.ssd_pcie = [] + n.lblk_devices = [{"name": f"sd{i}", "serial": s} + for i, s in enumerate(serials)] + return n + + def test_serial_overlap_online_is_already_added(self): + node = self._lblk_node(StorageNode.STATUS_ONLINE) + action, found = _classify_existing_endpoint_record( + self._db_with(node), "c1", "1.2.3.4:5000", [], lblk_serials=["S1"]) + self.assertEqual(action, "already_added") + self.assertIs(found, node) + + def test_serial_overlap_in_creation_is_cleanup(self): + node = self._lblk_node(StorageNode.STATUS_IN_CREATION) + action, _ = _classify_existing_endpoint_record( + self._db_with(node), "c1", "1.2.3.4:5000", [], lblk_serials=["S1"]) + self.assertEqual(action, "cleanup") + + def test_serial_overlap_other_status_is_conflict(self): + node = self._lblk_node(StorageNode.STATUS_OFFLINE) + action, _ = _classify_existing_endpoint_record( + self._db_with(node), "c1", "1.2.3.4:5000", [], lblk_serials=["S1"]) + self.assertEqual(action, "conflict") + + def test_no_serial_overlap_no_match(self): + node = self._lblk_node(StorageNode.STATUS_ONLINE, serials=("OTHER",)) + action, found = _classify_existing_endpoint_record( + self._db_with(node), "c1", "1.2.3.4:5000", [], lblk_serials=["S1"]) + self.assertIsNone(action) + self.assertIsNone(found) + + def test_different_endpoint_ignored(self): + node = self._lblk_node(StorageNode.STATUS_ONLINE) + action, _ = _classify_existing_endpoint_record( + self._db_with(node), "c1", "9.9.9.9:5000", [], lblk_serials=["S1"]) + self.assertIsNone(action) + + def test_nvme_pcie_overlap_still_works(self): + node = StorageNode() + node.uuid = "existing" + node.api_endpoint = "1.2.3.4:5000" + node.status = StorageNode.STATUS_ONLINE + node.ssd_pcie = ["0000:00:1e.0"] + action, _ = _classify_existing_endpoint_record( + self._db_with(node), "c1", "1.2.3.4:5000", ["0000:00:1e.0"]) + self.assertEqual(action, "already_added") + + +class TestDeviceModeValidation(unittest.TestCase): + + def test_accepts_both_modes_case_insensitive(self): + self.assertEqual(cluster_ops._validated_device_mode("nvme"), "nvme") + self.assertEqual(cluster_ops._validated_device_mode("LBLK"), "lblk") + + def test_none_defaults_to_nvme(self): + self.assertEqual(cluster_ops._validated_device_mode(None), "nvme") + + def test_rejects_unknown(self): + with self.assertRaises(ValueError): + cluster_ops._validated_device_mode("scsi") + + +class TestLblkConstants(unittest.TestCase): + + def test_placeholder_is_valid_bdf_and_never_a_device(self): + self.assertTrue(re.fullmatch( + r"[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]", + constants.LBLK_PCI_ALLOWED_PLACEHOLDER)) + self.assertEqual(constants.LBLK_PCI_ALLOWED_PLACEHOLDER, "0000:00:00.0") + + def test_excluded_prefixes_usable_with_startswith(self): + self.assertIsInstance(constants.LBLK_EXCLUDED_NAME_PREFIXES, tuple) + self.assertTrue("loop7".startswith(constants.LBLK_EXCLUDED_NAME_PREFIXES)) + self.assertFalse("sdb".startswith(constants.LBLK_EXCLUDED_NAME_PREFIXES)) + + def test_watchdog_thresholds_positive(self): + self.assertGreater(constants.AIO_HUNG_IO_STALL_POLLS, 0) + self.assertGreater(constants.AIO_DEVICE_ABSENT_POLLS, 0) + self.assertGreater(constants.AIO_QD_SAMPLING_PERIOD_US, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lblk_watchdog.py b/tests/unit/test_lblk_watchdog.py new file mode 100644 index 0000000000..30e56614f6 --- /dev/null +++ b/tests/unit/test_lblk_watchdog.py @@ -0,0 +1,265 @@ +# coding=utf-8 +"""Unit tests for the lblk hung-IO watchdog and device-disappearance sweep +in services/device_monitor.py. + +The watchdog is the control-plane replacement for bdev_nvme's +timeout_us/action_on_timeout (which AIO bdevs lack): queue-depth-sampled +iostat with no completion progress across N polls => the device is fed into +the SAME machinery an erroring nvme device hits (io_error + UNAVAILABLE, +countable LOCAL_FAILURE cause). Disappearance from the host inventory => +device_remove, the SPDK_BDEV_EVENT_REMOVE treatment. + +Covered: + - stall accumulation requires inflight IO on EVERY poll AND zero progress + - any completion progress resets the window + - RPC failure / missing bdevs / missing queue_depth freeze (never count) + - missing queue_depth re-arms qd-sampling + - threshold trip returns the device + - non-ONLINE devices are ignored and their tracking state cleared + - nvme (bdev_type != aio) devices are never touched + - presence sweep: absent-debounce, recovery clears the counter, inventory + failure freezes, serial OR name match counts as present + - action dispatch: 1 stalled -> io_error+UNAVAILABLE(LOCAL_FAILURE); + >=2 stalled -> node-level auto-restart; gone -> device_remove +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import constants +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.services import device_monitor +from simplyblock_core.services.device_monitor import ( + _check_aio_device_presence, + _check_aio_hung_io, + _sweep_aio_devices, +) + + +def _aio_dev(uid="dev-1", status=NVMeDevice.STATUS_ONLINE, serial="S1", + name="sdb", bdev_type="aio"): + d = NVMeDevice() + d.uuid = uid + d.status = status + d.serial_number = serial + d.device_name = name + d.bdev_type = bdev_type + d.nvme_bdev = f"aio_{serial}" + return d + + +def _node(devs, node_id="node-1"): + n = MagicMock() + n.get_id.return_value = node_id + n.nvme_devices = devs + return n + + +def _rpc_with_stats(stats_by_bdev): + rpc = MagicMock() + + def _stats(name): + entry = stats_by_bdev.get(name) + if entry is None: + return {"bdevs": []} + if isinstance(entry, Exception): + raise entry + return {"bdevs": [entry]} + + rpc.get_lvol_stats.side_effect = _stats + return rpc + + +def _stat(total_ops, queue_depth): + return {"num_read_ops": total_ops, "num_write_ops": 0, + "num_unmap_ops": 0, "queue_depth": queue_depth} + + +class WatchdogBase(unittest.TestCase): + def setUp(self): + device_monitor._aio_progress.clear() + device_monitor._aio_absent.clear() + + +class TestHungIoDetection(WatchdogBase): + + def test_first_poll_never_stalls(self): + dev = _aio_dev() + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + self.assertEqual(_check_aio_hung_io(_node([dev]), rpc), []) + + def test_stall_requires_threshold_consecutive_polls(self): + dev = _aio_dev() + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + # poll 1 primes; polls 2..N-1 accumulate below threshold + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS): + self.assertEqual(_check_aio_hung_io(node, rpc), []) + # poll that reaches the threshold trips + self.assertEqual(_check_aio_hung_io(node, rpc), [dev]) + + def test_progress_resets_window(self): + dev = _aio_dev() + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS): + _check_aio_hung_io(node, rpc) + # completions advanced -> reset + rpc2 = _rpc_with_stats({dev.nvme_bdev: _stat(101, 5)}) + self.assertEqual(_check_aio_hung_io(node, rpc2), []) + # stalling again needs the full window again + rpc3 = _rpc_with_stats({dev.nvme_bdev: _stat(101, 5)}) + self.assertEqual(_check_aio_hung_io(node, rpc3), []) + + def test_zero_queue_depth_is_idle_not_stall(self): + dev = _aio_dev() + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 0)}) + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS + 2): + self.assertEqual(_check_aio_hung_io(node, rpc), []) + + def test_rpc_exception_freezes_counter(self): + dev = _aio_dev() + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS): + _check_aio_hung_io(node, rpc) + # one failing poll must neither trip nor reset + bad = _rpc_with_stats({dev.nvme_bdev: RuntimeError("rpc down")}) + self.assertEqual(_check_aio_hung_io(node, bad), []) + # next good stalled poll trips (counter was frozen, not reset) + self.assertEqual(_check_aio_hung_io(node, rpc), [dev]) + + def test_empty_bdevs_freezes_counter(self): + dev = _aio_dev() + node = _node([dev]) + rpc = _rpc_with_stats({}) # no entry -> {"bdevs": []} + self.assertEqual(_check_aio_hung_io(node, rpc), []) + self.assertNotIn(dev.get_id(), device_monitor._aio_progress) + + def test_missing_queue_depth_rearms_sampling_and_freezes(self): + dev = _aio_dev() + node = _node([dev]) + stat = {"num_read_ops": 1, "num_write_ops": 0, "num_unmap_ops": 0} + rpc = _rpc_with_stats({dev.nvme_bdev: stat}) + self.assertEqual(_check_aio_hung_io(node, rpc), []) + rpc.bdev_set_qd_sampling_period.assert_called_once_with( + dev.nvme_bdev, constants.AIO_QD_SAMPLING_PERIOD_US) + + def test_non_online_device_ignored_and_state_cleared(self): + dev = _aio_dev() + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + _check_aio_hung_io(node, rpc) + self.assertIn(dev.get_id(), device_monitor._aio_progress) + dev.status = NVMeDevice.STATUS_UNAVAILABLE + self.assertEqual(_check_aio_hung_io(node, rpc), []) + self.assertNotIn(dev.get_id(), device_monitor._aio_progress) + + def test_nvme_devices_never_touched(self): + dev = _aio_dev(bdev_type="nvme") + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS + 2): + self.assertEqual(_check_aio_hung_io(node, rpc), []) + rpc.get_lvol_stats.assert_not_called() + + +class TestDevicePresence(WatchdogBase): + + def _node_with_inventory(self, devs, inventory): + node = _node(devs) + client = MagicMock() + client.get_blockdevices.return_value = (inventory, None) + node.client.return_value = client + return node + + def test_present_by_serial(self): + dev = _aio_dev(serial="S1", name="sdb") + # renamed on host: serial still matches + node = self._node_with_inventory([dev], [{"name": "sdx", "serial": "S1"}]) + self.assertEqual(_check_aio_device_presence(node), []) + self.assertNotIn(dev.get_id(), device_monitor._aio_absent) + + def test_present_by_name_fallback(self): + dev = _aio_dev(serial="S1", name="sdb") + node = self._node_with_inventory([dev], [{"name": "sdb", "serial": "OTHER"}]) + self.assertEqual(_check_aio_device_presence(node), []) + + def test_absent_debounced_then_reported(self): + dev = _aio_dev() + node = self._node_with_inventory([dev], [{"name": "sdz", "serial": "ZZ"}]) + for _ in range(constants.AIO_DEVICE_ABSENT_POLLS - 1): + self.assertEqual(_check_aio_device_presence(node), []) + self.assertEqual(_check_aio_device_presence(node), [dev]) + + def test_reappearance_clears_counter(self): + dev = _aio_dev(serial="S1") + gone = self._node_with_inventory([dev], []) + # inventory [] is falsy -> unknown, so use a non-matching entry + gone = self._node_with_inventory([dev], [{"name": "x", "serial": "y"}]) + _check_aio_device_presence(gone) + back = self._node_with_inventory([dev], [{"name": "sdb", "serial": "S1"}]) + self.assertEqual(_check_aio_device_presence(back), []) + self.assertNotIn(dev.get_id(), device_monitor._aio_absent) + + def test_inventory_failure_freezes(self): + dev = _aio_dev() + node = _node([dev]) + node.client.side_effect = RuntimeError("agent down") + for _ in range(constants.AIO_DEVICE_ABSENT_POLLS + 2): + self.assertEqual(_check_aio_device_presence(node), []) + self.assertNotIn(dev.get_id(), device_monitor._aio_absent) + + def test_no_aio_devices_no_inventory_call(self): + dev = _aio_dev(bdev_type="nvme") + node = _node([dev]) + self.assertEqual(_check_aio_device_presence(node), []) + node.client.assert_not_called() + + +class TestSweepActions(WatchdogBase): + + def _sweep(self, node, stalled=None, gone=None): + with patch.object(device_monitor, "_check_aio_device_presence", + return_value=gone or []), \ + patch.object(device_monitor, "_check_aio_hung_io", + return_value=stalled or []), \ + patch.object(device_monitor, "device_controller") as dc, \ + patch.object(device_monitor, "tasks_controller") as tc: + _sweep_aio_devices(node) + return dc, tc + + def test_single_stalled_marks_unavailable_with_countable_cause(self): + dev = _aio_dev() + node = _node([dev]) + dc, tc = self._sweep(node, stalled=[dev]) + dc.device_set_io_error.assert_called_once_with(dev.get_id(), True) + dc.device_set_unavailable.assert_called_once_with( + dev.get_id(), cause=device_monitor.CAUSE_LOCAL_FAILURE) + tc.add_node_to_auto_restart.assert_not_called() + + def test_two_stalled_escalates_to_node_restart(self): + d1, d2 = _aio_dev("dev-1", serial="S1"), _aio_dev("dev-2", serial="S2") + node = _node([d1, d2]) + dc, tc = self._sweep(node, stalled=[d1, d2]) + tc.add_node_to_auto_restart.assert_called_once_with(node) + dc.device_set_unavailable.assert_not_called() + + def test_gone_device_removed_with_countable_cause(self): + dev = _aio_dev() + node = _node([dev]) + dc, _ = self._sweep(node, gone=[dev]) + dc.device_remove.assert_called_once_with( + dev.get_id(), cause=device_monitor.CAUSE_LOCAL_FAILURE) + + def test_stall_tracking_cleared_after_action(self): + dev = _aio_dev() + device_monitor._aio_progress[dev.get_id()] = (100, 3) + node = _node([dev]) + self._sweep(node, stalled=[dev]) + self.assertNotIn(dev.get_id(), device_monitor._aio_progress) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/web/api/v2/test_storage_node_endpoints.py b/tests/unit/web/api/v2/test_storage_node_endpoints.py index 859ca945f3..f383f33374 100644 --- a/tests/unit/web/api/v2/test_storage_node_endpoints.py +++ b/tests/unit/web/api/v2/test_storage_node_endpoints.py @@ -56,6 +56,7 @@ def test_creates_add_node_task(self, client, db, cluster, tasks_controller): 'spdk_sys_mem': None, 'failure_domain': None, 'expansion': False, + 'force_format': False, }) # Default response format is 'identifier': body is the task id assert response.json() == TASK_ID From 193ed7ef2e4029f4a53e1778044db88f9e1a441a Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 5 Aug 2026 12:56:18 +0200 Subject: [PATCH 2/9] Fix lblk sys-memory sizing: 0.2% of capacity, not full capacity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first AWS lblk deploy (2x50G EBS per 32 GiB host) failed every `sn configure --lblk` with "Free memory ... less than required 127228418457": node_config_min_sys_memory charged 2 GiB + the FULL device capacity. The nvme path nominally does the same but always measures zero — capacity is read via `nvme list` after the devices were unbound from the kernel driver — so the de-facto contract (and the documented intent, "plus 0.2% of the storage") is a small fraction. Apply the documented 0.2% factor for lblk. Co-Authored-By: Claude Fable 5 --- simplyblock_core/utils/__init__.py | 18 +++++++++++++++--- tests/unit/test_lblk_eligibility.py | 10 +++++++--- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index bfa6797fbf..f8bc776185 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -1795,12 +1795,24 @@ def node_config_device_count(node) -> int: return len(node.get("lblk_devices") or []) or len(node.get("ssd_pcis") or []) +# Sys-memory sizing intent (see generate_automated_deployment_config): +# "RAM 4GB min. Plus 0.2% of the storage." The nvme path nominally adds the +# FULL device capacity but in practice always measures 0 — capacity is read +# via `nvme list` AFTER the devices were unbound from the kernel driver. The +# lblk path knows the real sizes, so it applies the documented 0.2% factor +# (2026-08-05 AWS run: summing full capacity demanded 102 GiB sys memory for +# 2x50G EBS volumes on 32 GiB hosts and failed every `sn configure --lblk`). +SYS_MEMORY_STORAGE_FACTOR = 0.002 + + def node_config_min_sys_memory(node) -> int: - """Minimum system memory for a node-config entry: 2 GiB + total device - capacity. lblk entries carry their sizes; nvme goes through nvme-cli.""" + """Minimum system memory for a node-config entry: 2 GiB + 0.2% of total + device capacity. lblk entries carry their sizes; nvme goes through + nvme-cli.""" lblk = node.get("lblk_devices") or [] if lblk: - total = 2147483648 + sum(int(e.get("size") or 0) for e in lblk) + capacity = sum(int(e.get("size") or 0) for e in lblk) + total = 2147483648 + int(capacity * SYS_MEMORY_STORAGE_FACTOR) logger.debug(f"Minimum system memory is {humanbytes(total)}") return int(total) return calculate_minimum_sys_memory(node.get("ssd_pcis") or []) diff --git a/tests/unit/test_lblk_eligibility.py b/tests/unit/test_lblk_eligibility.py index 6b6a8fe723..f1fc1db984 100644 --- a/tests/unit/test_lblk_eligibility.py +++ b/tests/unit/test_lblk_eligibility.py @@ -247,9 +247,13 @@ def test_device_count_nvme(self): def test_device_count_missing_keys(self): self.assertEqual(utils.node_config_device_count({}), 0) - def test_min_sys_memory_lblk_sums_sizes(self): - node = {"lblk_devices": [{"size": 10}, {"size": 32}]} - self.assertEqual(utils.node_config_min_sys_memory(node), 2147483648 + 42) + def test_min_sys_memory_lblk_uses_capacity_factor(self): + node = {"lblk_devices": [{"size": 50 << 30}, {"size": 50 << 30}]} + expected = 2147483648 + int((100 << 30) * utils.SYS_MEMORY_STORAGE_FACTOR) + self.assertEqual(utils.node_config_min_sys_memory(node), expected) + # ~2.2 GiB total — NOT 2 GiB + full capacity (the bug the first AWS + # lblk deploy hit: 102 GiB demanded on 32 GiB hosts). + self.assertLess(utils.node_config_min_sys_memory(node), 3 << 30) def test_min_sys_memory_nvme_delegates(self): node = {"ssd_pcis": ["0000:00:1e.0"], "lblk_devices": []} From 457450217cd0d5c117d2ce5805f2e748043ae9ad Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 5 Aug 2026 15:30:58 +0200 Subject: [PATCH 3/9] Fix zombie-SPDK cleanup on add-node failure + add JM-mesh activation gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two control-plane gaps exposed by the 2026-08-05 lblk soak bring-up (both generic, neither lblk-specific): 1. Zombie SPDK on failure cleanup. When add-node/restart aborts with "node did not come up", _kill_spdk_until_dead verified death via spdk_process_is_up — an RPC-Unix-socket probe that false-negatives an SPDK which booted but never brought its RPC up. Combined with spdk_process_kill's deliberately detached container remove (fast peer termination) losing the race against the container restart policy, the "confirmed down" SPDK survived, squatting ~all hugepages and starving every subsequent add-node retry on the host. New agent endpoint spdk_process_cleanup: clears the restart policy, removes synchronously, and reports success only when the containers are verifiably GONE (k8s agent: alias of its already-synchronous pod-delete-and-poll). _kill_spdk_until_dead prefers it and falls back to the legacy kill + socket-poll for older agents. 2. JM-mesh activation gate. Nodes that joined through add-node retries ended with peers missing their remote_jm_* controllers; the cluster activated and reported healthy while a third of the journal mesh was unreachable. First journal load excluded those JMs, n_safe_jms collapsed and JCERR cascaded cluster-wide. New storage_node_ops.verify_jm_mesh_coverage(): every ONLINE node must hold live remote bdevs for the remote JMs it references, with a one-shot _connect_to_remote_jm_devs repair. Wired into _cluster_activate: FRESH activation fails on unrepaired holes; RE-ACTIVATION is a recovery path that may legitimately run with one or two nodes unhealthy — the verifier skips JMs whose owner is not ONLINE and the gate only warns. tests: tests/unit/test_jm_mesh_and_spdk_cleanup.py (11 cases); full unit tier 1000 green, ruff clean. Co-Authored-By: Claude Fable 5 --- simplyblock_core/cluster_ops.py | 21 ++ simplyblock_core/snode_client.py | 9 + simplyblock_core/storage_node_ops.py | 112 ++++++++++ .../api/internal/storage_node/docker.py | 64 ++++++ .../api/internal/storage_node/kubernetes.py | 14 ++ tests/unit/test_jm_mesh_and_spdk_cleanup.py | 199 ++++++++++++++++++ 6 files changed, 419 insertions(+) create mode 100644 tests/unit/test_jm_mesh_and_spdk_cleanup.py diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index aa53de7a9e..fb02d3d02e 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -1577,6 +1577,27 @@ def _set_node_ana(node_id) -> None: # want headroom for an unplanned failure concurrent with a rollout.) utils.set_storage_mcp_max_unavailable(cl_id, cluster.max_fault_tolerance) + # JM mesh gate (2026-08-05 incident: nodes joined via add-node retries + # activated with peers missing their remote_jm controllers — the cluster + # reported healthy while a third of the journal mesh was unreachable, + # and the first journal load collapsed n_safe_jms into a cluster-wide + # JCERR). FRESH activation must not complete over such a hole; a + # RE-ACTIVATION is a recovery path that may legitimately run with one + # or two nodes unhealthy, so it repairs best-effort and only warns — + # the verifier already skips JMs whose owner node is not ONLINE. + if cluster.ha_type == "ha": + jm_problems = storage_node_ops.verify_jm_mesh_coverage(cl_id, repair=True) + if jm_problems: + if is_fresh_activation: + set_cluster_status(cl_id, ols_status) + raise ValueError( + "Failed to activate cluster: JM mesh coverage incomplete " + "(journal quorum would silently run degraded): " + + "; ".join(jm_problems)) + logger.warning( + "JM mesh coverage incomplete on re-activation (continuing — " + "recovery path): %s", "; ".join(jm_problems)) + set_cluster_status(cl_id, Cluster.STATUS_ACTIVE) logger.info("Cluster activated successfully") diff --git a/simplyblock_core/snode_client.py b/simplyblock_core/snode_client.py index 247e0e6e61..17783ee558 100644 --- a/simplyblock_core/snode_client.py +++ b/simplyblock_core/snode_client.py @@ -187,6 +187,15 @@ def join_swarm(self, cluster_ip, join_token, db_connection, cluster_id): def spdk_process_kill(self, rpc_port, cluster_id=None): return self._request("GET", "spdk_process_kill", {"rpc_port": rpc_port, "cluster_id": cluster_id}) + def spdk_process_cleanup(self, rpc_port, cluster_id=None): + """Slow, authoritative SPDK teardown: restart policy cleared, remove + synchronous, success only when the containers/pod are verifiably + GONE. Use on failure-cleanup paths (spdk_process_kill is the fast + peer-termination sibling whose detached remove can lose against a + restart policy).""" + return self._request("GET", "spdk_process_cleanup", + {"rpc_port": rpc_port, "cluster_id": cluster_id}) + def leave_swarm(self): return True # return self._request("GET", "leave_swarm") diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 5cb74ef8cd..5114f6b9b1 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -170,6 +170,27 @@ def _kill_spdk_until_dead(snode: StorageNode, max_attempts=3, poll_per_attempt_s # poll_per_attempt_sec of CPU per attempt. rounds_per_attempt = max(1, int(poll_per_attempt_sec / poll_interval)) for attempt in range(1, max_attempts + 1): + # Prefer the authoritative container-level cleanup: it clears the + # restart policy and removes synchronously, so success means the + # container is verifiably GONE — not merely "RPC socket down". + # spdk_process_is_up probes the RPC Unix socket, which false- + # negatives an SPDK that booted but never brought its RPC up; its + # zombie container then squats the host's hugepages and starves + # every subsequent add/restart attempt (2026-08-05 incident, and + # the resurrection race: kill's detached remove vs restart policy). + try: + ret, _err = snode_api.spdk_process_cleanup(snode.rpc_port, snode.cluster_id) + if ret: + logger.info( + "SPDK on %s cleaned up (container-level, attempt %d/%d)", + snode.get_id(), attempt, max_attempts, + ) + return True + except Exception as e: + # Older agent without the endpoint, or transient failure — + # fall back to the legacy kill + socket-poll below. + logger.debug("spdk_process_cleanup unavailable on %s: %s", + snode.get_id(), e) try: snode_api.spdk_process_kill(snode.rpc_port, snode.cluster_id) except Exception as e: @@ -2023,6 +2044,97 @@ def _peer_reachable_via_jm_quorum(target_node_id, this_node: StorageNode, peer_p return not probed +def verify_jm_mesh_coverage(cluster_id, repair=True): + """Verify the JC journal mesh: every ONLINE node must hold a live remote + bdev for every remote JM it references (``jm_ids``) whose OWNER node is + itself ONLINE. Returns a list of problem strings (empty = healthy). + + Rationale (2026-08-05 incident): two nodes joined through add-node + retries and the peers never attached their ``remote_jm_*`` controllers; + the cluster activated and reported healthy while a third of the journal + mesh was unreachable. First journal load excluded those JMs, n_safe_jms + collapsed and JCERR cascaded cluster-wide. This check is the activation + gate for exactly that hole. + + ``repair=True`` re-runs _connect_to_remote_jm_devs once for nodes with + missing coverage before reporting. JMs whose owner is not ONLINE are + skipped — a re-activation with one or two unhealthy nodes must never be + blocked by their (legitimately absent) journals. + """ + db_controller = DBController() + nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id) + jm_owner_by_id = {} + for n in nodes: + if n.jm_device and n.jm_device.get_id(): + jm_owner_by_id[n.jm_device.get_id()] = n + + problems = [] + for node in nodes: + if node.status != StorageNode.STATUS_ONLINE or not node.enable_ha_jm: + continue + expected = {} + for entry in (node.remote_jm_devices or []): + owner_id = entry.node_id + if entry.remote_bdev: + expected[owner_id] = entry.remote_bdev + missing = [] + for jm_id in (node.jm_ids or []): + owner = jm_owner_by_id.get(jm_id) + if owner is None or owner.get_id() == node.get_id(): + continue + if owner.status != StorageNode.STATUS_ONLINE: + continue # recovery tolerance: absent owner, absent journal + remote_bdev = expected.get(owner.get_id()) + if not remote_bdev: + missing.append((jm_id, owner.get_id(), "")) + rpc_client = node.rpc_client(timeout=10, retry=2) + for owner_id, remote_bdev in expected.items(): + owner = None + for n in nodes: + if n.get_id() == owner_id: + owner = n + break + if owner is not None and owner.status != StorageNode.STATUS_ONLINE: + continue + try: + present = bool(rpc_client.get_bdevs(remote_bdev)) + except Exception: + present = False + if not present: + missing.append(("", owner_id, remote_bdev)) + + if missing and repair: + logger.warning( + f"JM mesh: node {node.get_id()} missing {len(missing)} remote " + f"JM bdev(s); attempting reconnect") + try: + fresh = db_controller.get_storage_node_by_id(node.get_id()) + fresh.remote_jm_devices = _connect_to_remote_jm_devs(fresh) + fresh.write_to_db(db_controller.kv_store) + still = [] + rpc_client = fresh.rpc_client(timeout=10, retry=2) + for jm_id, owner_id, remote_bdev in missing: + fixed = False + for entry in (fresh.remote_jm_devices or []): + if entry.node_id == owner_id and entry.remote_bdev: + try: + fixed = bool(rpc_client.get_bdevs(entry.remote_bdev)) + except Exception: + fixed = False + break + if not fixed: + still.append((jm_id, owner_id, remote_bdev)) + missing = still + except Exception as e: + logger.error(f"JM mesh repair failed for {node.get_id()}: {e}") + + for jm_id, owner_id, remote_bdev in missing: + problems.append( + f"node {node.get_id()}: unreachable remote JM of node " + f"{owner_id} (bdev {remote_bdev})") + return problems + + def _connect_to_remote_jm_devs(this_node: StorageNode, jm_ids=None, only_node_id=None): """Connect ``this_node`` to remote JM devices and return the refreshed remote-JM records. diff --git a/simplyblock_web/api/internal/storage_node/docker.py b/simplyblock_web/api/internal/storage_node/docker.py index ee5f62c325..c591e968e4 100644 --- a/simplyblock_web/api/internal/storage_node/docker.py +++ b/simplyblock_web/api/internal/storage_node/docker.py @@ -302,6 +302,70 @@ def _remove_one(container): return utils.get_response(True) +@api.get('/spdk_process_cleanup', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'boolean' + })}}}, +}) +def spdk_process_cleanup(query: utils.RPCPortParams): + """Synchronous, resurrection-proof teardown of ``spdk_`` and its + proxy, VERIFIED at the container level. + + spdk_process_kill is deliberately fast (detached remove) for the peer- + termination paths — but that leaves two gaps for the add-node/restart + FAILURE cleanup: the containers run with a restart policy that can + resurrect them after the SIGKILL if the detached remove loses the race + against a loaded dockerd, and spdk_process_is_up probes the RPC Unix + socket, so an SPDK that never brought its RPC up reads as "down" while + its container lives on holding all hugepages (2026-08-05 incident: the + zombie starved every add-node retry on the host). This endpoint is the + slow, authoritative sibling: disable the restart policy first, remove + synchronously, and only report success when the containers are GONE. + """ + from docker.errors import NotFound + + client = get_docker_client() + names = [f"/spdk_{query.rpc_port}", f"/spdk_proxy_{query.rpc_port}"] + ok = True + for name in names: + try: + container = client.containers.get(name) + except NotFound: + continue + except Exception as exc: + logger.error("cleanup: resolving %s failed: %s", name, exc) + ok = False + continue + try: + # No restart policy => dockerd cannot resurrect it between the + # kill and the (synchronous) remove below. + client.api.update_container(container.id, + restart_policy={"Name": "no"}) + except Exception as exc: + logger.warning("cleanup: clearing restart policy on %s failed: %s", + container.id[:12], exc) + try: + container.remove(force=True) + except NotFound: + pass + except Exception as exc: + logger.error("cleanup: remove(%s) failed: %s", container.id[:12], exc) + ok = False + # Verification: success means the names resolve to nothing. + for name in names: + try: + client.containers.get(name) + ok = False + logger.error("cleanup: %s still present after remove", name) + except NotFound: + pass + except Exception: + ok = False + if not ok: + return utils.get_response(None, "spdk container cleanup incomplete") + return utils.get_response(True) + + # Tight client timeout for the dockerd fall-through in spdk_process_is_up. # The docker-py default is 60s, which under post-outage Swarm reconciliation # (incident 2026-04-24, vm205) caused this endpoint to take 76-80s. The diff --git a/simplyblock_web/api/internal/storage_node/kubernetes.py b/simplyblock_web/api/internal/storage_node/kubernetes.py index 6642db0831..c501118f4c 100644 --- a/simplyblock_web/api/internal/storage_node/kubernetes.py +++ b/simplyblock_web/api/internal/storage_node/kubernetes.py @@ -617,6 +617,20 @@ def spdk_process_kill(query: utils.RPCPortParams): return utils.get_response(True) +@api.get('/spdk_process_cleanup', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'boolean' + })}}}, +}) +def spdk_process_cleanup(query: utils.RPCPortParams): + """Authoritative SPDK teardown for failure-cleanup paths. Pod deletion in + this deployment mode is already synchronous and verified (see + spdk_process_kill's poll-until-gone), so this is an alias kept for parity + with the docker agent, where kill (fast, detached remove) and cleanup + (slow, verified remove) are distinct.""" + return spdk_process_kill(query) + + def _is_pod_up(rpc_port, cluster_id): k8s_core_v1 = core_utils.get_k8s_core_client() pod_name = f"snode-spdk-pod-{rpc_port}-{cluster_id}" diff --git a/tests/unit/test_jm_mesh_and_spdk_cleanup.py b/tests/unit/test_jm_mesh_and_spdk_cleanup.py new file mode 100644 index 0000000000..079043ee17 --- /dev/null +++ b/tests/unit/test_jm_mesh_and_spdk_cleanup.py @@ -0,0 +1,199 @@ +# coding=utf-8 +"""Unit tests for the two 2026-08-05 incident fixes: + +1. verify_jm_mesh_coverage — the activation JM-mesh gate: every ONLINE + node must hold live remote bdevs for the remote JMs it references, + with owner-offline tolerance (re-activation with unhealthy nodes must + never be blocked) and a one-shot reconnect repair. + +2. _kill_spdk_until_dead — failure-path SPDK teardown must prefer the + container-level spdk_process_cleanup (verified-gone semantics) over + the RPC-socket liveness probe that false-negatives a booted-but- + RPC-dead SPDK (the hugepage-squatting zombie that starved add-node + retries). +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import storage_node_ops +from simplyblock_core.models.nvme_device import JMDevice, RemoteJMDevice +from simplyblock_core.models.storage_node import StorageNode + + +def _node(uuid, status=StorageNode.STATUS_ONLINE, enable_ha_jm=True, + jm_dev_id=None, jm_ids=(), remote_jms=()): + n = StorageNode() + n.uuid = uuid + n.status = status + n.enable_ha_jm = enable_ha_jm + n.cluster_id = "cluster-1" + if jm_dev_id: + jm = JMDevice() + jm.uuid = jm_dev_id + n.jm_device = jm + n.jm_ids = list(jm_ids) + n.remote_jm_devices = list(remote_jms) + return n + + +def _rjm(owner_id, remote_bdev): + r = RemoteJMDevice() + r.node_id = owner_id + r.remote_bdev = remote_bdev + r.jm_bdev = f"jm_{owner_id}" + return r + + +class TestJmMeshCoverage(unittest.TestCase): + + def _run(self, nodes, bdevs_by_node, repair=False, reconnect_result=None): + """bdevs_by_node: {node_uuid: set(bdev names present on that node)}""" + db = MagicMock() + db.get_storage_nodes_by_cluster_id.return_value = nodes + db.get_storage_node_by_id.side_effect = lambda nid: next( + n for n in nodes if n.get_id() == nid) + + def _rpc_for(node_self, **kw): + rpc = MagicMock() + present = bdevs_by_node.get(node_self.get_id(), set()) + rpc.get_bdevs.side_effect = lambda name: ( + [{"name": name}] if name in present else None) + return rpc + + patches = [ + patch.object(storage_node_ops, "DBController", return_value=db), + patch.object(StorageNode, "rpc_client", _rpc_for), + ] + if reconnect_result is not None: + patches.append(patch.object( + storage_node_ops, "_connect_to_remote_jm_devs", + return_value=reconnect_result)) + for p in patches: + p.start() + try: + return storage_node_ops.verify_jm_mesh_coverage("cluster-1", repair=repair) + finally: + for p in patches: + p.stop() + + def _two_nodes(self): + a = _node("node-a", jm_dev_id="jm-a", jm_ids=["jm-a", "jm-b"], + remote_jms=[_rjm("node-b", "remote_jm_node-bn1")]) + b = _node("node-b", jm_dev_id="jm-b", jm_ids=["jm-b", "jm-a"], + remote_jms=[_rjm("node-a", "remote_jm_node-an1")]) + return a, b + + def test_healthy_mesh(self): + a, b = self._two_nodes() + problems = self._run([a, b], { + "node-a": {"remote_jm_node-bn1"}, + "node-b": {"remote_jm_node-an1"}, + }) + self.assertEqual(problems, []) + + def test_missing_remote_bdev_reported(self): + a, b = self._two_nodes() + problems = self._run([a, b], { + "node-a": set(), # a cannot see b's JM + "node-b": {"remote_jm_node-an1"}, + }) + self.assertEqual(len(problems), 1) + self.assertIn("node-a", problems[0]) + self.assertIn("node-b", problems[0]) + + def test_missing_record_reported(self): + # node-a references jm-b in jm_ids but has NO remote record at all — + # the exact 2026-08-05 hole. + a = _node("node-a", jm_dev_id="jm-a", jm_ids=["jm-a", "jm-b"], + remote_jms=[]) + b = _node("node-b", jm_dev_id="jm-b", jm_ids=["jm-b"]) + problems = self._run([a, b], {"node-a": set(), "node-b": set()}) + self.assertTrue(any("node-a" in p and "node-b" in p for p in problems)) + + def test_offline_owner_tolerated(self): + # Re-activation rule: a JM whose owner is not ONLINE is skipped. + a, b = self._two_nodes() + b.status = StorageNode.STATUS_OFFLINE + problems = self._run([a, b], {"node-a": set()}) + self.assertEqual(problems, []) + + def test_offline_referencing_node_skipped(self): + a, b = self._two_nodes() + a.status = StorageNode.STATUS_OFFLINE + problems = self._run([a, b], { + "node-b": {"remote_jm_node-an1"}, + }) + self.assertEqual(problems, []) + + def test_repair_fixes_coverage(self): + a, b = self._two_nodes() + # Initially missing on node-a; after reconnect the returned record's + # bdev IS present. + problems = self._run( + [a, b], + {"node-a": {"remote_jm_node-bn1_new"}, + "node-b": {"remote_jm_node-an1"}}, + repair=True, + reconnect_result=[_rjm("node-b", "remote_jm_node-bn1_new")], + ) + self.assertEqual(problems, []) + + def test_repair_failure_still_reported(self): + a, b = self._two_nodes() + problems = self._run( + [a, b], + {"node-a": set(), "node-b": {"remote_jm_node-an1"}}, + repair=True, + reconnect_result=[_rjm("node-b", "remote_jm_node-bn1")], + ) + self.assertEqual(len(problems), 1) + + +class TestKillSpdkUntilDead(unittest.TestCase): + + def _snode(self): + snode = MagicMock() + snode.get_id.return_value = "node-1" + snode.rpc_port = 4423 + snode.cluster_id = "cluster-1" + snode.mgmt_ip = "10.0.0.1" + return snode + + def test_cleanup_success_short_circuits(self): + snode = self._snode() + api = snode.client.return_value + api.spdk_process_cleanup.return_value = (True, None) + self.assertTrue(storage_node_ops._kill_spdk_until_dead(snode)) + api.spdk_process_cleanup.assert_called_once_with(4423, "cluster-1") + api.spdk_process_kill.assert_not_called() + + def test_cleanup_unavailable_falls_back_to_kill(self): + snode = self._snode() + api = snode.client.return_value + api.spdk_process_cleanup.side_effect = Exception("404 not found") + api.spdk_process_is_up.return_value = (False, None) + self.assertTrue(storage_node_ops._kill_spdk_until_dead( + snode, max_attempts=1, poll_per_attempt_sec=1)) + api.spdk_process_kill.assert_called_once() + + def test_cleanup_incomplete_falls_back(self): + snode = self._snode() + api = snode.client.return_value + api.spdk_process_cleanup.return_value = (None, "cleanup incomplete") + api.spdk_process_is_up.return_value = (False, None) + self.assertTrue(storage_node_ops._kill_spdk_until_dead( + snode, max_attempts=1, poll_per_attempt_sec=1)) + api.spdk_process_kill.assert_called_once() + + def test_all_paths_fail_returns_false(self): + snode = self._snode() + api = snode.client.return_value + api.spdk_process_cleanup.return_value = (None, "nope") + api.spdk_process_is_up.return_value = (True, None) + self.assertFalse(storage_node_ops._kill_spdk_until_dead( + snode, max_attempts=1, poll_per_attempt_sec=0)) + + +if __name__ == "__main__": + unittest.main() From ccc6fdfe84b5224e1faa6a479374063891590c08 Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 6 Aug 2026 16:49:17 +0200 Subject: [PATCH 4/9] Switch SPDK fork image to ultra:md-journal-latest md-journal test campaigns now run against the fork's md-journal branch (blobstore metadata journal work) instead of ultra main. Co-Authored-By: Claude Fable 5 --- simplyblock_core/env_var | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplyblock_core/env_var b/simplyblock_core/env_var index 5abdecd440..0ad369b78b 100644 --- a/simplyblock_core/env_var +++ b/simplyblock_core/env_var @@ -2,4 +2,4 @@ SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev SIMPLY_BLOCK_VERSION=19.2.34 SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:main -SIMPLY_BLOCK_SPDK_ULTRA_IMAGE=public.ecr.aws/simply-block/ultra:main-latest +SIMPLY_BLOCK_SPDK_ULTRA_IMAGE=public.ecr.aws/simply-block/ultra:md-journal-latest From 70a89b332a2c4bfd976ac1b1862ad62854a7adae Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 14 Aug 2026 15:18:57 +0200 Subject: [PATCH 5/9] Add single-node (non-HA) cluster support: single journal, one-node lvol ops 1-node clusters deploy without HA journaling and activate as non-HA regardless of the chosen ha_type / EC schema: - add_node forces enable_ha_jm=False on is_single_node clusters (deterministic jm_vuid=1 / LVS_1 single local journal, no fabric export). - Activation treats a cluster with exactly one online node as non-HA: role assignment and the JM-mesh gate are skipped even with ha_type=ha, fresh nodes are forced to the single-journal shape, and the +1 spare-device requirement is dropped (a 2-unit node can activate EC 1+0). - Physical labels stay unused: the activation label rewrite now also re-syncs the per-device label copies the distrib cluster map reads. - Lvol lifecycle ops run on exactly one node: HA lvol creation downgrades to ha_type=single on hosts without a secondary (the unguarded secondary lookup previously 500'd on 2+-node non-HA clusters and only worked by accident on 1-node ones); empty role ids are never emitted into lvol.nodes; snapshot phase-2 sync deletes and snapshot health checks are gated on the lvol's ha_type instead of node topology. 47 new unit tests in test_single_node_cluster.py (shared with the lblk partition tests); unit tier 1310 green. --- simplyblock_core/cluster_ops.py | 37 +++- .../controllers/health_controller.py | 2 +- .../controllers/lvol_controller.py | 44 +++- .../controllers/snapshot_controller.py | 11 +- simplyblock_core/services/snapshot_monitor.py | 33 ++- simplyblock_core/storage_node_ops.py | 14 ++ tests/unit/test_single_node_cluster.py | 201 ++++++++++++++++++ 7 files changed, 310 insertions(+), 32 deletions(-) create mode 100644 tests/unit/test_single_node_cluster.py diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index fb02d3d02e..8d53512ce2 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -936,6 +936,21 @@ def _cluster_activate_impl(cl_id, force=False, force_lvstore_create=False) -> No raise +def is_single_node_activation(cluster, online_nodes) -> bool: + """A cluster with exactly one storage node activates as non-HA regardless + of the chosen ha_type / EC schema: no secondary roles, no HA journaling + (single local journal), no physical labels. This is what makes 1-node + deployments activatable with the API defaults (ha_type='ha').""" + return bool(cluster.is_single_node or len(online_nodes) == 1) + + +def activation_minimum_devices(cluster, single_node_cluster) -> int: + """ndcs+npcs devices are needed for placement; the +1 spare is rebuild + headroom that a single-node cluster (no data redundancy to rebuild onto + a spare) does not require.""" + return cluster.distr_ndcs + cluster.distr_npcs + (0 if single_node_cluster else 1) + + def _cluster_activate(cl_id, force=False, force_lvstore_create=False) -> None: cluster = db_controller.get_cluster_by_id(cl_id) @@ -978,7 +993,12 @@ def _cluster_activate(cl_id, force=False, force_lvstore_create=False) -> None: if dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE]: dev_count += 1 - minimum_devices = cluster.distr_ndcs + cluster.distr_npcs + 1 + single_node_cluster = is_single_node_activation(cluster, online_nodes) + if single_node_cluster and cluster.ha_type == "ha": + logger.warning("Single-node cluster: activating as non-HA " + "(no secondary nodes, single journal) regardless of ha_type") + + minimum_devices = activation_minimum_devices(cluster, single_node_cluster) if dev_count < minimum_devices: set_cluster_status(cl_id, ols_status) raise ValueError(f"Failed to activate cluster, No enough online device.. Minimum is {minimum_devices}") @@ -1070,6 +1090,17 @@ def _fd_fail(msg: str) -> None: node.physical_label = 0 else: node.physical_label = storage_node_ops.get_next_physical_device_order(node) + # Keep the per-device label copies in sync — the distrib cluster map + # emits dev.physical_label, not the node's, so a stale non-zero copy + # from node-add would re-enable label anti-affinity in the data plane. + for dev in node.nvme_devices: + dev.physical_label = node.physical_label + if single_node_cluster and node.enable_ha_jm and not node.lvstore: + # Fresh node in a single-node cluster: force the single-journal + # shape before the LVS is created (jm_vuid=1, no remote JMs). A + # node that already carries an lvstore keeps its shape. + logger.info(f"Single-node cluster: disabling HA journaling on node {node.get_id()}") + node.enable_ha_jm = False node.write_to_db() records = db_controller.get_cluster_capacity(cluster) @@ -1078,7 +1109,7 @@ def _fd_fail(msg: str) -> None: used_nodes_as_sec: t.List[str] = [] used_nodes_as_tertiary: t.List[str] = [] snodes = db_controller.get_storage_nodes_by_cluster_id(cl_id) - if cluster.ha_type == "ha": + if cluster.ha_type == "ha" and not single_node_cluster: for snode in snodes: # Do not assign secondary to removed node if snode.status == StorageNode.STATUS_REMOVED: @@ -1585,7 +1616,7 @@ def _set_node_ana(node_id) -> None: # RE-ACTIVATION is a recovery path that may legitimately run with one # or two nodes unhealthy, so it repairs best-effort and only warns — # the verifier already skips JMs whose owner node is not ONLINE. - if cluster.ha_type == "ha": + if cluster.ha_type == "ha" and not single_node_cluster: jm_problems = storage_node_ops.verify_jm_mesh_coverage(cl_id, repair=True) if jm_problems: if is_fresh_activation: diff --git a/simplyblock_core/controllers/health_controller.py b/simplyblock_core/controllers/health_controller.py index 543ef0ac62..9934008120 100644 --- a/simplyblock_core/controllers/health_controller.py +++ b/simplyblock_core/controllers/health_controller.py @@ -1005,7 +1005,7 @@ def check_snap(snap_id): snode = db_controller.get_storage_node_by_id(snap.lvol.node_id) check_primary = snode.rpc_client().get_bdevs(snap.snap_bdev) logger.info(f"Checking snap bdev: {snap.snap_bdev} on node: {snap.lvol.node_id} is {bool(check_primary)}") - if snode.secondary_node_id: + if snap.lvol.ha_type != "single" and snode.secondary_node_id: secondary_node = db_controller.get_storage_node_by_id(snode.secondary_node_id) check_secondary = secondary_node.rpc_client().get_bdevs(snap.snap_bdev) logger.info(f"Checking snap bdev: {snap.snap_bdev} on node: {snode.secondary_node_id} is {bool(check_secondary)}") diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index bdcbfc9230..b99e6f8138 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -404,6 +404,26 @@ def check_lvstore_object_limit(host_node, all_lvols, all_snaps, new_objects=1): return None +def resolve_effective_ha_type(ha_type, host_node): + """The ha_type an lvol can actually be created with on ``host_node``. + + A host without a secondary (single-node / non-HA cluster) cannot serve an + HA lvol — every lifecycle op must then run on exactly one node. Downgrade + instead of failing: cluster ha_type defaults to "ha" even on deployments + that never assign secondaries.""" + if ha_type == "ha" and not host_node.secondary_node_id: + return "single" + return ha_type + + +def role_secondary_ids(host_node): + """The host's non-empty secondary/tertiary node ids, in role order. + Non-HA topologies have none; never emit empty-string ids into + ``lvol.nodes`` (every ``lvol.nodes[1:]`` consumer would iterate them).""" + return [i for i in (host_node.secondary_node_id, + host_node.tertiary_node_id) if i] + + def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp=False, use_crypto=False, distr_vuid=0, max_rw_iops=0, max_rw_mbytes=0, max_r_mbytes=0, max_w_mbytes=0, with_snapshot=False, max_size=0, lvol_priority_class=0, @@ -619,14 +639,20 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp= logger.error(error) return False, error - s_node = db_controller.get_storage_node_by_id(host_node.secondary_node_id) + effective_ha_type = resolve_effective_ha_type(ha_type, host_node) + if effective_ha_type != ha_type: + logger.info(f"Host node {host_node.get_id()} has no secondary node; " + f"creating lvol with ha_type=single") + ha_type = effective_ha_type + lvol.ha_type = effective_ha_type + attr_name = f"active_{fabric}" - is_active_primary = getattr(host_node, attr_name) - is_active_secondary = getattr(s_node, attr_name) - if not is_active_primary: + if not getattr(host_node, attr_name): return False, f"Primary node fabric {fabric} is not active" - if not is_active_secondary: - return False, f"Secondary node fabric {fabric} is not active" + if ha_type == "ha": + s_node = db_controller.get_storage_node_by_id(host_node.secondary_node_id) + if not getattr(s_node, attr_name): + return False, f"Secondary node fabric {fabric} is not active" lvol.hostname = host_node.hostname lvol.node_id = host_node.get_id() @@ -766,10 +792,8 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp= execute_on_leader_with_failover, ) - # Build nodes list - secondary_ids = [host_node.secondary_node_id] - if host_node.tertiary_node_id: - secondary_ids.append(host_node.tertiary_node_id) + # Build nodes list (skip empty role ids — non-HA topologies) + secondary_ids = role_secondary_ids(host_node) lvol.nodes = [host_node.get_id()] + secondary_ids all_nodes = [host_node] diff --git a/simplyblock_core/controllers/snapshot_controller.py b/simplyblock_core/controllers/snapshot_controller.py index acb23babe4..075c6a7d56 100644 --- a/simplyblock_core/controllers/snapshot_controller.py +++ b/simplyblock_core/controllers/snapshot_controller.py @@ -615,10 +615,8 @@ def add(lvol_id, snapshot_name, backup=False, lock=True, all_snaps=None, all_lvo host_node = db_controller.get_storage_node_by_id(snode.get_id()) - # Build nodes list with all secondaries - secondary_ids = [host_node.secondary_node_id] - if host_node.tertiary_node_id: - secondary_ids.append(host_node.tertiary_node_id) + # Build nodes list with all secondaries (skip empty role ids) + secondary_ids = lvol_controller.role_secondary_ids(host_node) lvol.nodes = [host_node.get_id()] + secondary_ids # Detect leader via RPC (no status checks) @@ -1396,9 +1394,8 @@ def clone(snapshot_id, clone_name, new_size=0, pvc_name=None, pvc_namespace=None from simplyblock_core.storage_node_ops import check_non_leader_for_operation, queue_for_restart_drain host_node = snode - secondary_ids = [host_node.secondary_node_id] - if host_node.tertiary_node_id: - secondary_ids.append(host_node.tertiary_node_id) + # skip empty role ids — non-HA topologies + secondary_ids = lvol_controller.role_secondary_ids(host_node) lvol.nodes = [host_node.get_id()] + secondary_ids # Detect leader via RPC (no status checks) diff --git a/simplyblock_core/services/snapshot_monitor.py b/simplyblock_core/services/snapshot_monitor.py index 8660839702..accea28ee8 100644 --- a/simplyblock_core/services/snapshot_monitor.py +++ b/simplyblock_core/services/snapshot_monitor.py @@ -38,6 +38,26 @@ def _await_delete_completion(node, bdev_name, wait_sec): return ret +def sync_delete_peer_ids(lvol_ha_type, snode, primary_node_id): + """Node ids owing a phase-2 sync delete: every LVS member other than the + phase-1 node. ha_type=single snapshots were never registered on peers — + deriving the sync-delete set from node topology would send peers deletes + for registrations they never had, so their peer set is empty.""" + secondary_ids = [] + if lvol_ha_type != "single": + if snode.secondary_node_id: + secondary_ids.append(snode.secondary_node_id) + if snode.tertiary_node_id: + secondary_ids.append(snode.tertiary_node_id) + peer_ids = [] + if snode.get_id() != primary_node_id: + peer_ids.append(snode.get_id()) + for sec_id in secondary_ids: + if sec_id != primary_node_id: + peer_ids.append(sec_id) + return peer_ids + + def process_snap_delete_finish(snap, completed_node): """Phase-2 of the delete protocol (sync deletes + DB finalize). @@ -67,17 +87,8 @@ def process_snap_delete_finish(snap, completed_node): # Every LVS member other than the phase-1 node owes a sync delete (the # sync pass clears the peers' lvol registrations; it is per-node and # needs no leadership). - non_leaders = [] - secondary_ids = [] - if snode.secondary_node_id: - secondary_ids.append(snode.secondary_node_id) - if snode.tertiary_node_id: - secondary_ids.append(snode.tertiary_node_id) - if snode.get_id() != primary_node.get_id(): - non_leaders.append(db.get_storage_node_by_id(snode.get_id())) - for sec_id in secondary_ids: - if sec_id != primary_node.get_id(): - non_leaders.append(db.get_storage_node_by_id(sec_id)) + non_leaders = [db.get_storage_node_by_id(peer_id) for peer_id in + sync_delete_peer_ids(snap.lvol.ha_type, snode, primary_node.get_id())] if primary_node.status in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED, StorageNode.STATUS_DOWN]: any_sec_down = any( diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 5114f6b9b1..5dd38df7c1 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -2435,6 +2435,18 @@ def get_required_ha_jm_count(cluster) -> int: return 3 +def resolve_enable_ha_jm(cluster, enable_ha_jm) -> bool: + """Single-node clusters run without HA journaling: one local journal + (deterministic jm_vuid=1 / LVS_1), no fabric export, no remote JMs. This + is a deployment-time property of the cluster, overriding the CLI/API + default of enable_ha_jm=True.""" + if cluster.is_single_node and enable_ha_jm: + logger.info("Single-node cluster: disabling HA journaling for this node " + "(single local journal)") + return False + return enable_ha_jm + + def resolve_ha_jm_count(cluster, ha_jm_count) -> int: required_ha_jm_count = get_required_ha_jm_count(cluster) @@ -2575,6 +2587,8 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, logger.error("Cluster not found: %s", cluster_id) return False + enable_ha_jm = resolve_enable_ha_jm(cluster, enable_ha_jm) + ha_jm_count = resolve_ha_jm_count(cluster, ha_jm_count) # Failure-domain id is mandatory exactly when the cluster has the diff --git a/tests/unit/test_single_node_cluster.py b/tests/unit/test_single_node_cluster.py new file mode 100644 index 0000000000..26e07560cf --- /dev/null +++ b/tests/unit/test_single_node_cluster.py @@ -0,0 +1,201 @@ +# coding=utf-8 +"""Unit tests for single-node (non-HA) cluster support. + +Covered: + - lvol_controller.resolve_effective_ha_type: HA requests downgrade to + single on hosts without a secondary (every lifecycle op must run on + exactly one node), all other combinations pass through. + - lvol_controller.role_secondary_ids: never emits empty role ids into + lvol.nodes (non-HA topologies). + - snapshot_monitor.sync_delete_peer_ids: ha_type=single snapshots get no + peer sync deletes (they were never registered on peers); HA snapshots + get every LVS member except the phase-1 node. + - storage_node_ops.resolve_enable_ha_jm: single-node clusters force the + single-local-journal shape at add-node. + - cluster_ops.is_single_node_activation / activation_minimum_devices: + 1-node clusters activate as non-HA regardless of ha_type and without + the +1 spare-device requirement. + - health_controller.check_snap: secondary probe gated on the lvol's + ha_type, not on node topology. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import cluster_ops, storage_node_ops +from simplyblock_core.controllers import lvol_controller +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import snapshot_monitor + + +def _snode(node_id="node-1", secondary="", tertiary=""): + n = StorageNode() + n.uuid = node_id + n.secondary_node_id = secondary + n.tertiary_node_id = tertiary + return n + + +def _cluster(is_single_node=False, ha_type="ha", ndcs=1, npcs=1): + c = Cluster() + c.uuid = "cluster-1" + c.is_single_node = is_single_node + c.ha_type = ha_type + c.distr_ndcs = ndcs + c.distr_npcs = npcs + return c + + +class TestResolveEffectiveHaType(unittest.TestCase): + + def test_ha_without_secondary_downgrades_to_single(self): + self.assertEqual( + lvol_controller.resolve_effective_ha_type("ha", _snode()), "single") + + def test_ha_with_secondary_stays_ha(self): + self.assertEqual( + lvol_controller.resolve_effective_ha_type("ha", _snode(secondary="sec-1")), + "ha") + + def test_single_stays_single_regardless_of_secondary(self): + self.assertEqual( + lvol_controller.resolve_effective_ha_type("single", _snode(secondary="sec-1")), + "single") + + def test_single_stays_single_without_secondary(self): + self.assertEqual( + lvol_controller.resolve_effective_ha_type("single", _snode()), "single") + + +class TestRoleSecondaryIds(unittest.TestCase): + + def test_no_roles_yields_empty(self): + self.assertEqual(lvol_controller.role_secondary_ids(_snode()), []) + + def test_secondary_only(self): + self.assertEqual( + lvol_controller.role_secondary_ids(_snode(secondary="sec-1")), ["sec-1"]) + + def test_secondary_and_tertiary_in_role_order(self): + self.assertEqual( + lvol_controller.role_secondary_ids( + _snode(secondary="sec-1", tertiary="tert-1")), + ["sec-1", "tert-1"]) + + def test_tertiary_without_secondary_never_emits_empty_string(self): + # A demoted secondary must not leave "" in lvol.nodes. + self.assertEqual( + lvol_controller.role_secondary_ids(_snode(tertiary="tert-1")), + ["tert-1"]) + + +class TestSyncDeletePeerIds(unittest.TestCase): + + def test_single_lvol_same_node_has_no_peers(self): + snode = _snode("node-1", secondary="sec-1", tertiary="tert-1") + self.assertEqual( + snapshot_monitor.sync_delete_peer_ids("single", snode, "node-1"), []) + + def test_single_lvol_ignores_topology_even_across_nodes(self): + # Phase-1 completed elsewhere: the home node still owes its own sync + # delete, but topology peers must NOT be added for a single lvol. + snode = _snode("node-1", secondary="sec-1") + self.assertEqual( + snapshot_monitor.sync_delete_peer_ids("single", snode, "sec-1"), + ["node-1"]) + + def test_ha_lvol_full_member_set_minus_primary(self): + snode = _snode("node-1", secondary="sec-1", tertiary="tert-1") + self.assertEqual( + snapshot_monitor.sync_delete_peer_ids("ha", snode, "node-1"), + ["sec-1", "tert-1"]) + + def test_ha_lvol_phase1_on_secondary(self): + snode = _snode("node-1", secondary="sec-1", tertiary="tert-1") + self.assertEqual( + snapshot_monitor.sync_delete_peer_ids("ha", snode, "sec-1"), + ["node-1", "tert-1"]) + + def test_ha_lvol_without_roles(self): + snode = _snode("node-1") + self.assertEqual( + snapshot_monitor.sync_delete_peer_ids("ha", snode, "node-1"), []) + + +class TestResolveEnableHaJm(unittest.TestCase): + + def test_single_node_cluster_forces_disabled(self): + self.assertFalse(storage_node_ops.resolve_enable_ha_jm( + _cluster(is_single_node=True), True)) + + def test_single_node_cluster_disabled_stays_disabled(self): + self.assertFalse(storage_node_ops.resolve_enable_ha_jm( + _cluster(is_single_node=True), False)) + + def test_multi_node_cluster_passthrough_true(self): + self.assertTrue(storage_node_ops.resolve_enable_ha_jm( + _cluster(is_single_node=False), True)) + + def test_multi_node_cluster_passthrough_false(self): + self.assertFalse(storage_node_ops.resolve_enable_ha_jm( + _cluster(is_single_node=False), False)) + + +class TestSingleNodeActivation(unittest.TestCase): + + def test_flagged_cluster_is_single_node(self): + self.assertTrue(cluster_ops.is_single_node_activation( + _cluster(is_single_node=True), [_snode(), _snode("node-2")])) + + def test_one_online_node_is_single_node_even_unflagged(self): + self.assertTrue(cluster_ops.is_single_node_activation( + _cluster(is_single_node=False), [_snode()])) + + def test_two_nodes_unflagged_is_not_single_node(self): + self.assertFalse(cluster_ops.is_single_node_activation( + _cluster(is_single_node=False), [_snode(), _snode("node-2")])) + + def test_minimum_devices_drops_spare_for_single_node(self): + cluster = _cluster(ndcs=1, npcs=0) + self.assertEqual(cluster_ops.activation_minimum_devices(cluster, True), 1) + self.assertEqual(cluster_ops.activation_minimum_devices(cluster, False), 2) + + def test_minimum_devices_ec21(self): + cluster = _cluster(ndcs=2, npcs=1) + self.assertEqual(cluster_ops.activation_minimum_devices(cluster, True), 3) + self.assertEqual(cluster_ops.activation_minimum_devices(cluster, False), 4) + + +class TestCheckSnapHaGating(unittest.TestCase): + + def _run_check_snap(self, ha_type, secondary_node_id): + from simplyblock_core.controllers import health_controller + snap = MagicMock() + snap.snap_bdev = "LVS_1/SNAP_1" + snap.lvol.node_id = "node-1" + snap.lvol.ha_type = ha_type + + primary = MagicMock() + primary.secondary_node_id = secondary_node_id + primary.rpc_client.return_value.get_bdevs.return_value = [{"name": "x"}] + secondary = MagicMock() + secondary.rpc_client.return_value.get_bdevs.return_value = [{"name": "x"}] + + db = MagicMock() + db.get_snapshot_by_id.return_value = snap + db.get_storage_node_by_id.side_effect = ( + lambda nid: primary if nid == "node-1" else secondary) + with patch.object(health_controller, "DBController", return_value=db): + health_controller.check_snap("snap-1") + return primary, secondary + + def test_single_snap_never_probes_secondary(self): + primary, secondary = self._run_check_snap("single", "sec-1") + self.assertTrue(primary.rpc_client.called) + self.assertFalse(secondary.rpc_client.called) + + def test_ha_snap_probes_secondary(self): + primary, secondary = self._run_check_snap("ha", "sec-1") + self.assertTrue(primary.rpc_client.called) + self.assertTrue(secondary.rpc_client.called) From b242ad0644f8129897cebb23da717e2cb968bec8 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 14 Aug 2026 15:19:13 +0200 Subject: [PATCH 6/9] Add lblk partition support: partitions as storage units, journal via partition split Nodes can now run on partitions instead of full SSDs only (lblk mode): - Inventory (/blockdevices) emits partitions with stable identity: serial derived from the parent disk serial + PARTUUID, by-partuuid stable path, per-partition busy/holder/root detection. Restart resolution falls back to the PARTUUID when the parent serial changed. - Eligibility: partitions must be unmounted (not busy), unheld, non-root, writable; they are never auto-selected (explicit --blk-names/--blk-serials only), and a disk and its own partitions cannot both be selected. - Journal carve-out: partition-backed selections never relabel a whole drive - the smallest selected partition is SPLIT in two at configure time (sgdisk, GPT only, within the original partition bounds): a journal partition (--jm-percent of total capacity, 2 GiB floor, capped at half the split partition) and a data partition in the remainder. The journal entry is flagged in the node config; add-node/restart prefer it over smallest-device selection. - Minimum 2 partitions or SSDs per node, enforced at configure, add-node and config validation (plus at most one journal-flagged entry). - validate_arguments in node_configure tolerates minimal namespaces (pre-existing rebase fallout with main's max-subsystems test). New unit tests: test_lblk_partitions.py (inventory, split orchestration + mechanics, PARTUUID resolution, flagged-journal selection); eligibility tests extended for partition semantics. Unit tier 1310 green, ruff clean. --- simplyblock_cli/cli-reference.yaml | 8 +- simplyblock_cli/cli.py | 5 +- simplyblock_cli/clibase.py | 2 +- simplyblock_core/constants.py | 9 + simplyblock_core/storage_node_ops.py | 57 +++- simplyblock_core/utils/__init__.py | 163 ++++++++++-- simplyblock_web/node_configure.py | 30 ++- simplyblock_web/node_utils.py | 206 ++++++++++++++- tests/unit/test_lblk_eligibility.py | 77 +++++- tests/unit/test_lblk_partitions.py | 372 +++++++++++++++++++++++++++ 10 files changed, 873 insertions(+), 56 deletions(-) create mode 100644 tests/unit/test_lblk_partitions.py diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index ad3772f087..0d86506ea6 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -111,7 +111,7 @@ commands: type: str default: "" - name: "--lblk" - help: "Configure the node with Linux block devices (lblk cluster mode) instead of NVMe PCIe devices: eligible unmounted, unheld, unpartitioned whole disks are wrapped in SPDK AIO bdevs. Select devices with --blk-names, --blk-names-exclude or --blk-serials; without a selector, every eligible disk is used." + help: "Configure the node with Linux block devices (lblk cluster mode) instead of NVMe PCIe devices: eligible whole disks or partitions (unmounted, unheld; disks additionally unpartitioned) are wrapped in SPDK AIO bdevs. Select devices with --blk-names, --blk-names-exclude or --blk-serials; without a selector, every eligible whole disk is used (partitions must be selected explicitly). Minimum 2 partitions or SSDs per node. When the selection contains partitions, the smallest one is split in two at configure time: a journal partition (--jm-percent of total capacity) and a data partition." dest: lblk type: bool action: store_true @@ -133,6 +133,12 @@ commands: required: false type: str default: "" + - name: "--jm-percent" + help: "Journal size in percent of the node's total selected capacity when the journal is carved by splitting a selected partition (requires --lblk with partitions). Default: `3`." + dest: jm_percent + required: false + type: int + default: 3 - name: "--force" help: "Force format detected or passed nvme pci address to 4K and clean partitions. With --lblk: mark partitioned disks eligible; the partition wipe happens at add-node with --force-format." dest: force diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 66ae4efb3d..db232522a8 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -109,10 +109,11 @@ def init_storage_node__configure(self, subparser): subcommand.add_argument('--device-model', help='NVMe SSD model string, example: --model PM1628. Can be used alone to filter by model, or combined with --size-range to further filter by size.', type=str, default='', dest='device_model', required=False) subcommand.add_argument('--size-range', help='NVMe SSD device size range separated by -, can be X(m,g,t) or bytes as integer, example: --size-range 50G-1T or --size-range 1232345-67823987. Can be used alone to filter by size, or combined with --device-model to further filter by model.', type=str, default='', dest='size_range', required=False) subcommand.add_argument('--nvme-names', help='Comma separated list of nvme namespace names like nvme0n1,nvme1n1.', type=str, default='', dest='nvme_names', required=False) - subcommand.add_argument('--lblk', help='Configure the node with Linux block devices (lblk cluster mode) instead of NVMe PCIe devices: eligible unmounted, unheld, unpartitioned whole disks are wrapped in SPDK AIO bdevs. Select devices with --blk-names, --blk-names-exclude or --blk-serials; without a selector, every eligible disk is used.', dest='lblk', action='store_true') - subcommand.add_argument('--blk-names', help='Comma separated list of block device names to use, like sdb,sdc (requires --lblk). Requested devices must be eligible; a busy device is an error.', type=str, default='', dest='blk_names', required=False) + subcommand.add_argument('--lblk', help='Configure the node with Linux block devices (lblk cluster mode) instead of NVMe PCIe devices: eligible whole disks or partitions (unmounted, unheld; disks additionally unpartitioned) are wrapped in SPDK AIO bdevs. Select devices with --blk-names, --blk-names-exclude or --blk-serials; without a selector, every eligible whole disk is used (partitions must be selected explicitly). Minimum 2 partitions or SSDs per node. When the selection contains partitions, the smallest one is split in two at configure time: a journal partition (--jm-percent of total capacity) and a data partition.', dest='lblk', action='store_true') + subcommand.add_argument('--blk-names', help='Comma separated list of block device names to use, like sdb,sdc or nvme0n1p3 (requires --lblk). Requested devices must be eligible; a busy device is an error.', type=str, default='', dest='blk_names', required=False) subcommand.add_argument('--blk-names-exclude', help='Comma separated list of block device names to exclude, like sda (requires --lblk). All other eligible disks are used.', type=str, default='', dest='blk_names_exclude', required=False) subcommand.add_argument('--blk-serials', help='Comma separated list of block device serial numbers (or WWNs) to use (requires --lblk).', type=str, default='', dest='blk_serials', required=False) + subcommand.add_argument('--jm-percent', help='Journal size in percent of the node\'s total selected capacity when the journal is carved by splitting a selected partition (requires --lblk with partitions). Default: `3`.', type=int, default=3, dest='jm_percent', required=False) subcommand.add_argument('--force', help='Force format detected or passed nvme pci address to 4K and clean partitions. With --lblk: mark partitioned disks eligible; the partition wipe happens at add-node with --force-format.', dest='force', action='store_true') subcommand.add_argument('--calculate-hp-only', help='Calculate the minimum required huge pages, it depends on the following params: --cores-percentage, --sockets-to-use, --max-subsys, --nodes-per-socket, --number-of-devices.', dest='calculate_hp_only', action='store_true') subcommand.add_argument('--number-of-devices', help='Number of devices that will be used on this host. For calculating huge pages memory only.', type=int, dest='number_of_devices') diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index bd1da74fb4..25d4b77309 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -172,7 +172,7 @@ def storage_node__configure(self, sub_command, args): pci_allowed, pci_blocked, force=args.force, device_model=args.device_model, size_range=args.size_range, cores_percentage=cores_percentage, nvme_names=nvme_names, calculate_hp_only=args.calculate_hp_only, number_of_devices=number_of_devices, - lblk_selection=lblk_selection) + lblk_selection=lblk_selection, jm_percent=int(getattr(args, 'jm_percent', 3) or 3)) def storage_node__deploy_cleaner(self, sub_command, args): storage_ops.deploy_cleaner() diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 2c0ed037c1..13d3ebc90a 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -94,6 +94,15 @@ def get_config_var(name, default=None): # Kernel block devices never eligible for lblk data placement. LBLK_EXCLUDED_NAME_PREFIXES = ("ram", "loop", "sr", "fd", "zram", "nbd", "md", "dm-", "drbd") +# Minimum storage units (whole SSDs or partitions) per lblk node: one journal +# home plus at least one data device. +LBLK_MIN_DEVICES_PER_NODE = 2 +# Journal sizing when the journal is carved out of a selected unit by +# splitting it in two (partition-backed lblk nodes): jm_percent of the node's +# total selected capacity, floored here, and never more than +# LBLK_JM_SPLIT_MAX_FRACTION of the unit being split. +LBLK_JM_MIN_SIZE = 2 * 1024 * 1024 * 1024 +LBLK_JM_SPLIT_MAX_FRACTION = 0.5 PMEM_DIR = '/tmp/pmem' diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 5dd38df7c1..c556d960bc 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -2496,6 +2496,26 @@ def _cluster_add_lock_heartbeat(db_controller, cluster_id, owner, stop_event): return +def _find_flagged_journal_device(snode, devices): + """The device matching a journal-flagged lblk config entry, or None. + + Partition-backed lblk nodes get their journal from the configure-time + partition split, which flags the resulting journal partition in the node + config (``journal: true``). Size-based selection would be wrong there: + the journal partition is not necessarily the smallest unit.""" + jm_serials = {e.get("serial") for e in (snode.lblk_devices or []) + if e.get("journal") and e.get("serial")} + if not jm_serials: + return None + for dev in devices: + if dev.serial_number in jm_serials: + return dev + logger.warning(f"Journal-flagged lblk entry {sorted(jm_serials)} not found " + f"among the node's devices; falling back to smallest-device " + f"journal selection") + return None + + def _classify_existing_endpoint_record(db_controller, cluster_id, node_addr, ssd_pcie, lblk_serials=None): """Classify a pre-existing storage-node record for ``node_addr`` that owns @@ -2735,11 +2755,20 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, return False # Phase 1: lblk requires journal-on-device (the GPT-partition JM # mode detaches/re-attaches NVMe controllers to re-examine). + # Partition-backed nodes get their journal from the configure-time + # partition split (journal-flagged entry) — still the + # journal-on-device layout, the "device" being that partition. if num_partitions_per_dev != 0 and jm_percent != 0: logger.error("lblk device mode requires --enable-journal-device " "(journal on a dedicated device); partitioned " "journal mode is not supported") return False + if len(lblk_configured) < constants.LBLK_MIN_DEVICES_PER_NODE: + logger.error( + f"lblk device mode requires at least " + f"{constants.LBLK_MIN_DEVICES_PER_NODE} partitions or SSDs " + f"per node; the node config carries {len(lblk_configured)}") + return False elif lblk_configured and not ssd_pcie: logger.error( "The node config carries 'lblk_devices' but this cluster runs " @@ -3377,10 +3406,15 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, # prepare devices if snode.num_partitions_per_dev == 0 or snode.jm_percent == 0: - jm_device = nvme_devs[0] - for index, nvme in enumerate(nvme_devs): - if nvme.size < jm_device.size: - jm_device = nvme + # Partition-backed lblk nodes carry an explicit journal entry + # (the configure-time partition split); otherwise the + # smallest device becomes the journal. + jm_device = _find_flagged_journal_device(snode, nvme_devs) + if jm_device is None: + jm_device = nvme_devs[0] + for index, nvme in enumerate(nvme_devs): + if nvme.size < jm_device.size: + jm_device = nvme jm_device.status = NVMeDevice.STATUS_JM ret = _prepare_cluster_devices_jm_on_dev(snode, nvme_devs) @@ -4991,10 +5025,12 @@ def _restart_storage_node_impl( # prepare devices on new node if snode.num_partitions_per_dev == 0 or snode.jm_percent == 0: - jm_device = snode.nvme_devices[0] - for index, nvme in enumerate(snode.nvme_devices): - if nvme.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_NEW] and nvme.size < jm_device.size: - jm_device = nvme + jm_device = _find_flagged_journal_device(snode, snode.nvme_devices) + if jm_device is None: + jm_device = snode.nvme_devices[0] + for index, nvme in enumerate(snode.nvme_devices): + if nvme.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_NEW] and nvme.size < jm_device.size: + jm_device = nvme jm_device.status = NVMeDevice.STATUS_JM if snode.jm_device and snode.jm_device.get_id(): @@ -6330,7 +6366,8 @@ def upgrade_automated_deployment_config(): def generate_automated_deployment_config(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_allowed, pci_blocked, cores_percentage=0, force=False, device_model="", size_range="", nvme_names=None, k8s=False, - calculate_hp_only=False, number_of_devices=0, lblk_selection=None): + calculate_hp_only=False, number_of_devices=0, lblk_selection=None, + jm_percent=3): # Reject an over-cap max_lvol here rather than only in the CLI: this is the # single entry point shared by `sn configure` and the k8s node-configure # job, and the value it writes into NODES_CONFIG_FILE becomes the node's @@ -6361,7 +6398,7 @@ def generate_automated_deployment_config(max_lvol, max_prov, sockets_to_use, nod nodes_config, system_info = utils.generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_allowed, pci_blocked, cores_percentage, force=force, device_model=device_model, size_range=size_range, nvme_names=nvme_names, - lblk_selection=lblk_selection) + lblk_selection=lblk_selection, jm_percent=jm_percent) if not nodes_config or not nodes_config.get("nodes"): return False utils.store_config_file(nodes_config, constants.NODES_CONFIG_FILE, create_read_only_file=True) diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index f8bc776185..dd806084dd 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -1426,18 +1426,26 @@ def aio_bdev_name_for_serial(serial: str) -> str: def resolve_lblk_entries(configured_entries, host_devices): """Match the node's configured lblk devices against the live host inventory, SERIAL-FIRST: kernel names shift across reboots, so the stored - name is only a fallback for devices without a resolvable serial. Returns - ``(resolved, missing)`` where resolved entries carry the CURRENT - name/path/by-id.""" + name is only a fallback for devices without a resolvable serial. Partition + entries additionally resolve by PARTUUID — it survives a parent-disk + serial change (e.g. a hypervisor re-exposing the volume) while the + derived partition serial would not. Returns ``(resolved, missing)`` + where resolved entries carry the CURRENT name/path/by-id.""" by_serial = {d["serial"]: d for d in host_devices} by_name = {d["name"]: d for d in host_devices} + by_partuuid = {d["partuuid"].lower(): d for d in host_devices + if d.get("partuuid")} resolved, missing = [], [] for entry in configured_entries: - live = by_serial.get(entry.get("serial")) or by_name.get(entry.get("name")) + live = by_serial.get(entry.get("serial")) + if live is None and entry.get("partuuid"): + live = by_partuuid.get(entry["partuuid"].lower()) + if live is None: + live = by_name.get(entry.get("name")) if live is None: missing.append(entry) continue - resolved.append({ + resolved_entry = { "name": live["name"], "current_path": live["device_path"], "serial": entry.get("serial") or live["serial"], @@ -1446,7 +1454,13 @@ def resolve_lblk_entries(configured_entries, host_devices): "numa": int(live.get("numa_node", entry.get("numa", -1))), "model": live.get("model", ""), "has_partitions": bool(live.get("has_partitions")), - }) + } + if entry.get("type") == "part" or live.get("type") == "part": + resolved_entry["type"] = "part" + resolved_entry["partuuid"] = live.get("partuuid") or entry.get("partuuid", "") + if entry.get("journal"): + resolved_entry["journal"] = True + resolved.append(resolved_entry) return resolved, missing @@ -1687,29 +1701,34 @@ def filter_eligible_block_devices(devices, include_names=None, exclude_names=Non """Eligibility filter for the lblk cluster mode (pure — unit-testable). ``devices`` is the list produced by node_utils.get_block_devices_info(). - A device is eligible iff it is a whole disk, not a special device - (LBLK_EXCLUDED_NAME_PREFIXES), carries no mountpoint anywhere in its - subtree, has no holders (LVM/md/dm-crypt), does not back the root - filesystem, is not read-only, has a non-zero size, and is unpartitioned - unless ``force_format`` (the actual wipe happens at add-node). + Both whole disks and partitions are eligible storage units. Common + requirements: not a special device (LBLK_EXCLUDED_NAME_PREFIXES), no + mountpoint anywhere in the subtree (a partition must be unmounted — not + busy — to be used), no holders (LVM/md/dm-crypt), does not back the root + filesystem, not read-only, non-zero size. A whole disk must additionally + be unpartitioned unless ``force_format`` (the actual wipe happens at + add-node); a partition only needs to be idle — its siblings may be in + use by the OS or other software. Selection is one of: ``include_names`` (explicitly requested names must exist AND be eligible — a busy requested device is a hard error), ``exclude_names`` (all eligible minus these), ``include_serials`` (matched against the serial/WWN identity). Without a selection, every - eligible disk is taken. + eligible whole disk is taken (partitions are never auto-selected — they + must be requested explicitly by name or serial). Returns ``(eligible_devices, rejected)`` where rejected is a list of ``(device_dict, reason)``. Raises ValueError on a requested-but- - ineligible name/serial or on duplicate serials among the selection. + ineligible name/serial, on duplicate serials among the selection, or on + a selection containing both a disk and one of its own partitions. """ include_names = set(include_names or []) exclude_names = set(exclude_names or []) include_serials = set(include_serials or []) def _ineligible_reason(dev): - if dev.get("type") != "disk": - return "not a whole disk" + if dev.get("type") not in ("disk", "part"): + return "not a disk or partition" if dev["name"].startswith(constants.LBLK_EXCLUDED_NAME_PREFIXES): return "special device type" if dev.get("mounted_in_subtree"): @@ -1722,7 +1741,7 @@ def _ineligible_reason(dev): return "read-only" if not dev.get("size"): return "zero size" - if dev.get("has_partitions") and not force_format: + if dev.get("type") == "disk" and dev.get("has_partitions") and not force_format: return "partitioned (pass --force to format at add-node)" return None @@ -1750,7 +1769,10 @@ def _ineligible_reason(dev): f"no eligible block device found for serial(s): {sorted(missing_serials)}") selected = [by_serial[s] for s in sorted(include_serials)] else: - selected = [d for d in eligible if d["name"] not in exclude_names] + # Auto-selection takes whole disks only: silently absorbing idle + # partitions of otherwise-used disks would be a data-loss trap. + selected = [d for d in eligible + if d["name"] not in exclude_names and d.get("type") == "disk"] serials = [d["serial"] for d in selected] dupes = {s for s in serials if serials.count(s) > 1} @@ -1758,6 +1780,17 @@ def _ineligible_reason(dev): raise ValueError( f"duplicate serial number(s) among selected block devices: {sorted(dupes)}; " f"device identity requires unique serials per node") + + # A whole disk selected for format and one of its own partitions selected + # as a unit cannot coexist — the disk wipe would destroy the partition. + selected_disk_names = {d["name"] for d in selected if d.get("type") == "disk"} + conflicting = sorted(d["name"] for d in selected + if d.get("type") == "part" + and d.get("parent_name") in selected_disk_names) + if conflicting: + raise ValueError( + f"selection contains partition(s) {conflicting} of a disk that is " + f"itself selected; select either the whole disk or its partitions") return selected, rejected @@ -1779,13 +1812,89 @@ def detect_lblk_devices(include_names=None, exclude_names=None, f"block device {dev['name']} has no hardware serial/WWN; using " f"synthetic identity {dev['serial']} (stable across reboots " f"only while size and by-id path are unchanged)") - result[dev["name"]] = { + entry = { "name": dev["name"], "serial": dev["serial"], "by_id": dev.get("by_id_path", ""), "size": int(dev["size"]), "numa": int(dev.get("numa_node", -1)), } + if dev.get("type") == "part": + entry["type"] = "part" + entry["partuuid"] = dev.get("partuuid", "") + entry["parent_serial"] = dev.get("parent_serial", "") + result[dev["name"]] = entry + if len(result) < constants.LBLK_MIN_DEVICES_PER_NODE: + raise ValueError( + f"lblk mode requires at least {constants.LBLK_MIN_DEVICES_PER_NODE} " + f"partitions or SSDs per node; only {len(result)} eligible unit(s) " + f"selected: {sorted(result)}") + return result + + +def _lblk_entry_from_inventory(dev) -> dict: + """Config-entry shape (detect_lblk_devices) from an inventory dict.""" + entry = { + "name": dev["name"], + "serial": dev["serial"], + "by_id": dev.get("by_id_path", ""), + "size": int(dev["size"]), + "numa": int(dev.get("numa_node", -1)), + } + if dev.get("type") == "part": + entry["type"] = "part" + entry["partuuid"] = dev.get("partuuid", "") + entry["parent_serial"] = dev.get("parent_serial", "") + return entry + + +def split_lblk_journal_partition(lblk_entries, jm_percent=3): + """Carve the journal for a partition-backed lblk node. + + When the selection contains partitions, the journal can neither dedicate + a whole drive (journal-on-device) nor relabel one (the rest of the disk + is not ours) — instead the SMALLEST selected partition is split in two: + a journal partition and a data partition covering the remainder. The + journal is sized at ``jm_percent`` of the node's total selected capacity, + floored at LBLK_JM_MIN_SIZE and capped at LBLK_JM_SPLIT_MAX_FRACTION of + the partition being split. + + Whole-disk-only selections are returned unchanged (they keep the + journal-on-device layout: the smallest disk becomes the journal at + add-node). Idempotent: a selection already carrying a journal-flagged + entry is returned unchanged. + + Runs on the storage node during `sn configure`. Returns the updated + ``{name: entry}`` dict with the journal entry flagged ``journal: True``. + """ + partitions = {n: e for n, e in lblk_entries.items() if e.get("type") == "part"} + if not partitions: + return lblk_entries + if any(e.get("journal") for e in lblk_entries.values()): + return lblk_entries + + target_name = min(partitions, key=lambda n: partitions[n]["size"]) + target = partitions[target_name] + total_size = sum(e["size"] for e in lblk_entries.values()) + jm_bytes = max(total_size * int(jm_percent) // 100, constants.LBLK_JM_MIN_SIZE) + max_jm = int(target["size"] * constants.LBLK_JM_SPLIT_MAX_FRACTION) + if jm_bytes > max_jm: + raise ValueError( + f"journal needs {jm_bytes} bytes ({jm_percent}% of {total_size}, " + f"min {constants.LBLK_JM_MIN_SIZE}) but the smallest selected " + f"partition {target_name} ({target['size']} bytes) may contribute " + f"at most {max_jm}; provide a larger partition") + + logger.info(f"Splitting partition {target_name} into a {jm_bytes}-byte " + f"journal partition and a data partition") + jm_dev, data_dev = node_utils.split_partition_for_journal(target_name, jm_bytes) + + result = {n: e for n, e in lblk_entries.items() if n != target_name} + jm_entry = _lblk_entry_from_inventory(jm_dev) + jm_entry["journal"] = True + result[jm_entry["name"]] = jm_entry + data_entry = _lblk_entry_from_inventory(data_dev) + result[data_entry["name"]] = data_entry return result @@ -2123,7 +2232,7 @@ def regenerate_config(new_config, old_config, force=False): def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_allowed, pci_blocked, cores_percentage=0, force=False, device_model="", size_range="", nvme_names=None, - lblk_selection=None): + lblk_selection=None, jm_percent=3): system_info = {} nodes_config: dict = {"nodes": []} @@ -2145,6 +2254,9 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a exclude_names=lblk_selection.get("names_exclude"), include_serials=lblk_selection.get("serials"), force_format=force) + # Partition-backed nodes: carve the journal by splitting the + # smallest selected partition in two (journal + data remainder). + lblk_entries = split_lblk_journal_partition(lblk_entries, jm_percent=jm_percent) except ValueError as e: logger.error(str(e)) return False, False @@ -2565,6 +2677,19 @@ def validate_node_config(node): logger.error(f"lblk_devices entry '{entry.get('name')}' needs a positive integer " f"'size' in node: {node.get('socket')}") return False + if lblk_devices: + if len(lblk_devices) < constants.LBLK_MIN_DEVICES_PER_NODE: + logger.error( + f"lblk mode requires at least {constants.LBLK_MIN_DEVICES_PER_NODE} " + f"partitions or SSDs per node; node {node.get('socket')} carries " + f"{len(lblk_devices)}") + return False + journal_entries = [e.get("name") for e in lblk_devices if e.get("journal")] + if len(journal_entries) > 1: + logger.error( + f"lblk_devices carries more than one journal-flagged entry " + f"{journal_entries} in node: {node.get('socket')}") + return False if not node["isolated"]: logger.error(f"'isolated' list is empty in node: {node.get('socket')}") diff --git a/simplyblock_web/node_configure.py b/simplyblock_web/node_configure.py index 76c8488dfc..63115bef8b 100755 --- a/simplyblock_web/node_configure.py +++ b/simplyblock_web/node_configure.py @@ -191,6 +191,15 @@ def parse_arguments() -> argparse.Namespace: dest='blk_serials', required=False ) + parser.add_argument( + '--jm-percent', + help='Journal size in percent of the node\'s total selected capacity when the ' + 'journal is carved by splitting a selected partition (requires --lblk with partitions)', + type=int, + default=3, + dest='jm_percent', + required=False + ) return parser.parse_args() @@ -231,15 +240,23 @@ def validate_arguments(args: argparse.Namespace) -> None: "pci-allowed and pci-blocked cannot be both specified" ) - use_lblk = bool(args.lblk or args.blk_names or args.blk_names_exclude or args.blk_serials) - if use_lblk and not args.lblk: + # getattr defaults: validate_arguments is also driven with minimal + # namespaces (tests, callers predating the lblk selectors). + lblk = getattr(args, 'lblk', False) + blk_names = getattr(args, 'blk_names', '') + blk_names_exclude = getattr(args, 'blk_names_exclude', '') + blk_serials = getattr(args, 'blk_serials', '') + use_lblk = bool(lblk or blk_names or blk_names_exclude or blk_serials) + if use_lblk and not lblk: raise argparse.ArgumentError( None, "--blk-names/--blk-names-exclude/--blk-serials require --lblk") - if use_lblk and (args.pci_allowed or args.pci_blocked or args.device_model - or args.size_range or args.nvme_names): + if use_lblk and (args.pci_allowed or args.pci_blocked + or getattr(args, 'device_model', '') + or getattr(args, 'size_range', '') + or getattr(args, 'nvme_names', '')): raise argparse.ArgumentError( None, "--lblk cannot be combined with NVMe device selection options") - if sum([bool(args.blk_names), bool(args.blk_names_exclude), bool(args.blk_serials)]) > 1: + if sum([bool(blk_names), bool(blk_names_exclude), bool(blk_serials)]) > 1: raise argparse.ArgumentError( None, "Choose only one of --blk-names, --blk-names-exclude, --blk-serials") @@ -325,7 +342,8 @@ def main() -> None: size_range=args.size_range, nvme_names=nvme_names, k8s=True, - lblk_selection=lblk_selection + lblk_selection=lblk_selection, + jm_percent=int(args.jm_percent or 3) ) except argparse.ArgumentError as e: diff --git a/simplyblock_web/node_utils.py b/simplyblock_web/node_utils.py index 9a4ebd40ae..aebf86ba8d 100644 --- a/simplyblock_web/node_utils.py +++ b/simplyblock_web/node_utils.py @@ -203,6 +203,48 @@ def _disk_by_id_path(name: str) -> str: return candidates[0] +def _partition_by_id_path(name: str, partuuid: str) -> str: + """Preferred stable path for a partition: /dev/disk/by-partuuid/ + (stable across disk renames and unaffected by by-id link churn), falling + back to a /dev/disk/by-id/*-part* symlink. Empty when none exists.""" + import os + if partuuid: + path = f"/dev/disk/by-partuuid/{partuuid.lower()}" + try: + if os.path.realpath(path) == f"/dev/{name}": + return path + except OSError: + pass + by_id_dir = "/dev/disk/by-id" + target = f"/dev/{name}" + candidates: List[str] = [] + try: + for entry in os.listdir(by_id_dir): + if "-part" not in entry: + continue + path = os.path.join(by_id_dir, entry) + try: + if os.path.realpath(path) == target: + candidates.append(path) + except OSError: + continue + except OSError: + return "" + if not candidates: + return "" + candidates.sort(key=lambda p: (0 if "/wwn-" in p.replace("\\", "/") else 1, p)) + return candidates[0] + + +def _partition_holders(disk_name: str, part_name: str) -> List[str]: + """Holders of a single partition (/sys/block///holders).""" + import os + try: + return sorted(set(os.listdir(f"/sys/block/{disk_name}/{part_name}/holders"))) + except OSError: + return [] + + def _root_disk_names() -> List[str]: """Kernel names of the disk(s) backing the root filesystem.""" out, _, rc = shell_utils.run_command("findmnt -no SOURCE /") @@ -228,20 +270,28 @@ def _subtree_mounted(dev: dict) -> bool: def get_block_devices_info() -> List[dict]: - """Inventory of whole-disk block devices for the lblk cluster mode. + """Inventory of block devices (whole disks AND their partitions) for the + lblk cluster mode. + + One dict per lsblk TYPE=disk entry plus one per TYPE=part child, carrying + everything the control plane needs for eligibility filtering, identity + (serial-first) and AIO bdev creation. Sizes are bytes (lsblk -b). + + Disk identity: SERIAL, falling back to WWN; devices with neither get a + synthetic-stable id derived from hostname|by-id-or-name|size so identity + survives reboots. - One dict per lsblk TYPE=disk entry, carrying everything the control - plane needs for eligibility filtering, identity (serial-first) and AIO - bdev creation. Sizes are bytes (lsblk -b). Serial falls back to WWN; - devices with neither get a synthetic-stable id derived from - hostname|by-id-or-name|size so identity survives reboots. + Partition identity: partitions have no lsblk SERIAL of their own, so the + serial is derived from the parent disk's serial plus the PARTUUID + ("-part-") — stable across disk renames and + unique per partition. Partitions without a PARTUUID get a synthetic id. """ import hashlib import socket logger.debug("function:get_block_devices_info start") out, err, rc = shell_utils.run_command( - "lsblk -J -b -o NAME,TYPE,SIZE,SERIAL,WWN,MOUNTPOINT,MODEL,ROTA,RO,VENDOR,PKNAME") + "lsblk -J -b -o NAME,TYPE,SIZE,SERIAL,WWN,MOUNTPOINT,MODEL,ROTA,RO,VENDOR,PKNAME,PARTUUID") if rc != 0: logger.error("Error running lsblk: %s", err) return [] @@ -269,6 +319,7 @@ def get_block_devices_info() -> List[dict]: seed = f"{hostname}|{by_id_path or name}|{dev.get('size') or 0}" serial = "SYN-" + hashlib.sha1(seed.encode()).hexdigest()[:16] synthetic = True + numa_node = int(_read_sysfs(f"/sys/block/{name}/device/numa_node") or -1) devices.append({ "name": name, "device_path": f"/dev/{name}", @@ -286,12 +337,151 @@ def get_block_devices_info() -> List[dict]: "holders": _disk_holders(name), "is_root_disk": name in root_disks, "by_id_path": by_id_path, - "numa_node": int(_read_sysfs(f"/sys/block/{name}/device/numa_node") or -1), + "numa_node": numa_node, }) + for child in children: + if child.get("type") != "part": + continue + part_name = child.get("name", "") + partuuid = (child.get("partuuid") or "").strip() + part_synthetic = False + if partuuid: + part_serial = f"{serial}-part-{partuuid.lower()}" + else: + seed = f"{hostname}|{serial}|{part_name}|{child.get('size') or 0}" + part_serial = "SYN-" + hashlib.sha1(seed.encode()).hexdigest()[:16] + part_synthetic = True + devices.append({ + "name": part_name, + "device_path": f"/dev/{part_name}", + "type": "part", + "size": int(child.get("size") or 0), + "serial": part_serial, + "serial_synthetic": part_synthetic or synthetic, + "partuuid": partuuid, + "parent_name": name, + "parent_serial": serial, + "wwn": wwn, + "model": (dev.get("model") or "").strip(), + "vendor": (dev.get("vendor") or "").strip(), + "rota": bool(dev.get("rota")), + "ro": bool(child.get("ro") or dev.get("ro")), + "has_partitions": False, + "mounted_in_subtree": _subtree_mounted(child), + "holders": _partition_holders(name, part_name), + "is_root_disk": part_name in root_disks, + "by_id_path": _partition_by_id_path(part_name, partuuid), + "numa_node": numa_node, + }) logger.debug("function:get_block_devices_info end") return devices +SB_GPT_PARTITION_TYPECODE = "6527994e-2c5a-4eec-9613-8f5944074e8b" + + +def split_partition_for_journal(part_name: str, jm_bytes: int) -> Tuple[dict, dict]: + """Split an existing GPT partition into two: a journal partition of + ``jm_bytes`` at its original start and a data partition covering the + remainder. Used by lblk nodes running on partitions, where the journal + can neither own a whole drive nor may we relabel one (the rest of the + disk belongs to the OS or other software). + + The parent disk's partition table is modified ONLY within the bounds of + the partition being split. The partition must be idle (unmounted, no + holders, not backing root). Returns ``(jm_device, data_device)`` — the + two new partitions' inventory dicts (get_block_devices_info shape). + Raises ValueError on any precondition or tool failure. + """ + import math + import os + + inventory = {d["name"]: d for d in get_block_devices_info()} + part = inventory.get(part_name) + if part is None or part.get("type") != "part": + raise ValueError(f"partition {part_name} not found") + if part.get("mounted_in_subtree"): + raise ValueError(f"partition {part_name} is mounted (busy)") + if part.get("holders"): + raise ValueError(f"partition {part_name} is held by {part['holders']} (busy)") + if part.get("is_root_disk"): + raise ValueError(f"partition {part_name} backs the root filesystem") + parent = part.get("parent_name", "") + if not parent: + raise ValueError(f"cannot determine parent disk of {part_name}") + + out, _, rc = shell_utils.run_command(f"lsblk -ndo PTTYPE /dev/{parent}") + if rc != 0 or out.strip() != "gpt": + raise ValueError( + f"disk {parent} has partition table {out.strip() or 'unknown'!r}; " + f"splitting a partition for the journal requires GPT") + + sys_part = f"/sys/block/{parent}/{part_name}" + try: + part_number = int(_read_sysfs(f"{sys_part}/partition")) + start_sector = int(_read_sysfs(f"{sys_part}/start")) + size_sectors = int(_read_sysfs(f"{sys_part}/size")) + except (ValueError, TypeError): + raise ValueError(f"cannot read geometry of {part_name} from sysfs") + + # 1 MiB alignment (2048 x 512b sectors) for the data partition start. + align = 2048 + jm_sectors = int(math.ceil(jm_bytes / 512 / align) * align) + end_sector = start_sector + size_sectors - 1 + data_start = start_sector + jm_sectors + if data_start + align > end_sector: + raise ValueError( + f"partition {part_name} ({size_sectors * 512} bytes) is too small " + f"to split into a {jm_bytes}-byte journal plus a data partition") + + cmds = [ + f"sgdisk -d {part_number} /dev/{parent}", + (f"sgdisk -a 1 -n {part_number}:{start_sector}:{data_start - 1} " + f"-t {part_number}:{SB_GPT_PARTITION_TYPECODE} -c {part_number}:sb_jm /dev/{parent}"), + (f"sgdisk -a 1 -n 0:{data_start}:{end_sector} " + f"-t 0:{SB_GPT_PARTITION_TYPECODE} -c 0:sb_data /dev/{parent}"), + ] + for cmd in cmds: + out, err, rc = shell_utils.run_command(cmd) + if rc != 0: + raise ValueError(f"{cmd} failed (rc={rc}): {err or out}") + + _, _, rc = shell_utils.run_command(f"partprobe /dev/{parent}") + if rc != 0: + _, err, rc = shell_utils.run_command(f"partx -u /dev/{parent}") + if rc != 0: + raise ValueError(f"failed to re-read partition table of {parent}: {err}") + shell_utils.run_command("udevadm settle -t 5") + + # Identify the two new partitions by their start sectors. + jm_name = data_name = "" + try: + for entry in os.listdir(f"/sys/block/{parent}"): + if not entry.startswith(parent): + continue + e_start = _read_sysfs(f"/sys/block/{parent}/{entry}/start") + if not e_start: + continue + if int(e_start) == start_sector: + jm_name = entry + elif int(e_start) == data_start: + data_name = entry + except OSError: + pass + if not jm_name or not data_name: + raise ValueError( + f"split of {part_name} completed but the new partitions were not " + f"found on {parent} (journal at sector {start_sector}, data at " + f"{data_start})") + + inventory = {d["name"]: d for d in get_block_devices_info()} + if jm_name not in inventory or data_name not in inventory: + raise ValueError( + f"new partitions {jm_name}/{data_name} missing from inventory " + f"after split of {part_name}") + return inventory[jm_name], inventory[data_name] + + def wipe_block_device_signatures(device_name: str) -> Tuple[bool, str]: """Wipe partition-table / filesystem signatures from a whole disk (`--force-format` on lblk add-node). Re-validates that the device is not diff --git a/tests/unit/test_lblk_eligibility.py b/tests/unit/test_lblk_eligibility.py index f1fc1db984..1607616dde 100644 --- a/tests/unit/test_lblk_eligibility.py +++ b/tests/unit/test_lblk_eligibility.py @@ -55,9 +55,32 @@ def test_clean_disk_is_eligible(self): self.assertEqual([d["name"] for d in sel], ["sdb"]) self.assertEqual(rej, []) - def test_partition_type_rejected(self): - reasons = self._reasons([_blk("sdb1", dtype="part")]) - self.assertIn("not a whole disk", reasons["sdb1"]) + def test_non_disk_non_part_type_rejected(self): + reasons = self._reasons([_blk("dax0.0", dtype="lvm")]) + self.assertIn("not a disk or partition", reasons["dax0.0"]) + + def test_idle_partition_is_eligible_when_requested(self): + devs = [_blk("sdb1", dtype="part")] + sel, _ = utils.filter_eligible_block_devices(devs, include_names=["sdb1"]) + self.assertEqual([d["name"] for d in sel], ["sdb1"]) + + def test_mounted_partition_rejected(self): + reasons = self._reasons([_blk("sdb1", dtype="part", mounted=True)]) + self.assertIn("busy", reasons["sdb1"]) + + def test_partition_never_auto_selected(self): + devs = [_blk("sdb"), _blk("sdc1", dtype="part")] + sel, _ = utils.filter_eligible_block_devices(devs) + self.assertEqual([d["name"] for d in sel], ["sdb"]) + + def test_disk_and_own_partition_conflict(self): + part = _blk("sdb1", dtype="part") + part["parent_name"] = "sdb" + devs = [_blk("sdb", parts=True), part] + with self.assertRaises(ValueError) as ctx: + utils.filter_eligible_block_devices( + devs, include_names=["sdb", "sdb1"], force_format=True) + self.assertIn("itself selected", str(ctx.exception)) def test_special_prefixes_rejected(self): for name in ("ram0", "loop3", "sr0", "zram1", "nbd0", "md127", "dm-0", "drbd0", "fd0"): @@ -148,16 +171,35 @@ class TestDetectLblkDevices(unittest.TestCase): def test_maps_config_entry_shape(self): devs = [_blk("sdb", serial="S1", by_id="/dev/disk/by-id/wwn-0x1", - size=42, numa=1)] + size=42, numa=1), + _blk("sdc", serial="S2", size=42, numa=0)] with patch.object(utils.node_utils, "get_block_devices_info", return_value=devs): result = utils.detect_lblk_devices() - self.assertEqual(result, { - "sdb": {"name": "sdb", "serial": "S1", - "by_id": "/dev/disk/by-id/wwn-0x1", "size": 42, "numa": 1}, - }) + self.assertEqual(result["sdb"], { + "name": "sdb", "serial": "S1", + "by_id": "/dev/disk/by-id/wwn-0x1", "size": 42, "numa": 1}) + + def test_partition_config_entry_carries_identity(self): + part = _blk("sdc1", dtype="part", serial="S2-part-uuid1", size=42, numa=0) + part["partuuid"] = "uuid1" + part["parent_serial"] = "S2" + devs = [_blk("sdb", serial="S1", size=42), part] + with patch.object(utils.node_utils, "get_block_devices_info", return_value=devs): + result = utils.detect_lblk_devices(include_names=["sdb", "sdc1"]) + self.assertEqual(result["sdc1"]["type"], "part") + self.assertEqual(result["sdc1"]["partuuid"], "uuid1") + self.assertEqual(result["sdc1"]["parent_serial"], "S2") + + def test_fewer_than_minimum_units_raises(self): + devs = [_blk("sdb", serial="S1")] + with patch.object(utils.node_utils, "get_block_devices_info", return_value=devs): + with self.assertRaises(ValueError) as ctx: + utils.detect_lblk_devices() + self.assertIn("at least 2", str(ctx.exception)) def test_synthetic_serial_warns_but_passes(self): - devs = [_blk("sdb", serial="SYN-abc123", synthetic=True)] + devs = [_blk("sdb", serial="SYN-abc123", synthetic=True), + _blk("sdc", serial="S2")] with patch.object(utils.node_utils, "get_block_devices_info", return_value=devs), \ patch.object(utils, "logger") as mock_logger: result = utils.detect_lblk_devices() @@ -292,7 +334,24 @@ def test_valid_nvme_config(self): self.assertTrue(utils.validate_node_config(self._node(ssd_pcis=["0000:00:1e.0"]))) def test_valid_lblk_config(self): + node = self._node(lblk_devices=[{"name": "sdb", "serial": "S1", "size": 100}, + {"name": "sdc", "serial": "S2", "size": 100}]) + self.assertTrue(utils.validate_node_config(node)) + + def test_single_lblk_entry_rejected(self): node = self._node(lblk_devices=[{"name": "sdb", "serial": "S1", "size": 100}]) + self.assertFalse(utils.validate_node_config(node)) + + def test_two_journal_flags_rejected(self): + node = self._node(lblk_devices=[ + {"name": "sdb1", "serial": "S1", "size": 100, "journal": True}, + {"name": "sdb2", "serial": "S2", "size": 100, "journal": True}]) + self.assertFalse(utils.validate_node_config(node)) + + def test_one_journal_flag_valid(self): + node = self._node(lblk_devices=[ + {"name": "sdb1", "serial": "S1", "size": 100, "journal": True}, + {"name": "sdb2", "serial": "S2", "size": 100}]) self.assertTrue(utils.validate_node_config(node)) def test_nvme_config_without_lblk_key_still_valid(self): diff --git a/tests/unit/test_lblk_partitions.py b/tests/unit/test_lblk_partitions.py new file mode 100644 index 0000000000..c1c09f6b79 --- /dev/null +++ b/tests/unit/test_lblk_partitions.py @@ -0,0 +1,372 @@ +# coding=utf-8 +"""Unit tests for lblk partition support. + +Covered: + - node_utils.get_block_devices_info: partitions emitted with derived + identity (parent serial + PARTUUID), busy/root/holder detection, + synthetic fallback without a PARTUUID. + - utils.split_lblk_journal_partition: whole-disk selections unchanged, + idempotency on a journal-flagged selection, smallest-partition choice, + jm sizing (percent of total, floor, max-fraction cap), resulting entry + shapes (journal flag, replacement of the split entry). + - node_utils.split_partition_for_journal: preconditions (missing, mounted, + held, non-GPT), sgdisk command sequence, sector math and alignment, new + partition discovery. + - utils.resolve_lblk_entries: PARTUUID fallback resolution and journal + flag carry-over. + - storage_node_ops._find_flagged_journal_device. +""" + +import json +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import constants, storage_node_ops, utils +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_web import node_utils + +GIB = 1024 * 1024 * 1024 + + +def _entry(name, serial=None, size=100 * GIB, dtype="disk", partuuid="", + parent_serial="", journal=False): + e = {"name": name, "serial": serial or f"SER-{name}", "by_id": "", + "size": size, "numa": 0} + if dtype == "part": + e["type"] = "part" + e["partuuid"] = partuuid + e["parent_serial"] = parent_serial + if journal: + e["journal"] = True + return e + + +def _inv(name, serial=None, size=100 * GIB, dtype="disk", partuuid="", + parent_name="", parent_serial="", mounted=False, holders=None, + root=False): + return { + "name": name, "device_path": f"/dev/{name}", "type": dtype, + "size": size, "serial": serial or f"SER-{name}", + "serial_synthetic": False, "partuuid": partuuid, + "parent_name": parent_name, "parent_serial": parent_serial, + "wwn": "", "model": "MODEL-X", "vendor": "ACME", "rota": False, + "ro": False, "has_partitions": False, "mounted_in_subtree": mounted, + "holders": holders or [], "is_root_disk": root, "by_id_path": "", + "numa_node": 0, + } + + +class TestInventoryPartitions(unittest.TestCase): + LSBLK = { + "blockdevices": [ + {"name": "nvme1n1", "type": "disk", "size": 200 * GIB, + "serial": "VOL-A", "wwn": "", "mountpoint": None, + "model": "EBS", "rota": False, "ro": False, "vendor": "AWS", + "children": [ + {"name": "nvme1n1p1", "type": "part", "size": 50 * GIB, + "partuuid": "AAAA-01", "mountpoint": None, "ro": False}, + {"name": "nvme1n1p2", "type": "part", "size": 150 * GIB, + "partuuid": "AAAA-02", "mountpoint": "/data", "ro": False}, + ]}, + {"name": "nvme2n1", "type": "disk", "size": 100 * GIB, + "serial": "", "wwn": "", "mountpoint": None, + "model": "EBS", "rota": False, "ro": False, "vendor": "AWS", + "children": [ + {"name": "nvme2n1p1", "type": "part", "size": 100 * GIB, + "partuuid": "", "mountpoint": None, "ro": False}, + ]}, + ] + } + + def _inventory(self): + with patch.object(node_utils.shell_utils, "run_command", + return_value=(json.dumps(self.LSBLK), "", 0)), \ + patch.object(node_utils, "_root_disk_names", return_value=[]), \ + patch.object(node_utils, "_disk_holders", return_value=[]), \ + patch.object(node_utils, "_partition_holders", return_value=[]), \ + patch.object(node_utils, "_disk_by_id_path", return_value=""), \ + patch.object(node_utils, "_partition_by_id_path", return_value=""), \ + patch.object(node_utils, "_read_sysfs", return_value="0"): + return {d["name"]: d for d in node_utils.get_block_devices_info()} + + def test_partitions_emitted_with_parent_identity(self): + inv = self._inventory() + self.assertIn("nvme1n1p1", inv) + p1 = inv["nvme1n1p1"] + self.assertEqual(p1["type"], "part") + self.assertEqual(p1["serial"], "VOL-A-part-aaaa-01") + self.assertEqual(p1["parent_name"], "nvme1n1") + self.assertEqual(p1["parent_serial"], "VOL-A") + self.assertEqual(p1["partuuid"], "AAAA-01") + self.assertFalse(p1["serial_synthetic"]) + + def test_mounted_partition_marked_busy_but_parent_subtree_too(self): + inv = self._inventory() + self.assertTrue(inv["nvme1n1p2"]["mounted_in_subtree"]) + self.assertFalse(inv["nvme1n1p1"]["mounted_in_subtree"]) + # the parent disk carries the subtree mount and its partitions + self.assertTrue(inv["nvme1n1"]["mounted_in_subtree"]) + self.assertTrue(inv["nvme1n1"]["has_partitions"]) + + def test_partition_without_partuuid_gets_synthetic_serial(self): + inv = self._inventory() + p = inv["nvme2n1p1"] + self.assertTrue(p["serial"].startswith("SYN-")) + self.assertTrue(p["serial_synthetic"]) + + def test_disk_without_serial_still_parents_partition_identity(self): + inv = self._inventory() + # parent got a synthetic serial; the partition inherits it as parent_serial + self.assertTrue(inv["nvme2n1"]["serial"].startswith("SYN-")) + self.assertEqual(inv["nvme2n1p1"]["parent_serial"], inv["nvme2n1"]["serial"]) + + +class TestSplitLblkJournalPartition(unittest.TestCase): + + def test_whole_disk_selection_unchanged(self): + entries = {"sdb": _entry("sdb"), "sdc": _entry("sdc")} + with patch.object(utils.node_utils, "split_partition_for_journal") as split: + out = utils.split_lblk_journal_partition(entries) + self.assertEqual(out, entries) + split.assert_not_called() + + def test_idempotent_when_journal_already_flagged(self): + entries = { + "sdb1": _entry("sdb1", dtype="part", journal=True), + "sdb2": _entry("sdb2", dtype="part"), + } + with patch.object(utils.node_utils, "split_partition_for_journal") as split: + out = utils.split_lblk_journal_partition(entries) + self.assertEqual(out, entries) + split.assert_not_called() + + def test_smallest_partition_is_split(self): + entries = { + "p_big": _entry("p_big", dtype="part", size=100 * GIB, partuuid="B"), + "p_small": _entry("p_small", dtype="part", size=50 * GIB, partuuid="S"), + } + jm_inv = _inv("p_small_jm", serial="S-jm", dtype="part", size=4 * GIB, + partuuid="NEW-JM", parent_serial="PAR") + data_inv = _inv("p_small_data", serial="S-data", dtype="part", + size=46 * GIB, partuuid="NEW-DATA", parent_serial="PAR") + with patch.object(utils.node_utils, "split_partition_for_journal", + return_value=(jm_inv, data_inv)) as split: + out = utils.split_lblk_journal_partition(entries, jm_percent=3) + split.assert_called_once() + self.assertEqual(split.call_args[0][0], "p_small") + # 3% of 150 GiB = 4.5 GiB > 2 GiB floor + self.assertEqual(split.call_args[0][1], int(150 * GIB * 3 // 100)) + self.assertNotIn("p_small", out) + self.assertIn("p_big", out) + self.assertTrue(out["p_small_jm"]["journal"]) + self.assertEqual(out["p_small_jm"]["type"], "part") + self.assertEqual(out["p_small_jm"]["partuuid"], "NEW-JM") + self.assertNotIn("journal", out["p_small_data"]) + + def test_jm_floor_applies_for_small_capacity(self): + entries = { + "p1": _entry("p1", dtype="part", size=10 * GIB), + "p2": _entry("p2", dtype="part", size=10 * GIB), + } + jm_inv = _inv("p1_jm", dtype="part", size=2 * GIB) + data_inv = _inv("p1_data", dtype="part", size=8 * GIB) + with patch.object(utils.node_utils, "split_partition_for_journal", + return_value=(jm_inv, data_inv)) as split: + utils.split_lblk_journal_partition(entries, jm_percent=3) + # 3% of 20 GiB = 0.6 GiB < LBLK_JM_MIN_SIZE floor + self.assertEqual(split.call_args[0][1], constants.LBLK_JM_MIN_SIZE) + + def test_journal_too_big_for_smallest_partition_raises(self): + entries = { + "p1": _entry("p1", dtype="part", size=3 * GIB), + "p2": _entry("p2", dtype="part", size=500 * GIB), + } + with patch.object(utils.node_utils, "split_partition_for_journal"): + with self.assertRaises(ValueError) as ctx: + utils.split_lblk_journal_partition(entries, jm_percent=3) + self.assertIn("provide a larger partition", str(ctx.exception)) + + def test_mixed_selection_splits_partition_not_disk(self): + entries = { + "sdb": _entry("sdb", size=10 * GIB), # smaller than the partition + "p1": _entry("p1", dtype="part", size=100 * GIB), + } + jm_inv = _inv("p1_jm", dtype="part", size=3 * GIB) + data_inv = _inv("p1_data", dtype="part", size=97 * GIB) + with patch.object(utils.node_utils, "split_partition_for_journal", + return_value=(jm_inv, data_inv)) as split: + out = utils.split_lblk_journal_partition(entries) + # only partitions are split candidates, never the (smaller) whole disk + self.assertEqual(split.call_args[0][0], "p1") + self.assertIn("sdb", out) + + +class TestSplitPartitionForJournal(unittest.TestCase): + """node_utils.split_partition_for_journal with mocked host state.""" + + PARENT = "nvme1n1" + PART = "nvme1n1p2" + + def _run(self, jm_bytes=2 * GIB, part_kwargs=None, pttype="gpt", + start=2048, size_sectors=8 * GIB // 512, sgdisk_rc=0): + part = _inv(self.PART, dtype="part", parent_name=self.PARENT, + **(part_kwargs or {})) + # post-split inventory contains the two new partitions + jm_sectors = -(-jm_bytes // 512 // 2048) * 2048 + data_start = start + jm_sectors + new_jm = _inv(self.PART, dtype="part", partuuid="NEW-JM", + parent_name=self.PARENT, serial="PJM") + new_data = _inv("nvme1n1p9", dtype="part", partuuid="NEW-DATA", + parent_name=self.PARENT, serial="PDATA") + + inventories = [[part], [new_jm, new_data]] + + def fake_inventory(): + return inventories.pop(0) if len(inventories) > 1 else inventories[0] + + sysfs = { + f"/sys/block/{self.PARENT}/{self.PART}/partition": "2", + f"/sys/block/{self.PARENT}/{self.PART}/start": str(start), + f"/sys/block/{self.PARENT}/{self.PART}/size": str(size_sectors), + f"/sys/block/{self.PARENT}/{self.PART}/start_after": "", + } + # after the split, sysfs lists the two new partitions + post_split_children = { + self.PART: str(start), + "nvme1n1p9": str(data_start), + } + + commands = [] + + def fake_run(cmd): + commands.append(cmd) + if cmd.startswith("lsblk -ndo PTTYPE"): + return pttype, "", 0 + if cmd.startswith("sgdisk"): + return "", "", sgdisk_rc + return "", "", 0 + + def fake_sysfs(path): + for child, st in post_split_children.items(): + if path.endswith(f"/{child}/start"): + return st + return sysfs.get(path, "") + + with patch.object(node_utils, "get_block_devices_info", + side_effect=fake_inventory), \ + patch.object(node_utils.shell_utils, "run_command", + side_effect=fake_run), \ + patch.object(node_utils, "_read_sysfs", side_effect=fake_sysfs), \ + patch("os.listdir", return_value=[self.PART, "nvme1n1p9"]): + result = node_utils.split_partition_for_journal(self.PART, jm_bytes) + return result, commands + + def test_happy_path_commands_and_result(self): + (jm, data), commands = self._run() + self.assertEqual(jm["partuuid"], "NEW-JM") + self.assertEqual(data["partuuid"], "NEW-DATA") + sg = [c for c in commands if c.startswith("sgdisk")] + self.assertEqual(len(sg), 3) + self.assertIn(f"-d 2 /dev/{self.PARENT}", sg[0]) + # journal recreated at the original start with the original number + self.assertIn("-n 2:2048:", sg[1]) + self.assertIn(node_utils.SB_GPT_PARTITION_TYPECODE, sg[1]) + # data partition takes the first free number, starts 1MiB-aligned + jm_sectors = -(-2 * GIB // 512 // 2048) * 2048 + self.assertIn(f"-n 0:{2048 + jm_sectors}:", sg[2]) + + def test_non_gpt_refused(self): + with self.assertRaises(ValueError) as ctx: + self._run(pttype="dos") + self.assertIn("GPT", str(ctx.exception)) + + def test_mounted_partition_refused(self): + with self.assertRaises(ValueError) as ctx: + self._run(part_kwargs={"mounted": True}) + self.assertIn("busy", str(ctx.exception)) + + def test_held_partition_refused(self): + with self.assertRaises(ValueError) as ctx: + self._run(part_kwargs={"holders": ["dm-0"]}) + self.assertIn("held", str(ctx.exception)) + + def test_partition_too_small_refused(self): + with self.assertRaises(ValueError) as ctx: + self._run(jm_bytes=16 * GIB, size_sectors=8 * GIB // 512) + self.assertIn("too small", str(ctx.exception)) + + def test_sgdisk_failure_raises(self): + with self.assertRaises(ValueError) as ctx: + self._run(sgdisk_rc=1) + self.assertIn("sgdisk", str(ctx.exception)) + + def test_missing_partition_refused(self): + with patch.object(node_utils, "get_block_devices_info", return_value=[]): + with self.assertRaises(ValueError) as ctx: + node_utils.split_partition_for_journal("nope1", GIB) + self.assertIn("not found", str(ctx.exception)) + + +class TestResolvePartitionEntries(unittest.TestCase): + + def test_partuuid_fallback_when_parent_serial_changed(self): + # hypervisor re-exposed the volume: parent serial (and thus the + # derived partition serial) changed, PARTUUID survived. + configured = [_entry("sdb1", serial="OLD-part-aaaa", dtype="part", + partuuid="AAAA", parent_serial="OLD")] + live = [_inv("sdz1", serial="NEW-part-aaaa", dtype="part", + partuuid="AAAA", parent_serial="NEW")] + resolved, missing = utils.resolve_lblk_entries(configured, live) + self.assertEqual(missing, []) + self.assertEqual(resolved[0]["name"], "sdz1") + self.assertEqual(resolved[0]["type"], "part") + + def test_journal_flag_carried_through_resolution(self): + configured = [_entry("sdb1", serial="S1", dtype="part", + partuuid="AAAA", journal=True)] + live = [_inv("sdb1", serial="S1", dtype="part", partuuid="AAAA")] + resolved, _ = utils.resolve_lblk_entries(configured, live) + self.assertTrue(resolved[0]["journal"]) + + def test_disk_entries_unaffected(self): + configured = [_entry("sdb", serial="S1")] + live = [_inv("sdb", serial="S1")] + resolved, _ = utils.resolve_lblk_entries(configured, live) + self.assertNotIn("type", resolved[0]) + self.assertNotIn("journal", resolved[0]) + + +class TestFindFlaggedJournalDevice(unittest.TestCase): + + def _snode(self, lblk_devices): + n = StorageNode() + n.uuid = "node-1" + n.lblk_devices = lblk_devices + return n + + def _dev(self, serial): + d = MagicMock() + d.serial_number = serial + return d + + def test_flagged_entry_matched_by_serial(self): + snode = self._snode([ + {"name": "p1", "serial": "S-JM", "journal": True}, + {"name": "p2", "serial": "S-DATA"}, + ]) + devs = [self._dev("S-DATA"), self._dev("S-JM")] + found = storage_node_ops._find_flagged_journal_device(snode, devs) + self.assertIs(found, devs[1]) + + def test_no_flag_returns_none(self): + snode = self._snode([{"name": "sdb", "serial": "S1"}, + {"name": "sdc", "serial": "S2"}]) + devs = [self._dev("S1"), self._dev("S2")] + self.assertIsNone( + storage_node_ops._find_flagged_journal_device(snode, devs)) + + def test_flag_without_matching_device_returns_none(self): + snode = self._snode([{"name": "p1", "serial": "S-GONE", "journal": True}]) + devs = [self._dev("S-OTHER")] + self.assertIsNone( + storage_node_ops._find_flagged_journal_device(snode, devs)) From e89569d4ca09d64cda640895c57b7aadd5a5b743 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 14 Aug 2026 15:51:42 +0200 Subject: [PATCH 7/9] Add single-node lblk deployment script + partition soak (2 SSD / 2 part / 4 part) deploy_single_node_lblk.py deploys a 1-node non-HA lblk cluster on AWS in one of three device configurations: two whole EBS volumes (journal-on- device, EC 1+0), one volume with 2 GPT partitions (configure-time journal split, EC 1+1), or one volume with 4 partitions (EC 2+1). Self-contained (boto3+paramiko), emits cluster metadata JSON for the soak driver. single_node_partition_soak.py drives all three configs end-to-end: lvol create + nvme-tcp connect on the mgmt instance, crc32c-stamped data region plus mixed random IO, graceful `sn restart`, then a verify-only crc32c pass over the stamped region proving the data is available and uncorrupted after the node restart. Fleets are terminated on success and kept for debugging on failure. --- scripts/deploy_single_node_lblk.py | 303 ++++++++++++++++++++++++++ scripts/single_node_partition_soak.py | 217 ++++++++++++++++++ 2 files changed, 520 insertions(+) create mode 100644 scripts/deploy_single_node_lblk.py create mode 100644 scripts/single_node_partition_soak.py diff --git a/scripts/deploy_single_node_lblk.py b/scripts/deploy_single_node_lblk.py new file mode 100644 index 0000000000..c6f8c14a81 --- /dev/null +++ b/scripts/deploy_single_node_lblk.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Deploy a SINGLE-NODE lblk cluster on AWS in one of three device configs. + +Configs (the three shapes the single-node partition soak exercises): + 2ssd two EBS volumes as whole disks -> journal-on-device (smallest + disk becomes the journal), 1 data device, EC 1+0 + 2part one EBS volume carrying 2 GPT partitions -> `sn configure --lblk + --blk-names p1,p2` splits the smallest partition into journal + + data at configure time, 2 data devices, EC 1+1 + 4part one EBS volume carrying 4 GPT partitions -> split of the smallest + partition, 4 data devices, EC 2+1 + +The cluster is created with --is-single-node --device-mode lblk: activation +configures it non-HA with a single local journal regardless of the EC +schema, physical labels stay 0, and every lvol lifecycle op runs on the one +node (ha_type downgrade). + +Usage: + ./deploy_single_node_lblk.py --config 2part [--keep-cluster-metadata FILE] + +Writes cluster_metadata_single_node_.json next to this script +(mgmt/SN IPs + instance ids, cluster uuid, config) for the soak driver. +""" +import argparse +import json +import os +import re +import sys +import time +from concurrent.futures import ThreadPoolExecutor + +import boto3 +import paramiko + +# --- lab constants (us-east-1 test account; override via env) --------------- +AMI_ID = os.environ.get("SB_AMI_ID", "ami-0dfc569a8686b9320") # Rocky 9 +KEY_NAME = os.environ.get("SB_KEY_NAME", "mtes01") +KEY_PATH = os.path.expanduser(os.environ.get("SB_KEY_PATH", "~/.ssh/mtes01.pem")) +SUBNET_ID = os.environ.get("SB_SUBNET_ID", "subnet-0593459d6b931ee4c") +SG_ID = os.environ.get("SB_SG_ID", "sg-02e89a1372e9f39e9") +REGION = os.environ.get("SB_REGION", "us-east-1") +BRANCH = os.environ.get("SB_BRANCH", "md-journal") +USER = "ec2-user" +IFACE = "eth0" +MAX_LVOL = "50" +INSTANCE_TYPE = os.environ.get("SB_INSTANCE_TYPE", "m6i.2xlarge") +EBS_IOPS, EBS_TPUT = 6000, 500 + +CONFIGS = { + # volumes: (size_gb, ...) attached beyond the 30G root + # partitions: None = whole disks; N = create N equal GPT partitions on + # the single data volume and select them by name + # ec: (ndcs, npcs) for cluster create + "2ssd": {"volumes": (30, 100), "partitions": None, "ec": (1, 0)}, + "2part": {"volumes": (160,), "partitions": 2, "ec": (1, 1)}, + "4part": {"volumes": (220,), "partitions": 4, "ec": (2, 1)}, +} + +SBCTL = "sudo /usr/local/bin/sbctl -d" + + +# --- ssh helpers ------------------------------------------------------------- + +def ssh_exec(ip, cmds, get_output=False, check=False, timeout=900): + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, username=USER, key_filename=KEY_PATH, + allow_agent=False, look_for_keys=False, timeout=60) + results = [] + try: + for cmd in cmds: + print(f" [{ip}] $ {cmd}") + _, stdout, stderr = ssh.exec_command(cmd, timeout=timeout) + out = stdout.read().decode() + err = stderr.read().decode() + rc = stdout.channel.recv_exit_status() + if get_output: + results.append(out) + if rc != 0: + print(f" [{ip}] rc={rc}: {cmd}") + for line in (out.strip().splitlines() or [])[-15:]: + print(f" stdout: {line}") + for line in (err.strip().splitlines() or [])[-15:]: + print(f" stderr: {line}") + if check: + raise RuntimeError(f"Command failed on {ip} (rc={rc}): {cmd}") + finally: + ssh.close() + return results + + +def wait_for_ssh(ip, timeout=600): + deadline = time.time() + timeout + while time.time() < deadline: + try: + ssh_exec(ip, ["true"]) + return + except Exception: + time.sleep(10) + raise TimeoutError(f"SSH not ready on {ip} within {timeout}s") + + +# --- aws --------------------------------------------------------------------- + +def _block_mappings(volume_sizes): + mappings = [{"DeviceName": "/dev/sda1", + "Ebs": {"VolumeSize": 30, "DeleteOnTermination": True, + "VolumeType": "gp3"}}] + for i, size in enumerate(volume_sizes): + mappings.append({ + "DeviceName": f"/dev/sd{chr(ord('b') + i)}", + "Ebs": {"VolumeSize": size, "DeleteOnTermination": True, + "VolumeType": "gp3", "Iops": EBS_IOPS, + "Throughput": EBS_TPUT}, + }) + return mappings + + +def launch_instances(config_name): + ec2 = boto3.resource("ec2", region_name=REGION) + cfg = CONFIGS[config_name] + + def launch(name, mappings): + return ec2.create_instances( + ImageId=AMI_ID, InstanceType=INSTANCE_TYPE, MinCount=1, MaxCount=1, + KeyName=KEY_NAME, + NetworkInterfaces=[{"DeviceIndex": 0, "SubnetId": SUBNET_ID, + "Groups": [SG_ID], + "AssociatePublicIpAddress": True}], + BlockDeviceMappings=mappings, + TagSpecifications=[{"ResourceType": "instance", + "Tags": [{"Key": "Name", "Value": name}]}])[0] + + mgmt = launch(f"SB-1N-Mgmt-{config_name}", _block_mappings(())) + sn = launch(f"SB-1N-SN-{config_name}", _block_mappings(cfg["volumes"])) + for inst in (mgmt, sn): + inst.wait_until_running() + inst.reload() + return mgmt, sn + + +# --- device preparation ------------------------------------------------------- + +def data_disks(sn_ip): + """Non-root whole disks on the SN, name -> size_bytes.""" + out = ssh_exec(sn_ip, ["lsblk -bdno NAME,SIZE,TYPE"], get_output=True)[0] + root = ssh_exec(sn_ip, ["lsblk -no PKNAME $(findmnt -no SOURCE /) | head -1"], + get_output=True)[0].strip() + disks = {} + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 3 and parts[2] == "disk" and parts[0] != root: + disks[parts[0]] = int(parts[1]) + return disks + + +def make_partitions(sn_ip, disk, count): + """Create `count` equal GPT partitions on /dev/; return names.""" + cmds = [f"sudo sgdisk --zap-all /dev/{disk}"] + step = 100 // count + for i in range(count): + start = f"{i * step}%" + end = f"{(i + 1) * step}%" if i < count - 1 else "100%" + cmds.append(f"sudo parted -s /dev/{disk} mkpart p{i + 1} {start} {end}") + cmds.insert(1, f"sudo parted -s /dev/{disk} mklabel gpt") + cmds += [f"sudo partprobe /dev/{disk}", "sudo udevadm settle -t 10"] + ssh_exec(sn_ip, cmds, check=True) + out = ssh_exec(sn_ip, [f"lsblk -no NAME /dev/{disk} --raw"], get_output=True)[0] + names = [n for n in out.split() if n != disk] + if len(names) != count: + raise RuntimeError(f"expected {count} partitions on {disk}, got {names}") + return names + + +# --- deployment --------------------------------------------------------------- + +def install_sbcli(ips): + cmds = [ + "sudo dnf install git python3-pip nvme-cli fio gdisk parted -y", + "sudo /usr/bin/python3 -m pip install --upgrade pip setuptools wheel", + "sudo /usr/bin/python3 -m pip install ruamel.yaml", + f"sudo pip install git+https://github.com/simplyblock-io/sbcli@{BRANCH}" + " --upgrade --force --ignore-installed requests", + ] + with ThreadPoolExecutor(max_workers=len(ips)) as ex: + for t in [ex.submit(ssh_exec, ip, cmds, False, True) for ip in ips]: + t.result() + + +def get_cluster_uuid(mgmt_ip): + out = ssh_exec(mgmt_ip, [f"{SBCTL} cluster list"], get_output=True)[0] + m = re.search(r"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})", out) + if not m: + raise RuntimeError(f"no cluster uuid in: {out}") + return m.group(1) + + +def get_storage_node_uuid(mgmt_ip): + out = ssh_exec(mgmt_ip, [f"{SBCTL} sn list"], get_output=True)[0] + m = re.search(r"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})", out) + if not m: + raise RuntimeError(f"no storage node uuid in: {out}") + return m.group(1) + + +def deploy(config_name, keep_metadata_path=None): + cfg = CONFIGS[config_name] + print(f"=== single-node lblk deploy: config={config_name}, " + f"branch={BRANCH}, ec={cfg['ec']} ===") + + mgmt, sn = launch_instances(config_name) + mgmt_ip, sn_ip = mgmt.public_ip_address, sn.public_ip_address + sn_priv_ip = sn.private_ip_address + print(f"mgmt={mgmt_ip} sn={sn_ip} (priv {sn_priv_ip})") + for ip in (mgmt_ip, sn_ip): + wait_for_ssh(ip) + + install_sbcli([mgmt_ip, sn_ip]) + + ndcs, npcs = cfg["ec"] + ssh_exec(mgmt_ip, [ + f"{SBCTL} cluster create" + " --device-mode lblk --is-single-node" + f" --data-chunks-per-stripe {ndcs} --parity-chunks-per-stripe {npcs}" + ], check=True, timeout=2400) + + # configure: whole disks (auto-selection) or explicit partitions + if cfg["partitions"] is None: + configure = f"{SBCTL} sn configure --max-subsys {MAX_LVOL} --lblk" + else: + disks = data_disks(sn_ip) + disk = max(disks, key=lambda d: disks[d]) + names = make_partitions(sn_ip, disk, cfg["partitions"]) + configure = (f"{SBCTL} sn configure --max-subsys {MAX_LVOL} --lblk" + f" --blk-names {','.join(names)}") + ssh_exec(sn_ip, [configure], check=True) + + ssh_exec(sn_ip, [f"{SBCTL} sn deploy --isolate-cores --ifname {IFACE}"], + check=True) + ssh_exec(sn_ip, ["sudo reboot"]) + time.sleep(30) + wait_for_ssh(sn_ip) + print("SN back after reboot; waiting for SNodeAPI...") + time.sleep(60) + + cluster_uuid = get_cluster_uuid(mgmt_ip) + for attempt in range(5): + try: + ssh_exec(mgmt_ip, [ + f"{SBCTL} sn add-node {cluster_uuid} {sn_priv_ip}:5000 {IFACE}" + " --enable-journal-device" + ], check=True, timeout=1800) + break + except RuntimeError: + if attempt == 4: + raise + print(f" retrying add-node in 30s ({attempt + 2}/5)") + time.sleep(30) + + sn_list = ssh_exec(mgmt_ip, [f"{SBCTL} sn list"], get_output=True)[0] + if "online" not in sn_list: + raise RuntimeError(f"storage node not online:\n{sn_list}") + + ssh_exec(mgmt_ip, [f"{SBCTL} cluster activate {cluster_uuid}"], + check=True, timeout=2400) + ssh_exec(mgmt_ip, [f"{SBCTL} pool add pool01 {cluster_uuid}"], check=True) + + meta = { + "config": config_name, + "cluster_uuid": cluster_uuid, + "node_uuid": get_storage_node_uuid(mgmt_ip), + "mgmt_ip": mgmt_ip, + "sn_ip": sn_ip, + "sn_private_ip": sn_priv_ip, + "instance_ids": [mgmt.id, sn.id], + "ec": list(cfg["ec"]), + "branch": BRANCH, + } + path = keep_metadata_path or os.path.join( + os.path.dirname(os.path.abspath(__file__)), + f"cluster_metadata_single_node_{config_name}.json") + with open(path, "w") as f: + json.dump(meta, f, indent=2) + print(f"=== deploy DONE: {json.dumps(meta, indent=2)}") + return meta + + +def terminate(meta): + ec2 = boto3.client("ec2", region_name=REGION) + print(f"Terminating {meta['instance_ids']}") + ec2.terminate_instances(InstanceIds=meta["instance_ids"]) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--config", choices=sorted(CONFIGS), required=True) + ap.add_argument("--metadata", help="metadata output path") + args = ap.parse_args() + deploy(args.config, args.metadata) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/single_node_partition_soak.py b/scripts/single_node_partition_soak.py new file mode 100644 index 0000000000..9b72d9c15c --- /dev/null +++ b/scripts/single_node_partition_soak.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Single-node lblk soak across the three device configurations. + +For each config (2ssd, 2part, 4part — see deploy_single_node_lblk.py): + 1. deploy a fresh 1-node cluster (non-HA, single journal), + 2. create lvols and connect them on the mgmt instance (nvme-tcp client), + 3. lay down a crc32c-verified data region on each volume, then run a + mixed random-IO workload on a separate region, + 4. gracefully restart the storage node (`sn restart`), wait for the node + to come back online and the cluster to return to active, + 5. re-run the crc32c verify-only pass over the phase-3 region — data must + be available and UNCORRUPTED after the restart — plus a short mixed + workload to prove the volume is writable again, + 6. tear the fleet down (kept running on failure for debugging). + +Exit code 0 only if every configured config passes every check. + +Usage: + ./single_node_partition_soak.py # all three configs + ./single_node_partition_soak.py --configs 2part,4part + ./single_node_partition_soak.py --keep # never terminate fleets +""" +import argparse +import json +import os +import re +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from deploy_single_node_lblk import ( # noqa: E402 + CONFIGS, SBCTL, deploy, ssh_exec, terminate, +) + +LVOL_COUNT = 2 +LVOL_SIZE = "20G" +VERIFY_REGION = "4G" # crc32c-stamped region checked across the restart +MIX_REGION_OFFSET = "5G" # mixed workload region, disjoint from the verify one +MIX_RUNTIME_S = 120 +NODE_RESTART_TIMEOUT_S = 1200 + +FIO_COMMON = ("--direct=1 --ioengine=libaio --group_reporting --time_based=0" + " --randrepeat=0 --thread") + + +def _uuids(text): + return re.findall( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", text) + + +def create_and_connect_lvols(meta): + """Create LVOL_COUNT lvols and nvme-connect them on the mgmt instance. + Returns [(lvol_uuid, /dev/nvmeXnY), ...].""" + mgmt = meta["mgmt_ip"] + ssh_exec(mgmt, ["sudo modprobe nvme-tcp"], check=True) + lvols = [] + for i in range(LVOL_COUNT): + name = f"soak_vol{i + 1}" + ssh_exec(mgmt, [f"{SBCTL} lvol add {name} {LVOL_SIZE} pool01"], + check=True, timeout=600) + out = ssh_exec(mgmt, [f"{SBCTL} lvol list | grep {name}"], + get_output=True)[0] + ids = _uuids(out) + if not ids: + raise RuntimeError(f"no uuid for lvol {name}: {out}") + lvols.append(ids[0]) + + devices = [] + for lvol_id in lvols: + before = set(ssh_exec( + mgmt, ["ls /dev/nvme*n1 2>/dev/null || true"], + get_output=True)[0].split()) + connect_cmds = ssh_exec( + mgmt, [f"{SBCTL} lvol connect {lvol_id}"], get_output=True)[0] + ran = False + for line in connect_cmds.splitlines(): + line = line.strip() + if line.startswith("sudo nvme connect") or line.startswith("nvme connect"): + cmd = line if line.startswith("sudo") else f"sudo {line}" + # generous loss tolerance: the restart window must not drop + # the controller + if "ctrl-loss-tmo" not in cmd: + cmd += " --ctrl-loss-tmo=600" + ssh_exec(mgmt, [cmd], check=True) + ran = True + if not ran: + raise RuntimeError(f"lvol connect emitted no nvme connect command:" + f"\n{connect_cmds}") + time.sleep(3) + after = set(ssh_exec( + mgmt, ["ls /dev/nvme*n1 2>/dev/null || true"], + get_output=True)[0].split()) + new = sorted(after - before) + if len(new) != 1: + raise RuntimeError(f"expected one new nvme device, got {new}") + devices.append((lvol_id, new[0])) + print(f" lvol {lvol_id[:8]} -> {new[0]}") + return devices + + +def fio_verify_write(mgmt, dev): + ssh_exec(mgmt, [ + f"sudo fio --name=stamp {FIO_COMMON} --filename={dev} --rw=write" + f" --bs=256k --iodepth=8 --size={VERIFY_REGION}" + " --verify=crc32c --do_verify=0 --verify_state_save=0" + ], check=True, timeout=3600) + + +def fio_verify_read(mgmt, dev): + """crc32c verify-only pass over the stamped region — fails on any + corruption or read error.""" + ssh_exec(mgmt, [ + f"sudo fio --name=check {FIO_COMMON} --filename={dev} --rw=read" + f" --bs=256k --iodepth=8 --size={VERIFY_REGION}" + " --verify=crc32c --verify_only --verify_fatal=1 --verify_state_save=0" + ], check=True, timeout=3600) + + +def fio_mixed(mgmt, dev, runtime=MIX_RUNTIME_S): + ssh_exec(mgmt, [ + f"sudo fio --name=mix {FIO_COMMON} --filename={dev} --rw=randrw" + f" --rwmixread=70 --bs=16k --iodepth=16" + f" --offset={MIX_REGION_OFFSET} --size=4G" + f" --time_based=1 --runtime={runtime}" + ], check=True, timeout=runtime + 600) + + +def restart_storage_node(meta): + mgmt, node_id = meta["mgmt_ip"], meta["node_uuid"] + print(f" restarting storage node {node_id[:8]} ...") + ssh_exec(mgmt, [f"{SBCTL} sn restart {node_id}"], check=True, + timeout=NODE_RESTART_TIMEOUT_S) + deadline = time.time() + NODE_RESTART_TIMEOUT_S + while time.time() < deadline: + sn_list = ssh_exec(mgmt, [f"{SBCTL} sn list"], get_output=True)[0] + status = ssh_exec(mgmt, [f"{SBCTL} cluster list"], get_output=True)[0] + if "online" in sn_list and "active" in status: + print(" node online, cluster active") + return + time.sleep(15) + raise RuntimeError("storage node did not return to online/active in time") + + +def check_health(meta): + mgmt = meta["mgmt_ip"] + sn_list = ssh_exec(mgmt, [f"{SBCTL} sn list"], get_output=True)[0] + if "online" not in sn_list: + raise RuntimeError(f"node not online:\n{sn_list}") + lvol_list = ssh_exec(mgmt, [f"{SBCTL} lvol list"], get_output=True)[0] + for i in range(LVOL_COUNT): + if f"soak_vol{i + 1}" not in lvol_list: + raise RuntimeError(f"lvol soak_vol{i + 1} missing:\n{lvol_list}") + + +def run_config(config_name, keep=False): + print(f"\n########## config {config_name} ##########") + meta = deploy(config_name) + try: + devices = create_and_connect_lvols(meta) + + print("--- phase A: stamp crc32c regions + mixed workload ---") + for _, dev in devices: + fio_verify_write(meta["mgmt_ip"], dev) + for _, dev in devices: + fio_mixed(meta["mgmt_ip"], dev) + for _, dev in devices: + fio_verify_read(meta["mgmt_ip"], dev) + print("--- phase A OK (pre-restart data verified) ---") + + print("--- phase B: node restart ---") + restart_storage_node(meta) + check_health(meta) + + print("--- phase C: post-restart integrity ---") + time.sleep(10) # allow nvme reconnect to settle + for _, dev in devices: + fio_verify_read(meta["mgmt_ip"], dev) + for _, dev in devices: + fio_mixed(meta["mgmt_ip"], dev, runtime=60) + check_health(meta) + print(f"--- config {config_name} PASSED ---") + result = True + except Exception as e: + print(f"!!! config {config_name} FAILED: {e}") + print(f" fleet kept for debugging: {json.dumps(meta, indent=2)}") + return False, meta + if not keep: + terminate(meta) + return result, meta + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--configs", default=",".join(sorted(CONFIGS)), + help="comma-separated subset of: " + ",".join(sorted(CONFIGS))) + ap.add_argument("--keep", action="store_true", + help="keep fleets running even on success") + args = ap.parse_args() + + configs = [c.strip() for c in args.configs.split(",") if c.strip()] + unknown = set(configs) - set(CONFIGS) + if unknown: + ap.error(f"unknown config(s): {sorted(unknown)}") + + results = {} + for config_name in configs: + ok, _meta = run_config(config_name, keep=args.keep) + results[config_name] = ok + + print("\n========== single-node partition soak summary ==========") + for config_name, ok in results.items(): + print(f" {config_name}: {'PASS' if ok else 'FAIL'}") + return 0 if all(results.values()) else 1 + + +if __name__ == "__main__": + sys.exit(main()) From 4b2b90a5bb135a1e1a6f010a9af0af3e0c5f40b6 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 14 Aug 2026 19:32:55 +0200 Subject: [PATCH 8/9] Fix activation crash when no capacity record exists yet; pin CP image in soak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cluster activate read records[0]['size_total'] unguarded. On a freshly deployed cluster the capacity collector has not necessarily run yet — a single-node deployment reaches activation seconds after add-node — so the read raised a bare "list index out of range" and activation aborted. Fall back to the raw sum of the online data devices (create_lvstore takes max_size but sizes its distribs from DISTRIB_SIZE_BYTES; the value only feeds the reported cluster_max_size). Soak scripts, from the first AWS run: - pin SIMPLY_BLOCK_DOCKER_IMAGE on cluster create / sn deploy / sn add-node, resolved from the repo HEAD and verified on ECR. A control-plane stack from an older image DROPS unknown model fields on read-modify-write, so device_mode=lblk silently reverted to nvme and add-node then refused the lblk node config. - fio verify pass must replay the write job with --verify_only (fio then skips the writes and only reads back); the previous --rw=read pass would have verified nothing. - install from github.com/simplyblock/sbcli (the old org name only resolves through a redirect). --- scripts/deploy_single_node_lblk.py | 64 ++++++++++++++++++++++++--- scripts/single_node_partition_soak.py | 18 +++++--- simplyblock_core/cluster_ops.py | 16 ++++++- 3 files changed, 86 insertions(+), 12 deletions(-) diff --git a/scripts/deploy_single_node_lblk.py b/scripts/deploy_single_node_lblk.py index c6f8c14a81..493d185e75 100644 --- a/scripts/deploy_single_node_lblk.py +++ b/scripts/deploy_single_node_lblk.py @@ -46,6 +46,8 @@ INSTANCE_TYPE = os.environ.get("SB_INSTANCE_TYPE", "m6i.2xlarge") EBS_IOPS, EBS_TPUT = 6000, 500 +ECR_REPO = "public.ecr.aws/simply-block/simplyblock" + CONFIGS = { # volumes: (size_gb, ...) attached beyond the 30G root # partitions: None = whole disks; N = create N equal GPT partitions on @@ -56,6 +58,54 @@ "4part": {"volumes": (220,), "partitions": 4, "ec": (2, 1)}, } +def resolve_cp_image(): + """Control-plane image to run the cluster with. + + MUST match the branch the CLI is installed from: a CP stack from an + image that predates a model field DROPS that field on every + read-modify-write — a cluster created with device_mode=lblk silently + reverts to nvme within minutes under an older stack (2026-08-05), and + add-node then refuses the lblk node config. Derived from the repo HEAD + unless SB_CP_IMAGE overrides, and verified to exist on ECR. + """ + override = os.environ.get("SB_CP_IMAGE") + if override: + return override + import subprocess + sha = subprocess.check_output( + ["git", "rev-parse", "--short=8", "HEAD"], + cwd=os.path.dirname(os.path.abspath(__file__)), + ).decode().strip() + tag = f"{BRANCH}-{sha}" + if not ecr_tag_exists(tag): + raise RuntimeError( + f"CP image {ECR_REPO}:{tag} not published yet (CI still building " + f"HEAD {sha}?). Wait for CI, or set SB_CP_IMAGE to a published " + f"image that carries the same model fields.") + return f"{ECR_REPO}:{tag}" + + +def ecr_tag_exists(tag): + import urllib.error + import urllib.request + token = json.load(urllib.request.urlopen( + "https://public.ecr.aws/token/?scope=repository:" + "simply-block/simplyblock:pull", timeout=30))["token"] + req = urllib.request.Request( + f"https://public.ecr.aws/v2/simply-block/simplyblock/manifests/{tag}", + headers={"Authorization": f"Bearer {token}", + "Accept": "application/vnd.docker.distribution.manifest.v2+json, " + "application/vnd.oci.image.index.v1+json, " + "application/vnd.oci.image.manifest.v1+json"}) + try: + urllib.request.urlopen(req, timeout=30) + return True + except urllib.error.HTTPError as e: + if e.code == 404: + return False + raise + + SBCTL = "sudo /usr/local/bin/sbctl -d" @@ -179,7 +229,7 @@ def install_sbcli(ips): "sudo dnf install git python3-pip nvme-cli fio gdisk parted -y", "sudo /usr/bin/python3 -m pip install --upgrade pip setuptools wheel", "sudo /usr/bin/python3 -m pip install ruamel.yaml", - f"sudo pip install git+https://github.com/simplyblock-io/sbcli@{BRANCH}" + f"sudo pip install git+https://github.com/simplyblock/sbcli@{BRANCH}" " --upgrade --force --ignore-installed requests", ] with ThreadPoolExecutor(max_workers=len(ips)) as ex: @@ -205,8 +255,12 @@ def get_storage_node_uuid(mgmt_ip): def deploy(config_name, keep_metadata_path=None): cfg = CONFIGS[config_name] + cp_image = resolve_cp_image() + # Every command that starts or joins control-plane containers must pin + # the image (see resolve_cp_image). + sbctl_img = f"sudo SIMPLY_BLOCK_DOCKER_IMAGE={cp_image} /usr/local/bin/sbctl -d" print(f"=== single-node lblk deploy: config={config_name}, " - f"branch={BRANCH}, ec={cfg['ec']} ===") + f"branch={BRANCH}, ec={cfg['ec']}, image={cp_image} ===") mgmt, sn = launch_instances(config_name) mgmt_ip, sn_ip = mgmt.public_ip_address, sn.public_ip_address @@ -219,7 +273,7 @@ def deploy(config_name, keep_metadata_path=None): ndcs, npcs = cfg["ec"] ssh_exec(mgmt_ip, [ - f"{SBCTL} cluster create" + f"{sbctl_img} cluster create" " --device-mode lblk --is-single-node" f" --data-chunks-per-stripe {ndcs} --parity-chunks-per-stripe {npcs}" ], check=True, timeout=2400) @@ -235,7 +289,7 @@ def deploy(config_name, keep_metadata_path=None): f" --blk-names {','.join(names)}") ssh_exec(sn_ip, [configure], check=True) - ssh_exec(sn_ip, [f"{SBCTL} sn deploy --isolate-cores --ifname {IFACE}"], + ssh_exec(sn_ip, [f"{sbctl_img} sn deploy --isolate-cores --ifname {IFACE}"], check=True) ssh_exec(sn_ip, ["sudo reboot"]) time.sleep(30) @@ -247,7 +301,7 @@ def deploy(config_name, keep_metadata_path=None): for attempt in range(5): try: ssh_exec(mgmt_ip, [ - f"{SBCTL} sn add-node {cluster_uuid} {sn_priv_ip}:5000 {IFACE}" + f"{sbctl_img} sn add-node {cluster_uuid} {sn_priv_ip}:5000 {IFACE}" " --enable-journal-device" ], check=True, timeout=1800) break diff --git a/scripts/single_node_partition_soak.py b/scripts/single_node_partition_soak.py index 9b72d9c15c..1d6fe963e2 100644 --- a/scripts/single_node_partition_soak.py +++ b/scripts/single_node_partition_soak.py @@ -98,11 +98,18 @@ def create_and_connect_lvols(meta): return devices +# The verify pass must REPLAY the write job with --verify_only: fio then +# skips the writes and only reads back and checks the crc32c headers it +# wrote. A --rw=read job would not reproduce the same pattern layout and +# would verify nothing, so both jobs share one parameter string. +_FIO_VERIFY_JOB = (f"--name=stamp {FIO_COMMON} --rw=write --bs=256k" + f" --iodepth=8 --size={VERIFY_REGION}" + " --verify=crc32c --verify_state_save=0") + + def fio_verify_write(mgmt, dev): ssh_exec(mgmt, [ - f"sudo fio --name=stamp {FIO_COMMON} --filename={dev} --rw=write" - f" --bs=256k --iodepth=8 --size={VERIFY_REGION}" - " --verify=crc32c --do_verify=0 --verify_state_save=0" + f"sudo fio {_FIO_VERIFY_JOB} --filename={dev} --do_verify=0" ], check=True, timeout=3600) @@ -110,9 +117,8 @@ def fio_verify_read(mgmt, dev): """crc32c verify-only pass over the stamped region — fails on any corruption or read error.""" ssh_exec(mgmt, [ - f"sudo fio --name=check {FIO_COMMON} --filename={dev} --rw=read" - f" --bs=256k --iodepth=8 --size={VERIFY_REGION}" - " --verify=crc32c --verify_only --verify_fatal=1 --verify_state_save=0" + f"sudo fio {_FIO_VERIFY_JOB} --filename={dev}" + " --verify_only --verify_fatal=1" ], check=True, timeout=3600) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 8d53512ce2..fd5952b55c 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -984,6 +984,7 @@ def _cluster_activate(cl_id, force=False, force_lvstore_create=False) -> None: online_nodes = [] dev_count = 0 + raw_device_size = 0 for node in snodes: if node.is_secondary_node: # pass continue @@ -993,6 +994,7 @@ def _cluster_activate(cl_id, force=False, force_lvstore_create=False) -> None: if dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE]: dev_count += 1 + raw_device_size += int(dev.size or 0) single_node_cluster = is_single_node_activation(cluster, online_nodes) if single_node_cluster and cluster.ha_type == "ha": logger.warning("Single-node cluster: activating as non-HA " @@ -1103,8 +1105,20 @@ def _fd_fail(msg: str) -> None: node.enable_ha_jm = False node.write_to_db() + # Cluster raw capacity, for the reported cluster_max_size (create_lvstore + # takes it but sizes its distribs from DISTRIB_SIZE_BYTES instead). The + # capacity collector has not necessarily run yet on a freshly deployed + # cluster — a single-node deployment reaches activation seconds after + # add-node — and the unguarded records[0] aborted activation with a bare + # "list index out of range". Fall back to the raw device sum. records = db_controller.get_cluster_capacity(cluster) - max_size = records[0]['size_total'] + if records: + max_size = records[0]['size_total'] + else: + max_size = raw_device_size + logger.warning( + "No cluster capacity record yet (stats collector has not run); " + "using the raw online-device sum %s as cluster max size", max_size) used_nodes_as_sec: t.List[str] = [] used_nodes_as_tertiary: t.List[str] = [] From 0640f322300400e3c1a97e629a6e921b34b42f97 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 14 Aug 2026 20:59:55 +0200 Subject: [PATCH 9/9] Fix no-parity clusters reporting DEGRADED while healthy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_next_cluster_status treated "affected_nodes == distr_npcs" as being at the parity limit. With npcs=0 (EC 1+0, the natural single-node schema) k is 0, so the condition matched on affected_nodes == 0 — a perfectly healthy cluster reported DEGRADED forever. Observed on the live 1-node soak cluster 2026-08-14: single data device online, JM online, node health True, cluster DEGRADED. Require at least one affected node before the limit rule applies; the npcs>=1 paths are unchanged (the pre-existing failures in tests/integration/test_cluster_suspend_recovery.py are identical with and without this change). Also make the soak's restart wait case-insensitive: `sn list` prints the node status lowercase but `cluster list` prints ACTIVE, so the wait spun until timeout on restarts that had actually succeeded. --- scripts/single_node_partition_soak.py | 4 +- .../services/storage_node_monitor.py | 8 +- tests/unit/test_single_node_cluster.py | 81 +++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/scripts/single_node_partition_soak.py b/scripts/single_node_partition_soak.py index 1d6fe963e2..9db93c8ca3 100644 --- a/scripts/single_node_partition_soak.py +++ b/scripts/single_node_partition_soak.py @@ -140,7 +140,9 @@ def restart_storage_node(meta): while time.time() < deadline: sn_list = ssh_exec(mgmt, [f"{SBCTL} sn list"], get_output=True)[0] status = ssh_exec(mgmt, [f"{SBCTL} cluster list"], get_output=True)[0] - if "online" in sn_list and "active" in status: + # `sn list` prints the node status lowercase, `cluster list` prints + # the cluster status uppercase (ACTIVE) — compare case-insensitively. + if "online" in sn_list.lower() and "active" in status.lower(): print(" node online, cluster active") return time.sleep(15) diff --git a/simplyblock_core/services/storage_node_monitor.py b/simplyblock_core/services/storage_node_monitor.py index cfa7970c96..35ceb3d7b5 100644 --- a/simplyblock_core/services/storage_node_monitor.py +++ b/simplyblock_core/services/storage_node_monitor.py @@ -398,7 +398,13 @@ def get_next_cluster_status(cluster_id): return fd_status # if number of devices in the cluster unavailable on DIFFERENT nodes > k --> I cannot read and in some cases cannot write (suspended) - if affected_nodes == k and (not cluster.strict_node_anti_affinity or online_nodes >= (n + k)): + # + # affected_nodes > 0 guard: "we are exactly at the parity limit" only + # means degraded when something is actually affected. Without it a + # no-parity cluster (npcs=0, k=0 — the natural single-node schema) + # reports DEGRADED while perfectly healthy, because 0 == 0. + if affected_nodes > 0 and affected_nodes == k and ( + not cluster.strict_node_anti_affinity or online_nodes >= (n + k)): return Cluster.STATUS_DEGRADED elif jm_replication_tasks: return Cluster.STATUS_DEGRADED diff --git a/tests/unit/test_single_node_cluster.py b/tests/unit/test_single_node_cluster.py index 26e07560cf..28a3452e6b 100644 --- a/tests/unit/test_single_node_cluster.py +++ b/tests/unit/test_single_node_cluster.py @@ -167,6 +167,87 @@ def test_minimum_devices_ec21(self): self.assertEqual(cluster_ops.activation_minimum_devices(cluster, False), 4) +class TestNoParityClusterStatus(unittest.TestCase): + """A cluster with npcs=0 (EC 1+0 — the natural single-node schema) has + k=0, so the "we are exactly at the parity limit" DEGRADED rule matched + on affected_nodes == 0, i.e. while perfectly healthy. Live 1-node + cluster 2026-08-14 sat in DEGRADED with every device online.""" + + def _status(self, nodes, ndcs, npcs): + from simplyblock_core.services import storage_node_monitor as mod + cluster = MagicMock(spec=Cluster) + cluster.uuid = "cluster-1" + cluster.status = Cluster.STATUS_ACTIVE + cluster.distr_ndcs = ndcs + cluster.distr_npcs = npcs + cluster.max_fault_tolerance = max(npcs, 1) + cluster.strict_node_anti_affinity = False + cluster.enable_failure_domain = False + cluster.suspend_drain_complete = False + cluster.get_id = MagicMock(return_value="cluster-1") + with patch.object(mod, "db") as mock_db: + mock_db.get_cluster_by_id.return_value = cluster + mock_db.get_primary_storage_nodes_by_cluster_id.return_value = nodes + mock_db.get_job_tasks.return_value = [] + return mod.get_next_cluster_status("cluster-1") + + def _mock_node(self, uuid, status=StorageNode.STATUS_ONLINE, online_devs=1, + offline_devs=0): + from simplyblock_core.models.nvme_device import NVMeDevice + n = MagicMock(spec=StorageNode) + n.status = status + n.cluster_id = "cluster-1" + n.mgmt_ip = f"10.0.0.{abs(hash(uuid)) % 250 + 1}" + n.auto_restart_disabled = False + n.failure_domain = -1 + n.jm_vuid = 1 + n.rpc_port = 8080 + n.online_since = "" + n.down_since = "" + n.lvstore = "LVS_1" + n.lvstore_status = "ready" + n.get_id = MagicMock(return_value=uuid) + + def _dev(st, did): + d = MagicMock(spec=NVMeDevice) + d.status = st + d.get_id = MagicMock(return_value=did) + return d + + n.nvme_devices = ( + [_dev(NVMeDevice.STATUS_ONLINE, f"{uuid}-on-{i}") for i in range(online_devs)] + + [_dev(NVMeDevice.STATUS_UNAVAILABLE, f"{uuid}-off-{i}") + for i in range(offline_devs)]) + return n + + def test_healthy_single_node_no_parity_is_active(self): + node = self._mock_node("node-1", online_devs=1) + self.assertEqual(self._status([node], ndcs=1, npcs=0), + Cluster.STATUS_ACTIVE) + + def test_healthy_multi_node_no_parity_is_active(self): + nodes = [self._mock_node("node-1"), self._mock_node("node-2")] + self.assertEqual(self._status(nodes, ndcs=1, npcs=0), + Cluster.STATUS_ACTIVE) + + def test_no_parity_with_a_failed_device_is_not_active(self): + # k=0 tolerates nothing: an affected node must still leave ACTIVE. + nodes = [self._mock_node("node-1", online_devs=1), + self._mock_node("node-2", online_devs=1, offline_devs=1)] + self.assertNotEqual(self._status(nodes, ndcs=1, npcs=0), + Cluster.STATUS_ACTIVE) + + def test_healthy_parity_cluster_unaffected_by_the_guard(self): + # npcs>=1 with nothing affected was ACTIVE before and stays ACTIVE. + # The k>0 "exactly at the limit -> DEGRADED" path needs the + # data-plane probe fixtures and is covered in + # tests/integration/test_cluster_suspend_recovery.py. + nodes = [self._mock_node("node-1", online_devs=2), + self._mock_node("node-2", online_devs=2)] + self.assertEqual(self._status(nodes, ndcs=1, npcs=1), + Cluster.STATUS_ACTIVE) + + class TestCheckSnapHaGating(unittest.TestCase): def _run_check_snap(self, ha_type, secondary_node_id):