inline-checksum: optional per-cluster CRC validation (rebased onto main, TD.100226.1)

scripts/setup_lab_perf_test1.py

@@ -99,7 +99,7 @@ def wait_for_ssh(ip, timeout=300):
     return False
 
 
-def ssh_exec(ip, cmds, get_output=False, check=False):
+def ssh_exec(ip, cmds, get_output=False, check=False, timeout=600):
     results = []
     for cmd in cmds:
         print(f"  [{ip}] $ {cmd}")
@@ -108,7 +108,7 @@ def ssh_exec(ip, cmds, get_output=False, check=False):
             env=_ssh_env(),
             capture_output=True,
             text=True,
-            timeout=600,
+            timeout=timeout,
         )
         out = proc.stdout
         err = proc.stderr

scripts/setup_perf_test1.py

@@ -19,7 +19,7 @@
 KEY_PATH = os.path.expanduser("~/.ssh/mtes01.pem")
 AZ = "us-east-1a"
 SG_NAME = "default"
-BRANCH = "main"
+BRANCH = "inline-checksum-validation"
 MAX_LVOL = "100"
 # --- Manual Network Config ---
 # Replace this with your actual Subnet ID (e.g., "subnet-0593459d6b931ee4c")
@@ -408,7 +408,7 @@ def main():
         "sudo dnf install git python3-pip nvme-cli -y",
         "sudo /usr/bin/python3 -m pip install --upgrade pip setuptools wheel",
         "sudo /usr/bin/python3 -m pip install ruamel.yaml",
-        "sudo pip install git+https://github.com/simplyblock-io/sbcli@main --upgrade --force --ignore-installed requests",
+        "sudo pip install git+https://github.com/simplyblock-io/sbcli@inline-checksum-validation --upgrade --force --ignore-installed requests",
         "echo 'export PATH=/usr/local/bin:$PATH' >> ~/.bashrc"
     ]
 
@@ -422,11 +422,21 @@ def main():
     # --- 5. Cluster Configuration (Phase 2) ---
     # Step 5a: Create cluster on mgmt (sequential, must complete first)
     print("Phase 2a: Creating cluster on management node...")
+    # --enable-inline-checksum is frozen at cluster create time (no
+    # mutator, no upgrade path). Pair with --enable-inline-checksum on
+    # `sn configure` below so the drives format to an LBAF with NVMe
+    # metadata (ms>=8) and alceml can run in md-on-device mode.
     ssh_exec(mgmt_ip, [
         # --dev enables developer-mode-only (private) args like --enable-hang-device.
         f"sudo /usr/local/bin/sbctl -d --dev cluster create --enable-node-affinity --max-subsys {MAX_LVOL}"
         " --enable-hang-device"
         " --data-chunks-per-stripe 2 --parity-chunks-per-stripe 2"
+        " --enable-inline-checksum"
+        # AWS NVMe is 512B logical but atomic at 4K, so it has no md-capable
+        # LBAF and never reformats to 4K. --4k_atomic lets fallback-mode inline
+        # checksum run on these 512B devices: it makes alceml create send
+        # cv_ignore_block_size=True so the data plane skips its >=4K block-size gate.
+        " --4k_atomic"
         # Swarm stack deploy inside cluster create pulls the full CP image set;
         # observed >10 min on cold registry pulls (2026-07-10, two deploys died
         # on the default 600 s channel timeout at "Deploying swarm stack").
@@ -435,9 +445,15 @@ def main():
 
     # Step 5b: Configure and deploy storage nodes in parallel
     print("Phase 2b: Configuring storage nodes...")
+    # --enable-inline-checksum here drives `sn configure`'s formatter to
+    # pick the smallest LBAF with ds=12, ms>=8 (8B+ NVMe metadata per
+    # 4K block) and force-reformat through the existing 4K-already-set
+    # early-out — required for alceml's md-on-device checksum_method=1.
+    # Devices with no md-capable LBAF format to plain 4K and fall back
+    # to checksum_method=2 (cv_fallback, ~1.17% capacity overhead).
     with ThreadPoolExecutor(max_workers=len(sn_ips)) as executor:
         tasks = [executor.submit(ssh_exec, ip, [
-            "sudo /usr/local/bin/sbctl -d sn configure"
+            "sudo /usr/local/bin/sbctl -d sn configure --enable-inline-checksum"
         ], check=True) for ip in sn_ips]
         for t in tasks:
             t.result()

simplyblock_cli/cli-reference.yaml

@@ -89,6 +89,14 @@ commands:
             dest: force
             type: bool
             action: store_true
+          - name: "--enable-inline-checksum"
+            help: >
+              When formatting (with --force), prefer an LBAF that supports >=8 bytes of NVMe metadata per block,
+              so alceml can run inline checksum validation in md-on-device mode. Drives with no md-capable LBAF
+              still format to plain 4K and will use the fallback layout.
+            dest: inline_checksum
+            type: bool
+            action: store_true
           - name: "--calculate-hp-only"
             help: "Calculate the minimum required huge pages, it depends on the following params: --sockets-to-use, --nodes-per-socket, --number-of-devices. Subsystem count and the vCPU budget are cluster-level settings."
             dest: calculate_hp_only
@@ -974,6 +982,23 @@ commands:
             dest: enable_failure_domain
             type: bool
             action: store_true
+          - name: "--enable-inline-checksum"
+            help: >
+              Enable inline CRC checksum validation on every IO for silent-data-error protection. Cannot be enabled or
+              disabled after cluster creation. Per-device alceml mode (md-on-device vs fallback) is auto-detected at
+              add-node.
+            dest: inline_checksum
+            type: bool
+            action: store_true
+          - name: "--4k_atomic"
+            help: >
+              Declare that devices guarantee 4K write atomicity even with a <4K logical block size (e.g. AWS NVMe is
+              512B but atomic at 4K). Allows fallback-mode inline checksum on such devices by skipping the data-plane
+              4K block-size requirement. Only meaningful with --enable-inline-checksum. Cannot be changed after cluster
+              creation.
+            dest: atomic_4k
+            type: bool
+            action: store_true
           - name: "--name"
             help: >
               Assigns a name to the newly created cluster.
@@ -1180,6 +1205,22 @@ commands:
             dest: enable_failure_domain
             type: bool
             action: store_true
+          - name: "--enable-inline-checksum"
+            help: >
+              Enable inline CRC checksum validation on every IO for silent-data-error protection. Cannot be enabled or
+              disabled after cluster creation.
+            dest: inline_checksum
+            type: bool
+            action: store_true
+          - name: "--4k_atomic"
+            help: >
+              Declare that devices guarantee 4K write atomicity even with a <4K logical block size (e.g. AWS NVMe is
+              512B but atomic at 4K). Allows fallback-mode inline checksum on such devices by skipping the data-plane
+              4K block-size requirement. Only meaningful with --enable-inline-checksum. Cannot be changed after cluster
+              creation.
+            dest: atomic_4k
+            type: bool
+            action: store_true
           - name: "--name"
             help: >
               Assigns a name to the newly created cluster.

simplyblock_cli/cli.py

@@ -106,6 +106,7 @@ def init_storage_node__configure(self, subparser):
         subcommand.add_argument('--size-range', help='NVMe SSD device size range separated by -, can be X(m,g,t) or bytes as integer, example: --size-range 50G-1T or --size-range 1232345-67823987. Can be used alone to filter by size, or combined with --device-model to further filter by model.', type=str, default='', dest='size_range', required=False)
         subcommand.add_argument('--nvme-names', help='Comma separated list of nvme namespace names like nvme0n1,nvme1n1.', type=str, default='', dest='nvme_names', required=False)
         subcommand.add_argument('--force', help='Force format detected or passed nvme pci address to 4K and clean partitions.', dest='force', action='store_true')
+        subcommand.add_argument('--enable-inline-checksum', help='When formatting (with --force), prefer an LBAF that supports >=8 bytes of NVMe metadata per block, so alceml can run inline checksum validation in md-on-device mode. Drives with no md-capable LBAF still format to plain 4K and will use the fallback layout.', dest='inline_checksum', action='store_true')
         subcommand.add_argument('--calculate-hp-only', help='Calculate the minimum required huge pages, it depends on the following params: --cores-percentage, --sockets-to-use, --max-subsys, --nodes-per-socket, --number-of-devices.', dest='calculate_hp_only', action='store_true')
         subcommand.add_argument('--number-of-devices', help='Number of devices that will be used on this host. For calculating huge pages memory only.', type=int, dest='number_of_devices')
 
@@ -440,6 +441,8 @@ def init_cluster__create(self, subparser):
             subcommand.add_argument('--disable-monitoring', help='Disable monitoring stack, false by default. Default: `false`.', dest='disable_monitoring', action='store_true')
         subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster.', dest='strict_node_anti_affinity', action='store_true')
         subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true')
+        subcommand.add_argument('--enable-inline-checksum', help='Enable inline CRC checksum validation on every IO for silent-data-error protection. Cannot be enabled or disabled after cluster creation. Per-device alceml mode (md-on-device vs fallback) is auto-detected at add-node.', dest='inline_checksum', action='store_true')
+        subcommand.add_argument('--4k_atomic', help='Declare that devices guarantee 4K write atomicity even with a <4K logical block size (e.g. AWS NVMe is 512B but atomic at 4K). Allows fallback-mode inline checksum on such devices by skipping the data-plane 4K block-size requirement. Only meaningful with --enable-inline-checksum. Cannot be changed after cluster creation.', dest='atomic_4k', action='store_true')
         subcommand.add_argument('--name', '-n', help='Assigns a name to the newly created cluster.', type=str, dest='name')
         subcommand.add_argument('--qpair-count', help='The NVMe/TCP transport qpair count per logical volume. Default: `32`.', type=range_type(0, 128), default=32, dest='qpair_count')
         subcommand.add_argument('--client-qpair-count', help='The default NVMe/TCP transport qpair count per logical volume for client. Default: `3`.', type=range_type(0, 128), default=3, dest='client_qpair_count')
@@ -480,6 +483,8 @@ def init_cluster__add(self, subparser):
             subcommand.add_argument('--inflight-io-threshold', help='The number of inflight IOs allowed before the IO queuing starts. Default: `4`.', type=int, default=4, dest='inflight_io_threshold')
         subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster."', dest='strict_node_anti_affinity', action='store_true')
         subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true')
+        subcommand.add_argument('--enable-inline-checksum', help='Enable inline CRC checksum validation on every IO for silent-data-error protection. Cannot be enabled or disabled after cluster creation.', dest='inline_checksum', action='store_true')
+        subcommand.add_argument('--4k_atomic', help='Declare that devices guarantee 4K write atomicity even with a <4K logical block size (e.g. AWS NVMe is 512B but atomic at 4K). Allows fallback-mode inline checksum on such devices by skipping the data-plane 4K block-size requirement. Only meaningful with --enable-inline-checksum. Cannot be changed after cluster creation.', dest='atomic_4k', action='store_true')
         subcommand.add_argument('--name', '-n', help='Assigns a name to the newly created cluster.', type=str, dest='name')
         subcommand.add_argument('--client-data-nic', help='Network interface name from client to use for logical volume connection.', type=str, dest='client_data_nic')
         subcommand.add_argument('--use-backup', help='The path to JSON file with S3/MinIO backup configuration.', type=str, dest='use_backup')

simplyblock_cli/clibase.py

@@ -158,7 +158,8 @@ def storage_node__configure(self, sub_command, args):
             max_lvol, max_prov, sockets_to_use, args.nodes_per_socket,
             pci_allowed, pci_blocked, force=args.force, device_model=args.device_model,
             size_range=args.size_range, vcpu_count=vcpu_count, nvme_names=nvme_names,
-            calculate_hp_only=args.calculate_hp_only, number_of_devices=number_of_devices)
+            calculate_hp_only=args.calculate_hp_only, number_of_devices=number_of_devices,
+            inline_checksum=args.inline_checksum)
 
     def storage_node__deploy_cleaner(self, sub_command, args):
         storage_ops.deploy_cleaner()
@@ -1311,6 +1312,7 @@ def cluster_add(self, args):
         is_single_node = args.is_single_node
         client_data_nic = args.client_data_nic
         enable_failure_domain = getattr(args, 'enable_failure_domain', False)
+        inline_checksum = getattr(args, 'inline_checksum', False)
 
         max_fault_tolerance = min(distr_npcs, 2) if distr_npcs >= 1 else 1
 
@@ -1319,6 +1321,7 @@ def cluster_add(self, args):
             with open(args.use_backup, 'r') as f:
                 backup_config = json.load(f)
 
+        atomic_4k = getattr(args, 'atomic_4k', False)
         return cluster_ops.add_cluster(
             blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit,
             distr_ndcs, distr_npcs, distr_bs, distr_chunk_bs, ha_type, enable_node_affinity,
@@ -1327,6 +1330,8 @@ def cluster_add(self, args):
             nvmf_base_port=args.nvmf_base_port, rpc_base_port=args.rpc_base_port, snode_api_port=args.snode_api_port,
             hashicorp_vault_settings=HashicorpVaultSettings({"base_url": args.hashicorp_vault_url}) if args.hashicorp_vault_url else None,
             enable_failure_domain=enable_failure_domain,
+            inline_checksum=inline_checksum,
+            atomic_4k=atomic_4k,
         )
 
     def cluster_create(self, args):
@@ -1366,6 +1371,8 @@ def cluster_create(self, args):
         enable_failure_domain = getattr(args, 'enable_failure_domain', False)
         # Private (developer-mode-only) arg: absent unless sbctl was run with --dev.
         enable_hang_device = getattr(args, "enable_hang_device", False)
+        inline_checksum = getattr(args, 'inline_checksum', False)
+        atomic_4k = getattr(args, 'atomic_4k', False)
 
         max_fault_tolerance = min(distr_npcs, 2) if distr_npcs >= 1 else 1
 
@@ -1390,6 +1397,8 @@ def cluster_create(self, args):
             max_subsys=args.max_subsys or 0,
             hugepages_mem=utils.parse_size(args.hugepages_mem) if args.hugepages_mem else 0,
             spdk_vcpu_count=args.vcpu_count or 0,
+            inline_checksum=inline_checksum,
+            atomic_4k=atomic_4k,
         )
 
     def query_yes_no(self, question, default="yes"):

simplyblock_core/cluster_ops.py

@@ -279,6 +279,8 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
                    enable_failure_domain=False,
                    enable_hang_device=False,
                    max_subsys=0, hugepages_mem=0, spdk_vcpu_count=0,
+                   inline_checksum=False,
+                   atomic_4k=False,
 ) -> str:
     if (distr_ndcs, distr_npcs) not in SUPPORTED_ERASURE_CODING_SCHEMES:
         raise ValueError("Unsupported erasure coding scheme")
@@ -419,6 +421,8 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
     cluster.disable_monitoring = disable_monitoring
     cluster.mode = mode
     cluster.full_page_unmap = False
+    cluster.inline_checksum = bool(inline_checksum)
+    cluster.atomic_4k = bool(atomic_4k)
     cluster.client_data_nic = client_data_nic or ""
     cluster.max_fault_tolerance = max_fault_tolerance
     cluster.nvmf_base_port = nvmf_base_port
@@ -521,6 +525,8 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn
                 nvmf_base_port=4420, rpc_base_port=8080, snode_api_port=50001,
                 hashicorp_vault_settings : t.Optional[HashicorpVaultSettings] = None,
                 enable_failure_domain=False,
+                inline_checksum=False,
+                atomic_4k=False,
 ) -> str:
     """Thin wrapper around _add_cluster_impl() that serializes create calls
     for the same name behind a ClusterCreateLock.
@@ -547,6 +553,8 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn
         client_data_nic=client_data_nic, max_fault_tolerance=max_fault_tolerance, backup_config=backup_config,
         nvmf_base_port=nvmf_base_port, rpc_base_port=rpc_base_port, snode_api_port=snode_api_port,
         hashicorp_vault_settings=hashicorp_vault_settings, enable_failure_domain=enable_failure_domain,
+        inline_checksum=inline_checksum,
+        atomic_4k=atomic_4k,
     )
     if not name:
         return _add_cluster_impl(**kwargs)
@@ -571,6 +579,8 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca
                 nvmf_base_port=4420, rpc_base_port=8080, snode_api_port=50001,
                 hashicorp_vault_settings : t.Optional[HashicorpVaultSettings] = None,
                 enable_failure_domain=False,
+                inline_checksum=False,
+                atomic_4k=False,
 ) -> str:
 
     clusters = db_controller.get_clusters()
@@ -688,6 +698,8 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca
     cluster.fabric_tcp = protocols["tcp"]
     cluster.fabric_rdma = protocols["rdma"]
     cluster.full_page_unmap = False
+    cluster.inline_checksum = bool(inline_checksum)
+    cluster.atomic_4k = bool(atomic_4k)
     cluster.client_data_nic = client_data_nic or ""
     cluster.max_fault_tolerance = max_fault_tolerance
     cluster.nvmf_base_port = nvmf_base_port

simplyblock_core/controllers/device_controller.py

@@ -417,12 +417,21 @@ def _def_create_device_stack(device_obj, snode, force=False, clear_data=False):
     alceml_id = device_obj.get_id()
     alceml_name = get_alceml_name(alceml_id)
     if not rpc_client.get_bdevs(alceml_name):
+        checksum_method, cache_size, cache_eviction_threshold = utils.alceml_checksum_params(cluster, device_obj)
+        if cluster.inline_checksum and not device_obj.md_supported:
+            logger.warning(
+                f"Inline checksum: device {device_obj.get_id()} ({device_obj.pcie_address}) has no NVMe metadata; "
+                f"alceml will run in fallback mode (extra md page, ~1.17%% capacity overhead)."
+            )
         ret = snode.create_alceml(
             alceml_name, nvme_bdev, alceml_id,
             pba_init_mode=3 if clear_data else 2,
             write_protection=cluster.distr_ndcs > 1,
             pba_page_size=cluster.page_size_in_blocks,
-            full_page_unmap=cluster.full_page_unmap
+            full_page_unmap=cluster.full_page_unmap,
+            checksum_method=checksum_method,
+            cache_size=cache_size,
+            cache_eviction_threshold=cache_eviction_threshold,
         )
 
         if not ret: