batch migration checksum issue workaround

simplyblock_core/controllers/migration_controller.py

@@ -1174,7 +1174,8 @@ def create_migration(lvol_id, target_node_id,
                         f"create_migration: listener on overlap {_node_id[:8]} "
                         f"(non-fatal): {_e}")
         else:
-            if not _rpc.subsystem_get(nqn):
+            _existing_subsys = _rpc.subsystem_get(nqn)
+            if not _existing_subsys:
                 if _min_cntlid in subsys_min_cntlid_used:
                     _min_cntlid = _min_cntlid + 10000
                 _rpc.subsystem_create(
@@ -1190,22 +1191,51 @@ def create_migration(lvol_id, target_node_id,
                         f"create_migration: allowed_hosts reapply on "
                         f"{_node_id[:8]} (non-fatal): {_e}")
 
+            # For a shared-namespace batch group, create_migration() runs once
+            # per member against the SAME nqn/listener -- guard against
+            # re-adding a listener that a prior member's precreate already
+            # established (listeners_create had no existence check and its
+            # result was never inspected, so a real failure on a later
+            # member's redundant add would previously have been silent).
+            _existing_listeners = {
+                (_l.get('trtype', '').lower(), _l.get('traddr'), str(_l.get('trsvcid')))
+                for _l in ((_existing_subsys or {}).get('listen_addresses') or [])
+            }
             for nic in _node.data_nics:
                 if not nic.ip4_address or nic.trtype.lower() != lvol.fabric:
                     continue
+                _listener_key = (nic.trtype.lower(), nic.ip4_address, str(_port))
+                if _listener_key in _existing_listeners:
+                    continue
                 try:
-                    _rpc.listeners_create(nqn, nic.trtype.lower(), nic.ip4_address,
-                                          _port, ana_state="inaccessible")
+                    _ret_listener = _rpc.listeners_create(
+                        nqn, nic.trtype.lower(), nic.ip4_address,
+                        _port, ana_state="inaccessible")
+                    if not _ret_listener:
+                        logger.warning(
+                            f"create_migration: listener add for {_node_id[:8]} "
+                            f"{nic.ip4_address}:{_port} returned falsy")
                 except Exception as _e:
                     logger.warning(
                         f"create_migration: listener on {_node_id[:8]} "
                         f"(non-fatal): {_e}")
 
-            _ns = _rpc.nvmf_subsystem_add_ns(nqn, _ns_bdev, lvol.uuid, lvol.guid)
+            # Pin the target namespace to the SAME nsid the source already
+            # uses, rather than letting SPDK auto-assign on the target
+            # subsystem. Auto-assignment just happens to reproduce the
+            # source's numbering when adds land in the same order on an
+            # empty subsystem -- it isn't enforced, and any stale/leftover
+            # namespace occupying a low nsid on the target (or add_ns calls
+            # racing/reordering across nodes) would silently diverge the
+            # source and target nsid maps for this lvol.
+            _ns = _rpc.nvmf_subsystem_add_ns(
+                nqn, _ns_bdev, lvol.uuid, lvol.guid,
+                nsid=lvol.ns_id if lvol.ns_id else None)
             if _ns:
                 logger.info(
                     f"create_migration: namespace {_ns_bdev} added on "
-                    f"{_tgt_label} {_node_id[:8]} nsid={_ns}")
+                    f"{_tgt_label} {_node_id[:8]} nsid={_ns} "
+                    f"(source nsid={lvol.ns_id})")
             else:
                 logger.warning(
                     f"create_migration: nvmf_subsystem_add_ns failed on "

simplyblock_core/models/lvol_migration_group.py

@@ -14,8 +14,14 @@
     N worker tasks copy their owned snapshot chains in parallel.
     snap_copy_done tracks which workers have finished.
 INTERMEDIATE
-    All workers take exactly one intermediate ('shrink') snapshot each.
-    intermediates_done tracks which workers have finished.
+    All workers take one intermediate ('shrink') snapshot each, in lockstep
+    rounds. intermediates_done tracks which workers have finished the
+    current round (intermediate_round). If any worker's dirty delta is
+    still above the threshold after a round, it flags itself in
+    intermediate_more_needed; once every worker has finished the round, the
+    orchestrator starts another synchronized round (all members retake a
+    snapshot together, even ones whose own delta was already low) if
+    intermediate_more_needed is non-empty and the round cap hasn't been hit.
 BATCH_MIGRATE
     Main calls bdev_lvol_batch_final_step with all lvols ordered by ns_id.
     batch_result is set to True on success, False on failure.
@@ -80,10 +86,22 @@ class LVolMigrationGroup(BaseModel):
     # waiting for the INTERMEDIATE phase signal from the main orchestrator.
     snap_copy_done: List[str] = []
 
-    # migration_ids that have taken and transferred their single intermediate
-    # snapshot and are waiting for batch_result.
+    # migration_ids that have taken and transferred their intermediate
+    # snapshot for the CURRENT intermediate_round and are waiting for either
+    # another round or batch_result. Cleared when a new round starts.
     intermediates_done: List[str] = []
 
+    # Which intermediate round is currently in flight (0-indexed; round 0 is
+    # always taken unconditionally). Incremented when the orchestrator starts
+    # another synchronized round.
+    intermediate_round: int = 0
+
+    # migration_ids that reported their dirty delta still exceeded the
+    # threshold after finishing intermediate_round. Cleared when a new round
+    # starts. Non-empty at the end of a round (and under the round cap)
+    # triggers another synchronized round for every member.
+    intermediate_more_needed: List[str] = []
+
     # migration_ids that have completed CLEANUP_SOURCE.
     cleanup_source_done: List[str] = []
 

simplyblock_core/services/hub_controller_manager.py

@@ -93,7 +93,7 @@ class HubControllerManager:
     # Seconds since the last acquire() before the GC triggers a detach.
     # Refreshed on every acquire() so concurrent migrations naturally keep
     # the controller alive without any reference counting.
-    IDLE_TIMEOUT = 300  # 5 minutes
+    IDLE_TIMEOUT = 1200  # 20 minutes
 
     # GC sweep period.
     GC_INTERVAL = 30