From 9f9942ca3b393f151bed840f519de468c5ff8590 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 21 Aug 2026 17:29:21 +0100 Subject: [PATCH 01/10] feat(replication): hold cutover_pending per-task for operator preconnect before ANA flip --- simplyblock_core/constants.py | 3 +++ .../services/tasks_runner_replication_final.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 8ed38500d..cb3e810b4 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -360,6 +360,9 @@ def get_config_var(name, default=None): # (snapshot -> wait replicated -> snapshot -> wait) before the final freeze. # Two rounds normally complete within 2 replication intervals + transfer time. REPL_CUTOVER_SHRINK_TIMEOUT_SEC = 900 +# Time the task holds in cutover_pending so the operator can pre-connect target +# NVMe paths before the ANA flip. Each volume's deadline is independent (non-blocking). +REPL_CUTOVER_PRECONNECT_WAIT_SEC = 10 SPDK_PROXY_MULTI_THREADING_ENABLED=True SPDK_PROXY_TIMEOUT=60*5 diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 697c590cf..33da3c17b 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -121,6 +121,21 @@ def task_runner(task: JobSchedule): return _finalize(task, False, err) params = task.function_params + # Hold in cutover_pending so the operator can pre-connect target NVMe paths + # before the ANA flip. Each volume has its own deadline so this never blocks + # other volumes' cutover tasks. + if "preconnect_deadline" not in params: + params["preconnect_deadline"] = int(time.time()) + constants.REPL_CUTOVER_PRECONNECT_WAIT_SEC + task.function_result = "cutover_pending: waiting for preconnect" + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return False + if int(time.time()) < params["preconnect_deadline"]: + task.function_result = "cutover_pending: waiting for preconnect" + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return False + try: ok, err = replication_final_step.run_cutover( src_node, tgt_node, lvol, From 46347cb0ba066ef9167254e9497b2615c2100619 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 21 Aug 2026 20:31:53 +0100 Subject: [PATCH 02/10] replace fixed preconnect timer with cutover_proceed flag; add POST /replication/cutover-proceed endpoint --- simplyblock_core/constants.py | 7 ++-- .../replication_policy_controller.py | 18 +++++++++ simplyblock_core/models/lvol_model.py | 4 ++ .../tasks_runner_replication_final.py | 38 ++++++++++++------- .../storage_pool/volume/replication.py | 15 ++++++++ 5 files changed, 65 insertions(+), 17 deletions(-) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index cb3e810b4..51fd2f85d 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -360,9 +360,10 @@ def get_config_var(name, default=None): # (snapshot -> wait replicated -> snapshot -> wait) before the final freeze. # Two rounds normally complete within 2 replication intervals + transfer time. REPL_CUTOVER_SHRINK_TIMEOUT_SEC = 900 -# Time the task holds in cutover_pending so the operator can pre-connect target -# NVMe paths before the ANA flip. Each volume's deadline is independent (non-blocking). -REPL_CUTOVER_PRECONNECT_WAIT_SEC = 10 +# Safety timeout for the operator preconnect signal. The task suspends indefinitely +# waiting for POST .../replication/cutover-proceed; this is the fallback deadline +# if the operator is unavailable. Cutover proceeds regardless after this many seconds. +REPL_CUTOVER_PROCEED_TIMEOUT_SEC = 120 SPDK_PROXY_MULTI_THREADING_ENABLED=True SPDK_PROXY_TIMEOUT=60*5 diff --git a/simplyblock_core/controllers/replication_policy_controller.py b/simplyblock_core/controllers/replication_policy_controller.py index d3da24c73..3f98127ee 100644 --- a/simplyblock_core/controllers/replication_policy_controller.py +++ b/simplyblock_core/controllers/replication_policy_controller.py @@ -359,6 +359,24 @@ def _failover_volumes(volumes, what): return results +def set_cutover_proceed(lvol_id): + """Signal that the operator has connected the target NVMe paths. + + Finds the cutover_pending LVolReplication for *lvol_id* (source side) and + sets cutover_proceed = True so the task runner advances past the wait. + + Returns the replication ID on success, raises KeyError when no matching + cutover_pending record is found. + """ + rep = _active_relationship(lvol_id) + if rep is None or rep.state != LVolReplication.STATE_CUTOVER_PENDING: + raise KeyError( + f"No cutover_pending replication found for volume {lvol_id}") + rep.cutover_proceed = True + rep.write_to_db(db.kv_store) + return rep.get_id() + + def get_relationship(lvol_id): """The replication relationship of *lvol_id*, source or target side. diff --git a/simplyblock_core/models/lvol_model.py b/simplyblock_core/models/lvol_model.py index 29ca15c6b..0b0eeb87d 100644 --- a/simplyblock_core/models/lvol_model.py +++ b/simplyblock_core/models/lvol_model.py @@ -144,6 +144,10 @@ class LVolReplication(BaseModel): # client keeps the same NQN/namespace across fail-over and migration. target_nqn: str = "" target_ns_id: int = 0 + # Set to True by POST .../replication/cutover-proceed once the operator has + # connected the target NVMe paths. The task runner waits for this before + # calling run_cutover(); REPL_CUTOVER_PROCEED_TIMEOUT_SEC is the safety fallback. + cutover_proceed: bool = False class LVolMini(BaseModel): lvol_uuid: str = "" diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 33da3c17b..bbb046f6e 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -121,20 +121,30 @@ def task_runner(task: JobSchedule): return _finalize(task, False, err) params = task.function_params - # Hold in cutover_pending so the operator can pre-connect target NVMe paths - # before the ANA flip. Each volume has its own deadline so this never blocks - # other volumes' cutover tasks. - if "preconnect_deadline" not in params: - params["preconnect_deadline"] = int(time.time()) + constants.REPL_CUTOVER_PRECONNECT_WAIT_SEC - task.function_result = "cutover_pending: waiting for preconnect" - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False - if int(time.time()) < params["preconnect_deadline"]: - task.function_result = "cutover_pending: waiting for preconnect" - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return False + # Wait for the operator to signal that target NVMe paths are connected + # (operator calls POST .../replication/cutover-proceed after its preconnect + # Job succeeds). REPL_CUTOVER_PROCEED_TIMEOUT_SEC is the safety fallback + # so cutover proceeds even if the operator is unavailable. + replication_id = params.get("replication_id") + if replication_id: + try: + rep = db.get_lvol_replication_by_id(replication_id) + if not rep.cutover_proceed: + if "cutover_proceed_timeout" not in params: + params["cutover_proceed_timeout"] = ( + int(time.time()) + constants.REPL_CUTOVER_PROCEED_TIMEOUT_SEC) + task.write_to_db(db.kv_store) + if int(time.time()) < params["cutover_proceed_timeout"]: + task.function_result = "cutover_pending: waiting for preconnect signal" + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return False + logger.warning( + "cutover proceed timeout for replication %s; proceeding without signal", + replication_id) + except KeyError: + logger.warning( + "replication record %s not found; proceeding with cutover", replication_id) try: ok, err = replication_final_step.run_cutover( diff --git a/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py b/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py index 339e5ef34..e716ccb8b 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/volume/replication.py @@ -159,6 +159,21 @@ def failback(cluster: Cluster, pool: StoragePool, volume: Volume, body: Failback return Response(status_code=204) +@api.post('/cutover-proceed', name='clusters:storage-pools:volumes:replication:cutover-proceed', + status_code=204, responses={204: {"content": None}}) +def cutover_proceed(cluster: Cluster, pool: StoragePool, volume: Volume) -> Response: + """Signal that target NVMe paths are connected and cutover may proceed. + + Called by the operator after its preconnect Job succeeds. The task runner + is suspended waiting for this signal; once set, it advances to the ANA flip. + """ + try: + replication_policy_controller.set_cutover_proceed(volume.get_id()) + except KeyError as exc: + raise HTTPException(404, str(exc)) + return Response(status_code=204) + + @api.get('/tasks', name='clusters:storage-pools:volumes:replication:tasks') def list_tasks(cluster: Cluster, pool: StoragePool, volume: Volume) -> List[TaskDTO]: return [TaskDTO.from_model(task) for task in lvol_controller.list_replication_tasks(volume.get_id())] From 62fe2c49d1e9b34890f42fc531b772de33310ac2 Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 24 Aug 2026 10:10:21 +0200 Subject: [PATCH 03/10] fix(health): a tertiary's expected hublvol paths include the secondary's The hublvol path repair computed its expected address set from the primary's data NICs alone. A secondary connects to the primary only, so that was right for it -- but a tertiary connects to the primary AND the secondary (4 paths), and with the primary's two paths present, missing_ips came out empty no matter what the secondary contributed. The len(ctrlrs) < 2 branch above cannot see it either: it only handles the secondary path being completely absent. Observed on the fresh 2026-08-24 deploy: BOTH tertiaries came up with exactly 3 of 4 hublvol paths -- the primary's two plus one of the secondary's, systematically missing the secondary's second NIC -- and nothing ever repaired them. The soak's baseline gate (paths must be a non-zero multiple of two) correctly refused to start on that cluster; the earlier runs' 6-11 minute "heal" waits after outages were the same defect showing mid-run, healed only when unrelated restart churn happened to reconcile the lvstore. Expected paths are now role-dependent: primary's NICs, plus the secondary's when the checked node is the tertiary and the secondary is ONLINE/DOWN. The reconcile call below already passed the secondary as a peer for tertiaries -- only the detection was blind. Also pins *.sh to LF via .gitattributes and normalizes the three staged shell scripts: a Windows checkout turned them CRLF and bash on the nodes refused them ("set: -\r: invalid option"), failing the provenance gate. Co-Authored-By: Claude Opus 5 --- .gitattributes | 1 + .../controllers/health_controller.py | 33 +++++++++++-- tests/unit/test_repair_gating.py | 46 +++++++++++++++++++ 3 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..dfdb8b771 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/simplyblock_core/controllers/health_controller.py b/simplyblock_core/controllers/health_controller.py index 7837c49e6..9fb82a9f5 100644 --- a/simplyblock_core/controllers/health_controller.py +++ b/simplyblock_core/controllers/health_controller.py @@ -416,11 +416,34 @@ def _check_sec_node_hublvol(node: StorageNode, auto_fix=False, primary_node_id=N # _collect_attached_ips holds the same rule for device controllers, # including the older single-entry/alternate_trids shape. attached_ips = storage_node_ops._collect_attached_ips(ret) - expected_ips = set() - for iface in primary_node.data_nics: - if (primary_node.active_rdma and iface.trtype == "RDMA") or (not primary_node.active_rdma and primary_node.active_tcp - and iface.trtype == "TCP"): - expected_ips.add(iface.ip4_address) + + def _data_ips(peer): + ips = set() + for iface in peer.data_nics: + if (peer.active_rdma and iface.trtype == "RDMA") or (not peer.active_rdma and peer.active_tcp + and iface.trtype == "TCP"): + ips.add(iface.ip4_address) + return ips + + # Expected paths depend on the ROLE of this node, not just on the + # primary. A secondary connects to the primary (2 paths); a + # tertiary connects to the primary AND the secondary (4 paths). + # Built from the primary alone, a tertiary that came up with only + # one of the secondary's two paths looked complete here — the + # primary's paths were all present, missing_ips was empty, and the + # 3-path controller sat unrepaired forever (both tertiaries on the + # fresh 2026-08-24 deploy, always missing the secondary's second + # NIC; the len(ctrlrs) < 2 branch above cannot see it either). + expected_ips = _data_ips(primary_node) + if is_sec2 and primary_node.secondary_node_id: + try: + _sec1 = db_controller.get_storage_node_by_id( + primary_node.secondary_node_id) + if _sec1.status in (StorageNode.STATUS_ONLINE, + StorageNode.STATUS_DOWN): + expected_ips |= _data_ips(_sec1) + except KeyError: + pass missing_ips = expected_ips - attached_ips if missing_ips: logger.info( diff --git a/tests/unit/test_repair_gating.py b/tests/unit/test_repair_gating.py index 94e6ce806..b66ce75ba 100644 --- a/tests/unit/test_repair_gating.py +++ b/tests/unit/test_repair_gating.py @@ -107,6 +107,52 @@ def test_no_repair_without_the_flag(self): def test_complete_controller_is_left_alone(self): self.assertFalse(self._call(self.BOTH_PATHS, repair_paths=True)) + def test_tertiary_missing_a_secondary_path_is_repaired(self): + """A tertiary connects to primary AND secondary. Three paths (primary + complete, secondary half-attached) must count as a missing path — on + the 2026-08-24 deploy both tertiaries sat at 3/4 forever because + expected_ips was built from the primary alone.""" + from simplyblock_core.controllers import health_controller as hc + from unittest.mock import MagicMock, patch + + def nic(ip): + iface = MagicMock(); iface.trtype = "TCP"; iface.ip4_address = ip + return iface + + primary = MagicMock() + primary.status = StorageNode.STATUS_ONLINE + primary.lvstore_status = "ready" + primary.active_rdma = False; primary.active_tcp = True + primary.data_nics = [nic("10.0.0.1"), nic("10.0.1.1")] + primary.hublvol.bdev_name = "LVS_1/hublvol" + primary.get_id.return_value = "primary-1" + primary.secondary_node_id = "sec-1" + + secondary = MagicMock() + secondary.status = StorageNode.STATUS_ONLINE + secondary.active_rdma = False; secondary.active_tcp = True + secondary.data_nics = [nic("10.0.0.2"), nic("10.0.1.2")] + + node = MagicMock() + node.status = StorageNode.STATUS_ONLINE + node.get_id.return_value = "tert-1" + # tertiary: primary's both paths + only ONE of the secondary's + node.rpc_client.return_value.bdev_nvme_controller_list.return_value = [ + {"ctrlrs": [ + {"state": "enabled", "trid": {"traddr": "10.0.0.1"}}, + {"state": "enabled", "trid": {"traddr": "10.0.1.1"}}, + {"state": "enabled", "trid": {"traddr": "10.0.0.2"}}]}] + node.lvstore_stack_tertiary = "primary-1" # makes is_sec2 True + + coordinator_cls = MagicMock() + with patch.object(hc, "DBController") as db, patch.object(hc, "_restart_owns_lvs", return_value=False), patch("simplyblock_core.utils.hublvol_reconnect." + "HublvolReconnectCoordinator", coordinator_cls): + db.return_value.get_storage_node_by_id.side_effect = lambda i: {"primary-1": primary, "sec-1": secondary}[i] + hc._check_sec_node_hublvol(node, primary_node_id="primary-1", + repair_paths=True) + self.assertTrue(coordinator_cls.return_value.reconcile.called, + "the missing secondary path was not reconciled") + def test_refused_when_the_primary_cannot_answer(self): self.assertFalse(self._call( self.ONE_PATH, primary_status=StorageNode.STATUS_UNREACHABLE, From 89ac86a667e2401058f82a1086c715c13e8a2604 Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 24 Aug 2026 10:41:17 +0200 Subject: [PATCH 04/10] fix(replication): fail-back evicts the recovered source's stale namespace The cutover clone keeps the ORIGINAL volume's NQN and nsid so the client reconnects to the same identity. Failing back to a RECOVERED source means that subsystem usually still exists there with the original volume's namespace at exactly that nsid -- data that is outdated by definition, since the other cluster served every write after the fail-over. add_ns then failed with -32602 on every retry and the cutover died on max retry: 2026-08-24, 5/5 fail-back cutovers, 40x "Failed to add bdev to subsystem" (first pristine lab where case 3 reached its cutover phase at all). _clone_from_last_replicated now evicts a namespace occupying the clone's nsid before add_lvol_on_node -- unless it is already the clone's own bdev, so re-runs stay idempotent; other namespaces and a missing subsystem are left alone. Forward migration (case 4) is unaffected: its destination is empty. Co-Authored-By: Claude Opus 5 --- .../controllers/lvol_controller.py | 38 +++++++++++ .../test_replication_chain_completeness.py | 68 +++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 10c8a02ac..ff12b0bc0 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3362,6 +3362,8 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps new_lvol.write_to_db(db_controller.kv_store) + _evict_stale_namespace(new_lvol, target_node) + lvol_bdev, error = add_lvol_on_node(new_lvol, target_node) if error: logger.error(error) @@ -3445,6 +3447,42 @@ def _last_replicated_target_snapshot(db_controller, lvol_id, cluster_id): return None +def _evict_stale_namespace(new_lvol, target_node): + """Make room for the preserved identity on a RECOVERED fail-back target. + + The cutover clone keeps the ORIGINAL volume's NQN and nsid so the client + reconnects to the same identity. Failing back to a recovered source means + that subsystem usually still exists there WITH the original volume's + namespace at exactly that nsid -- and its data is outdated by definition + (the other cluster served every write since the fail-over). add_ns then + fails with -32602 for every retry, and the whole cutover dies on max + retry: 2026-08-24, 5/5 fail-back cutovers, 40x "Failed to add bdev to + subsystem". Evict a namespace occupying the clone's nsid unless it is + already the clone's own bdev (idempotent re-run). + """ + try: + rpc = target_node.rpc_client() + subsystems = rpc.subsystem_get(new_lvol.nqn) + if not subsystems: + return + for ns in (subsystems[0].get("namespaces") or []): + if ns.get("nsid") != new_lvol.ns_id: + continue + if ns.get("bdev_name") == new_lvol.top_bdev: + return # already ours (re-run) + logger.info( + f"Fail-back cutover: evicting stale namespace nsid={ns.get('nsid')} " + f"(bdev {ns.get('bdev_name')}) from {new_lvol.nqn} on " + f"{target_node.get_id()} -- superseded by the failed-over data") + rpc.nvmf_subsystem_remove_ns(new_lvol.nqn, ns.get("nsid")) + return + except Exception as e: + # Best effort: if the subsystem is not there, add_lvol_on_node creates + # it; if the eviction genuinely failed, add_ns will say so loudly. + logger.warning(f"Stale-namespace check on {target_node.get_id()} for " + f"{new_lvol.nqn} raised: {e}") + + def _clone_from_last_replicated(db_controller, lvol_id, lvol, target_node, pool_uuid, cluster_id, attempts=3): """Pick the last fully replicated target snapshot and clone from it ATOMICALLY. diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 0d7e607a3..c31691090 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -263,3 +263,71 @@ def test_no_backward_task_for_a_policy_managed_clone(): assert gate in src, "the to-source enqueue must be gated on no forward policy" assert src.index(gate) < src.index("replicate_to_source=True"), \ "the gate must guard the to-source enqueue" + + +# --- fail-back cutover: the preserved identity must evict the stale ns ------ + + +class _EvictRPC: + def __init__(self, namespaces): + self.removed = [] + self._ns = namespaces + + def subsystem_get(self, nqn): + return [{"nqn": nqn, "namespaces": self._ns}] + + def nvmf_subsystem_remove_ns(self, nqn, nsid): + self.removed.append((nqn, nsid)) + return True + + +class _EvictNode: + def __init__(self, rpc): + self._rpc = rpc + + def get_id(self): + return "NODE_R" + + def rpc_client(self): + return self._rpc + + +class _CloneLvol: + nqn = "nqn.test:lvol:ORIG" + ns_id = 7 + top_bdev = "LVS_1/LVOL_CLONE" + + +def test_failback_evicts_the_recovered_sources_stale_namespace(): + """2026-08-24: on a recovered source the preserved-NQN subsystem still held + the ORIGINAL volume's namespace at the clone's nsid; add_ns failed -32602 + on every retry and 5/5 fail-back cutovers died on max retry.""" + from simplyblock_core.controllers import lvol_controller as lc + rpc = _EvictRPC([{"nsid": 7, "bdev_name": "LVS_1/LVOL_ORIG"}]) + lc._evict_stale_namespace(_CloneLvol(), _EvictNode(rpc)) + assert rpc.removed == [("nqn.test:lvol:ORIG", 7)] + + +def test_failback_eviction_is_idempotent_for_its_own_namespace(): + from simplyblock_core.controllers import lvol_controller as lc + rpc = _EvictRPC([{"nsid": 7, "bdev_name": "LVS_1/LVOL_CLONE"}]) + lc._evict_stale_namespace(_CloneLvol(), _EvictNode(rpc)) + assert rpc.removed == [] + + +def test_failback_eviction_leaves_other_namespaces_alone(): + from simplyblock_core.controllers import lvol_controller as lc + rpc = _EvictRPC([{"nsid": 3, "bdev_name": "LVS_1/OTHER"}]) + lc._evict_stale_namespace(_CloneLvol(), _EvictNode(rpc)) + assert rpc.removed == [] + + +def test_failback_eviction_tolerates_a_missing_subsystem(): + from simplyblock_core.controllers import lvol_controller as lc + + class _NoSubsysRPC(_EvictRPC): + def subsystem_get(self, nqn): + return None + rpc = _NoSubsysRPC([]) + lc._evict_stale_namespace(_CloneLvol(), _EvictNode(rpc)) # must not raise + assert rpc.removed == [] From 60342c0570b2f87c8896e0251f42396296a31580 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 24 Aug 2026 13:11:17 +0100 Subject: [PATCH 05/10] fix: correct subsys_port on target lvol clone to prevent ANA state -22 in migration --- simplyblock_core/controllers/lvol_controller.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index ff12b0bc0..eb64cefa6 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -3341,6 +3341,10 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps new_lvol.top_bdev = f"{new_lvol.lvs_name}/{new_lvol.lvol_bdev}" new_lvol.snapshot_name = snapshot.snap_bdev new_lvol.status = LVol.STATUS_IN_CREATION + # The source subsys_port is inherited from the deep copy but the target + # node may use a different per-lvstore port. Update it now so that + # suspend_lvol (and any future ANA flip) addresses the right listener. + new_lvol.subsys_port = target_node.get_lvol_subsys_port(target_node.lvstore) # Preserve the ORIGINAL subsystem NQN and namespace id: the client must # reconnect to the SAME NQN/NS on the target cluster — only the IP/port # differ. new_lvol is a deep copy of lvol, so nqn/ns_id are already From c2332ed9b4ebb49c6ce4f92b3018ef49f5f3ee44 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 24 Aug 2026 13:49:29 +0100 Subject: [PATCH 06/10] fix: use non-conflicting cntlid windows (4000+) for migration target clone to prevent duplicate cntlid kernel errors --- .../controllers/lvol_controller.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index eb64cefa6..28f657bad 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -1132,7 +1132,7 @@ def _lvol_secondary_index(lvol, node): return max(_lvol_path_index(lvol, node) - 1, 0) -def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0): +def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid=None): rpc_client = snode.rpc_client() # Refuse to attach a new namespace to a shared subsystem while any @@ -1165,7 +1165,8 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0): return _fail_after_bdev(lvol, rpc_client, str(e)) if resolve_subsys: - min_cntlid = lvol_min_cntlid(0 if is_primary else secondary_index + 1) + if min_cntlid is None: + min_cntlid = lvol_min_cntlid(0 if is_primary else secondary_index + 1) allow_any = not bool(lvol.allowed_hosts) logger.info("creating subsystem %s (allow_any_host=%s)", lvol.nqn, allow_any) ret = rpc_client.subsystem_create(lvol.nqn, lvol.ha_type, lvol.uuid, min_cntlid, @@ -3368,7 +3369,18 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps _evict_stale_namespace(new_lvol, target_node) - lvol_bdev, error = add_lvol_on_node(new_lvol, target_node) + # The target clone shares the source NQN. During preconnect the host has + # live paths to the source (cntlids 1, 1000, 2000) AND tries to add paths + # to the inaccessible target simultaneously. If target uses the same + # min_cntlid the kernel rejects it as a duplicate cntlid. Use windows + # above 4000 so source (1/1000/2000) and target never collide. + _tgt_cntlids = [ + random.randint(4001, 4500), # primary + random.randint(5001, 5500), # secondary + random.randint(6001, 6500), # tertiary + ] + + lvol_bdev, error = add_lvol_on_node(new_lvol, target_node, min_cntlid=_tgt_cntlids[0]) if error: logger.error(error) db_controller.release_lvol_ns_slot(new_lvol) @@ -3379,16 +3391,22 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps # Expose the volume on the secondary and tertiary target nodes too (HA), # so connect_lvol returns all client paths. + _tgt_cntlid_iter = iter(_tgt_cntlids[1:]) for peer_id in [target_node.secondary_node_id, target_node.tertiary_node_id]: if not peer_id: + next(_tgt_cntlid_iter, None) continue try: peer_node = db_controller.get_storage_node_by_id(peer_id) except KeyError: + next(_tgt_cntlid_iter, None) continue if peer_node.status != StorageNode.STATUS_ONLINE: + next(_tgt_cntlid_iter, None) continue - lvol_bdev, error = add_lvol_on_node(new_lvol, peer_node, is_primary=False) + _peer_cntlid = next(_tgt_cntlid_iter, None) + lvol_bdev, error = add_lvol_on_node(new_lvol, peer_node, is_primary=False, + min_cntlid=_peer_cntlid) if error: logger.error(error) # remove lvol from primary From 1ed3691fb9810c56238cddaa73c7bcd6465b9f3e Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 24 Aug 2026 14:40:28 +0100 Subject: [PATCH 07/10] fix: pass source lvol uuid as namespace UUID in target clone to prevent IDs don't match multipath rejection --- simplyblock_core/controllers/lvol_controller.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 28f657bad..5ff40db0a 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -1132,7 +1132,7 @@ def _lvol_secondary_index(lvol, node): return max(_lvol_path_index(lvol, node) - 1, 0) -def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid=None): +def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid=None, ns_uuid=None): rpc_client = snode.rpc_client() # Refuse to attach a new namespace to a shared subsystem while any @@ -1286,7 +1286,7 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid f"maps across the shared subsystem's paths)") requested_nsid = lvol.ns_id ret, err = rpc_client.nvmf_subsystem_add_ns2( - lvol.nqn, lvol.top_bdev, lvol.uuid, lvol.guid, nsid=requested_nsid) + lvol.nqn, lvol.top_bdev, ns_uuid or lvol.uuid, lvol.guid, nsid=requested_nsid) if err: if err and err["code"] == -32602 and lvol.namespace and lvol.node_id == snode.get_id(): logger.info("Error adding namespace to subsystem, finding new subsystem for namespaced lvol") @@ -3380,7 +3380,13 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps random.randint(6001, 6500), # tertiary ] - lvol_bdev, error = add_lvol_on_node(new_lvol, target_node, min_cntlid=_tgt_cntlids[0]) + # Preserve the source lvol's UUID as the NVMe namespace UUID so the kernel + # can recognise target paths as belonging to the same multipath namespace. + # new_lvol.uuid is a fresh DB key and must NOT be used as the namespace UUID. + _src_ns_uuid = lvol.uuid + + lvol_bdev, error = add_lvol_on_node(new_lvol, target_node, + min_cntlid=_tgt_cntlids[0], ns_uuid=_src_ns_uuid) if error: logger.error(error) db_controller.release_lvol_ns_slot(new_lvol) @@ -3406,7 +3412,7 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps continue _peer_cntlid = next(_tgt_cntlid_iter, None) lvol_bdev, error = add_lvol_on_node(new_lvol, peer_node, is_primary=False, - min_cntlid=_peer_cntlid) + min_cntlid=_peer_cntlid, ns_uuid=_src_ns_uuid) if error: logger.error(error) # remove lvol from primary From 9c5f32652966bda6b49c15aad48bb03db7c7c653 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 24 Aug 2026 15:20:47 +0100 Subject: [PATCH 08/10] removed simplyblock_core/controllers/health_controller.py changes --- .../controllers/health_controller.py | 33 +++---------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/simplyblock_core/controllers/health_controller.py b/simplyblock_core/controllers/health_controller.py index 9fb82a9f5..7837c49e6 100644 --- a/simplyblock_core/controllers/health_controller.py +++ b/simplyblock_core/controllers/health_controller.py @@ -416,34 +416,11 @@ def _check_sec_node_hublvol(node: StorageNode, auto_fix=False, primary_node_id=N # _collect_attached_ips holds the same rule for device controllers, # including the older single-entry/alternate_trids shape. attached_ips = storage_node_ops._collect_attached_ips(ret) - - def _data_ips(peer): - ips = set() - for iface in peer.data_nics: - if (peer.active_rdma and iface.trtype == "RDMA") or (not peer.active_rdma and peer.active_tcp - and iface.trtype == "TCP"): - ips.add(iface.ip4_address) - return ips - - # Expected paths depend on the ROLE of this node, not just on the - # primary. A secondary connects to the primary (2 paths); a - # tertiary connects to the primary AND the secondary (4 paths). - # Built from the primary alone, a tertiary that came up with only - # one of the secondary's two paths looked complete here — the - # primary's paths were all present, missing_ips was empty, and the - # 3-path controller sat unrepaired forever (both tertiaries on the - # fresh 2026-08-24 deploy, always missing the secondary's second - # NIC; the len(ctrlrs) < 2 branch above cannot see it either). - expected_ips = _data_ips(primary_node) - if is_sec2 and primary_node.secondary_node_id: - try: - _sec1 = db_controller.get_storage_node_by_id( - primary_node.secondary_node_id) - if _sec1.status in (StorageNode.STATUS_ONLINE, - StorageNode.STATUS_DOWN): - expected_ips |= _data_ips(_sec1) - except KeyError: - pass + expected_ips = set() + for iface in primary_node.data_nics: + if (primary_node.active_rdma and iface.trtype == "RDMA") or (not primary_node.active_rdma and primary_node.active_tcp + and iface.trtype == "TCP"): + expected_ips.add(iface.ip4_address) missing_ips = expected_ips - attached_ips if missing_ips: logger.info( From 33fbbf864e35ec9615cc295f51df25de7dd5926d Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 24 Aug 2026 18:08:44 +0100 Subject: [PATCH 09/10] fix: preserve source nsid on migration/failover target clone to prevent kernel IDs don't match for shared namespace N errors --- .../controllers/lvol_controller.py | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 5ff40db0a..527020dbb 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -1132,7 +1132,8 @@ def _lvol_secondary_index(lvol, node): return max(_lvol_path_index(lvol, node) - 1, 0) -def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid=None, ns_uuid=None): +def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid=None, ns_uuid=None, + primary_nsid=None): rpc_client = snode.rpc_client() # Refuse to attach a new namespace to a shared subsystem while any @@ -1276,7 +1277,13 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid # nsid (ns_id == 0, e.g. a drain-queued registration firing early) # must fail loudly instead of guessing. if is_primary: - requested_nsid = None + # primary_nsid is set by migration/failover callers to preserve the + # source cluster's nsid assignment. The client connects to both source + # and target under the same NQN during preconnect; if nsid positions + # differ, the kernel rejects the target namespaces as mismatched + # ("IDs don't match for shared namespace N"). Regular creates pass + # primary_nsid=None (auto-assign, the previous behaviour). + requested_nsid = primary_nsid else: if not lvol.ns_id: return _fail_after_bdev( @@ -1287,8 +1294,19 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid requested_nsid = lvol.ns_id ret, err = rpc_client.nvmf_subsystem_add_ns2( lvol.nqn, lvol.top_bdev, ns_uuid or lvol.uuid, lvol.guid, nsid=requested_nsid) - if err: - if err and err["code"] == -32602 and lvol.namespace and lvol.node_id == snode.get_id(): + if err: + if err["code"] == -32602 and lvol.namespace and lvol.node_id == snode.get_id(): + if primary_nsid is not None: + # Caller pinned a specific nsid (migration/failover preserve-nsid + # path). Re-claiming to a different subsystem would lose the shared + # NQN, making the target invisible to the client. Fail hard so the + # caller can diagnose rather than silently migrating into the wrong + # subsystem. _evict_stale_namespace should have cleared any occupant + # before this call; if we still got -32602 the state is unexpected. + return _fail_after_bdev( + lvol, rpc_client, + f"Failed to add bdev to subsystem at requested nsid={primary_nsid}: " + f"nsid already occupied and eviction did not clear it") logger.info("Error adding namespace to subsystem, finding new subsystem for namespaced lvol") # Re-claim transactionally, excluding the subsystem SPDK just # rejected (the DB count said it had room — SPDK is the authority @@ -3385,8 +3403,17 @@ def _create_target_lvol_clone(db_controller, lvol, target_node, pool_uuid, snaps # new_lvol.uuid is a fresh DB key and must NOT be used as the namespace UUID. _src_ns_uuid = lvol.uuid + # For migration/failover, preserve the source nsid so the kernel can + # match target paths to source paths under the same NQN. new_lvol is a + # deepcopy of the source lvol, so new_lvol.ns_id is already the source + # nsid. Passing it explicitly prevents auto-assignment from choosing a + # different position on the target (concurrent migration tasks for + # sibling namespaces would arrive in arbitrary order, diverging the nsid + # map from the source and triggering "IDs don't match for shared + # namespace N" in the client kernel during preconnect). lvol_bdev, error = add_lvol_on_node(new_lvol, target_node, - min_cntlid=_tgt_cntlids[0], ns_uuid=_src_ns_uuid) + min_cntlid=_tgt_cntlids[0], ns_uuid=_src_ns_uuid, + primary_nsid=new_lvol.ns_id) if error: logger.error(error) db_controller.release_lvol_ns_slot(new_lvol) From f66845fb08c1de0fc5f1cc8bf2cec8a76fb14202 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 24 Aug 2026 18:57:19 +0100 Subject: [PATCH 10/10] fix: unify cross-LVS -32602 handling for primary and secondary and fix _evict_stale_namespace KeyError on subsystem dict --- .../controllers/lvol_controller.py | 87 ++++++++++++------- 1 file changed, 55 insertions(+), 32 deletions(-) diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 527020dbb..7891fa61f 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -1295,37 +1295,60 @@ def add_lvol_on_node(lvol, snode, is_primary=True, secondary_index=0, min_cntlid ret, err = rpc_client.nvmf_subsystem_add_ns2( lvol.nqn, lvol.top_bdev, ns_uuid or lvol.uuid, lvol.guid, nsid=requested_nsid) if err: - if err["code"] == -32602 and lvol.namespace and lvol.node_id == snode.get_id(): - if primary_nsid is not None: - # Caller pinned a specific nsid (migration/failover preserve-nsid - # path). Re-claiming to a different subsystem would lose the shared - # NQN, making the target invisible to the client. Fail hard so the - # caller can diagnose rather than silently migrating into the wrong - # subsystem. _evict_stale_namespace should have cleared any occupant - # before this call; if we still got -32602 the state is unexpected. - return _fail_after_bdev( - lvol, rpc_client, - f"Failed to add bdev to subsystem at requested nsid={primary_nsid}: " - f"nsid already occupied and eviction did not clear it") - logger.info("Error adding namespace to subsystem, finding new subsystem for namespaced lvol") - # Re-claim transactionally, excluding the subsystem SPDK just - # rejected (the DB count said it had room — SPDK is the authority - # on its own namespace table). The lvol's record is rewritten in - # the same transaction, so its slot moves atomically from the - # rejected subsystem to the new one (or to a standalone one). - cluster = DBController().get_cluster_by_id(snode.cluster_id) - try: - DBController().claim_lvol_ns_slot( - lvol, snode, True, - standalone_nqn=cluster.nqn + ":lvol:" + lvol.uuid, - exclude_nqns={lvol.nqn}) - except SubsystemCapacityError as e: - logger.error(str(e)) - return _fail_after_bdev(lvol, rpc_client, str(e)) - return add_lvol_on_node(lvol, snode, is_primary=is_primary, secondary_index=secondary_index) - else: - return _fail_after_bdev( - lvol, rpc_client, "Failed to add bdev to subsystem") + if err["code"] == -32602 and lvol.namespace: + # -32602 from nvmf_subsystem_add_ns has two distinct causes: + # A) subsystem exists on this node but the nsid slot is occupied + # B) subsystem does not exist on this node at all (cross-LVS: + # the shared subsystem's LVS has no nodes in common with this + # lvol's LVS — e.g. a failover clone on LVS_6 whose shared + # subsystem lives on LVS_1) + # nvmf_subsystem_add_ns2 already ran the idempotency probe and + # returned the original error, so subsystem_get here distinguishes A/B. + subsys_here = rpc_client.subsystem_get(lvol.nqn) + if subsys_here is None: + # Case B — cross-LVS: subsystem not on this node. + # Primary: the namespace cannot be registered from this LVS; + # log a warning so the operator can investigate placement. + # Secondary: the primary already registered it on the subsystem's + # nodes; nothing more for this secondary to do. + # Either way, skip rather than deleting the bdev and retrying forever. + logger.warning( + "%s node %s: subsystem %s not present on this node " + "(cross-LVS namespaced lvol %s); skipping namespace add.", + "Primary" if is_primary else "Secondary", + snode.get_id(), lvol.nqn, lvol.get_id()) + return {'uuid': lvol.lvol_uuid, + 'driver_specific': {'lvol': {'blobid': lvol.blobid}}}, None + + # Case A — subsystem IS on this node; the nsid slot is occupied. + if lvol.node_id == snode.get_id(): + if primary_nsid is not None: + # Migration/failover: re-claiming to a different subsystem + # would lose the shared NQN. _evict_stale_namespace should + # have cleared the occupant; a persistent -32602 here means + # genuine unexpected state — fail hard. + return _fail_after_bdev( + lvol, rpc_client, + f"Failed to add bdev to subsystem at requested nsid={primary_nsid}: " + f"nsid occupied and eviction did not clear it") + logger.info("Error adding namespace to subsystem, finding new subsystem for namespaced lvol") + # Re-claim transactionally, excluding the subsystem SPDK just + # rejected (the DB count said it had room — SPDK is the authority + # on its own namespace table). The lvol's record is rewritten in + # the same transaction, so its slot moves atomically from the + # rejected subsystem to the new one (or to a standalone one). + cluster = DBController().get_cluster_by_id(snode.cluster_id) + try: + DBController().claim_lvol_ns_slot( + lvol, snode, True, + standalone_nqn=cluster.nqn + ":lvol:" + lvol.uuid, + exclude_nqns={lvol.nqn}) + except SubsystemCapacityError as e: + logger.error(str(e)) + return _fail_after_bdev(lvol, rpc_client, str(e)) + return add_lvol_on_node(lvol, snode, is_primary=is_primary, secondary_index=secondary_index) + return _fail_after_bdev( + lvol, rpc_client, "Failed to add bdev to subsystem") if is_primary: # Persist the target-assigned nsid; replicas re-add with exactly @@ -3520,7 +3543,7 @@ def _evict_stale_namespace(new_lvol, target_node): subsystems = rpc.subsystem_get(new_lvol.nqn) if not subsystems: return - for ns in (subsystems[0].get("namespaces") or []): + for ns in (subsystems.get("namespaces") or []): if ns.get("nsid") != new_lvol.ns_id: continue if ns.get("bdev_name") == new_lvol.top_bdev: