From 2300add371b7edeb8959d1d9ffc130d2011fe3c6 Mon Sep 17 00:00:00 2001 From: wmousa Date: Mon, 3 Aug 2026 17:36:39 +0200 Subject: [PATCH 01/27] fix(cluster-activate): repair stranded nodes instead of failing activation get_secondary_nodes() and get_secondary_nodes_2() each pair nodes one at a time via a greedy walk, preferring a domain/host-disjoint candidate from a shrinking shared pool. Nothing guarantees that walk closes a single cycle spanning every online node: it can close a cycle over a strict subset and strand the rest with zero candidates, even though a perfect pairing exists whenever there are 2+ online nodes (hit live: 12 nodes across 3 failure domains formed an 11-node secondary-pairing cycle, stranding the 12th and aborting activation with "No enough secondary nodes"). The tertiary assignment used by max_fault_tolerance >= 2 clusters (e.g. 2+2) has the identical structure and is subject to the same failure mode. Add splice_stranded_secondary() and splice_stranded_tertiary(): when a node is left with no candidates, splice it into an already-formed pairing edge (P->X becomes P->stranded->X) instead of giving up, preferring an edge where both sides differ from the stranded node's failure domain. The tertiary splice additionally re-validates host-disjointness against each side's own secondary partner, since a tertiary must be host-disjoint from both a primary and that primary's secondary. _cluster_activate falls back to these before raising, and only still fails if no pairing has been made at all yet. --- simplyblock_core/cluster_ops.py | 49 ++++-- simplyblock_core/storage_node_ops.py | 152 ++++++++++++++++++ tests/unit/test_failure_domain.py | 227 +++++++++++++++++++++++++++ 3 files changed, 414 insertions(+), 14 deletions(-) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 7e3b91376d..6e493a5eec 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -1062,6 +1062,18 @@ def _fd_fail(msg: str) -> None: used_nodes_as_sec: t.List[str] = [] used_nodes_as_tertiary: t.List[str] = [] snodes = db_controller.get_storage_nodes_by_cluster_id(cl_id) + # Process primaries grouped by failure domain. get_secondary_nodes/ + # get_secondary_nodes_2 (and their splice repairs) already sort their own + # candidate scan by domain, which alone is enough to keep the assignment + # domain-disjoint when domains are evenly sized. But once any node needs + # splice-repair (uneven domain sizes, some conflict unavoidable), the + # repair works off whatever partial assignment already exists -- so which + # primary gets processed first still changes the outcome. Grouping here + # too makes the result deterministic instead of order-dependent in that + # case. A no-op when FD is disabled (all nodes share one failure_domain). + # Fresh FD+HA activation bypasses this fallback via fd_desired_layout, + # but reactivation and non-HA/non-fresh paths still rely on it. + snodes = sorted(snodes, key=lambda n: n.failure_domain) if cluster.ha_type == "ha": for snode in snodes: # Do not assign secondary to removed node @@ -1081,16 +1093,21 @@ def _fd_fail(msg: str) -> None: secondary_nodes = [fd_desired_layout[snode.get_id()][0]] else: secondary_nodes = storage_node_ops.get_secondary_nodes(snode) - if not secondary_nodes: + if secondary_nodes: + snode = db_controller.get_storage_node_by_id(snode.get_id()) + snode.secondary_node_id = secondary_nodes[0] + snode.write_to_db() + sec_node = db_controller.get_storage_node_by_id(snode.secondary_node_id) + sec_node.lvstore_stack_secondary = snode.get_id() + sec_node.write_to_db() + elif not storage_node_ops.splice_stranded_secondary(snode): + # get_secondary_nodes()'s greedy walk closed a cycle that + # excludes this node, and there isn't even one existing + # pairing left to splice it into (only possible this early + # in the pass, before 2+ pairings exist). set_cluster_status(cl_id, ols_status) raise ValueError("Failed to activate cluster, No enough secondary nodes") - snode = db_controller.get_storage_node_by_id(snode.get_id()) - snode.secondary_node_id = secondary_nodes[0] - snode.write_to_db() - sec_node = db_controller.get_storage_node_by_id(snode.secondary_node_id) - sec_node.lvstore_stack_secondary = snode.get_id() - sec_node.write_to_db() used_nodes_as_sec.append(snode.secondary_node_id) # Assign second secondary when max_fault_tolerance >= 2 @@ -1109,15 +1126,19 @@ def _fd_fail(msg: str) -> None: exclude_failure_domains=[sec_node.failure_domain], exclude_physical_labels=[sec_node.physical_label], ) - if not secondary_nodes_2: + if secondary_nodes_2: + snode.tertiary_node_id = secondary_nodes_2[0] + snode.write_to_db() + sec_node_2 = db_controller.get_storage_node_by_id(snode.tertiary_node_id) + sec_node_2.lvstore_stack_tertiary = snode.get_id() + sec_node_2.write_to_db() + elif not storage_node_ops.splice_stranded_tertiary(snode): + # get_secondary_nodes_2()'s greedy walk closed a cycle that + # excludes this node, and there isn't even one existing + # tertiary pairing left to splice it into. set_cluster_status(cl_id, ols_status) raise ValueError("Failed to activate cluster, not enough nodes for dual fault tolerance") - - snode.tertiary_node_id = secondary_nodes_2[0] - snode.write_to_db() - sec_node_2 = db_controller.get_storage_node_by_id(snode.tertiary_node_id) - sec_node_2.lvstore_stack_tertiary = snode.get_id() - sec_node_2.write_to_db() + snode = db_controller.get_storage_node_by_id(snode.get_id()) used_nodes_as_tertiary.append(snode.tertiary_node_id) # Pass 1: bring up the primary LVS on every online primary node. diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index dd3e7d6d9e..a1d3ccdc08 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -9884,6 +9884,78 @@ def _candidates(forbidden_fds, forbidden_labels): return [] +def splice_stranded_secondary(stranded_node) -> bool: + """Fold a node get_secondary_nodes() could not place into the pairing + graph already built by the in-progress cluster_activate() pass. + + get_secondary_nodes() walks primaries in order, greedily picking the most + domain/host-disjoint unclaimed candidate for each. That greedy walk has no + mechanism to guarantee the resulting secondary_node_id/lvstore_stack_secondary + edges close a cycle spanning every online node: it can close a cycle over a + strict subset and leave the remaining node(s) with zero unclaimed + candidates, even though a perfect pairing trivially exists whenever there + are 2+ online nodes (observed 2026-08-03: 12 nodes across 3 failure + domains formed an 11-node cycle, stranding the 12th and aborting + activation). + + Rather than reworking the greedy walk into a global matching solver, this + repairs the one failure mode it has: pick any already-formed edge P->X + (P.secondary_node_id == X.get_id()) and splice the stranded node in + between, P->stranded->X. This always succeeds as long as at least one + edge already exists (guaranteed once 2+ pairings have been made this + activation pass) and turns the cycle that edge belongs to into one that + also covers the stranded node, without disturbing any other node. Prefers + an edge where both P and X differ from the stranded node's failure domain + (falling back to a host-disjoint-only edge), mirroring get_secondary_nodes' + own anti-affinity tiering. + """ + db_controller = DBController() + all_nodes = db_controller.get_storage_nodes_by_cluster_id(stranded_node.cluster_id) + edges = [n for n in all_nodes if n.secondary_node_id and n.get_id() != stranded_node.get_id()] + + def _host_disjoint(p, x): + return p.mgmt_ip != stranded_node.mgmt_ip and x.mgmt_ip != stranded_node.mgmt_ip + + def _domain_mismatch_score(p, x): + if stranded_node.failure_domain < 0: + return 0 + return sum(1 for n in (p, x) if n.failure_domain != stranded_node.failure_domain) + + best = None + best_score = -1 + for p in edges: + x = db_controller.get_storage_node_by_id(p.secondary_node_id) + if not x or x.get_id() == stranded_node.get_id() or not _host_disjoint(p, x): + continue + score = _domain_mismatch_score(p, x) + if score > best_score: + best_score, best = score, (p, x) + + if best is None: + return False + + p, x = best + logger.warning( + "get_secondary_nodes found no candidate for node %s; splicing it into " + "the existing pairing %s -> %s (domain-mismatch score %d/2).", + stranded_node.get_id(), p.get_id(), x.get_id(), best_score) + + p = db_controller.get_storage_node_by_id(p.get_id()) + p.secondary_node_id = stranded_node.get_id() + p.write_to_db() + + stranded_node = db_controller.get_storage_node_by_id(stranded_node.get_id()) + stranded_node.lvstore_stack_secondary = p.get_id() + stranded_node.secondary_node_id = x.get_id() + stranded_node.write_to_db() + + x = db_controller.get_storage_node_by_id(x.get_id()) + x.lvstore_stack_secondary = stranded_node.get_id() + x.write_to_db() + + return True + + def get_secondary_nodes_2(current_node: StorageNode, exclude_ids=None, exclude_mgmt_ips=None, exclude_failure_domains=None, exclude_physical_labels=None): """Get candidate nodes for second secondary assignment (dual fault tolerance). @@ -9974,6 +10046,86 @@ def _candidates(forbidden_fds, forbidden_labels): return [] +def splice_stranded_tertiary(stranded_node) -> bool: + """Tertiary-assignment counterpart to splice_stranded_secondary. + + get_secondary_nodes_2()'s greedy walk has the identical dead-end risk as + get_secondary_nodes(): it can close a tertiary-pairing cycle over a + subset of online nodes and strand the rest, even though a valid + assignment exists — this can surface on any cluster with + max_fault_tolerance >= 2 (e.g. a 2+2 layout), the same way + splice_stranded_secondary's bug surfaced on the plain secondary pass. + + Splices the stranded node into an already-formed tertiary edge P->X + (P.tertiary_node_id == X.get_id()), same idea as the secondary case: + P->stranded->X. The extra wrinkle here is that a tertiary must be + host-disjoint from BOTH a primary and that primary's OWN secondary (a + single host outage must not take out two of the four HA journal members) + — so splicing changes what "valid" means on both sides of the edge, and + each side is re-checked against the other's current secondary_node_id, + not just against each other. + """ + db_controller = DBController() + all_nodes = db_controller.get_storage_nodes_by_cluster_id(stranded_node.cluster_id) + by_id = {n.get_id(): n for n in all_nodes} + stranded_sec = by_id.get(stranded_node.secondary_node_id) if stranded_node.secondary_node_id else None + + def _valid_tertiary(primary, primary_sec, candidate): + if candidate.get_id() == primary.get_id(): + return False + if candidate.mgmt_ip == primary.mgmt_ip: + return False + if primary_sec and candidate.mgmt_ip == primary_sec.mgmt_ip: + return False + return True + + def _domain_mismatch_score(*nodes): + if stranded_node.failure_domain < 0: + return 0 + return sum(1 for n in nodes if n.failure_domain != stranded_node.failure_domain) + + edges = [n for n in all_nodes if n.tertiary_node_id and n.get_id() != stranded_node.get_id()] + + best = None + best_score = -1 + for p in edges: + x = by_id.get(p.tertiary_node_id) + if not x or x.get_id() == stranded_node.get_id(): + continue + p_sec = by_id.get(p.secondary_node_id) if p.secondary_node_id else None + if not _valid_tertiary(p, p_sec, stranded_node): + continue + if not _valid_tertiary(stranded_node, stranded_sec, x): + continue + score = _domain_mismatch_score(p, x) + if score > best_score: + best_score, best = score, (p, x) + + if best is None: + return False + + p, x = best + logger.warning( + "get_secondary_nodes_2 found no candidate for node %s; splicing it into " + "the existing tertiary pairing %s -> %s (domain-mismatch score %d/2).", + stranded_node.get_id(), p.get_id(), x.get_id(), best_score) + + p = db_controller.get_storage_node_by_id(p.get_id()) + p.tertiary_node_id = stranded_node.get_id() + p.write_to_db() + + stranded_node = db_controller.get_storage_node_by_id(stranded_node.get_id()) + stranded_node.lvstore_stack_tertiary = p.get_id() + stranded_node.tertiary_node_id = x.get_id() + stranded_node.write_to_db() + + x = db_controller.get_storage_node_by_id(x.get_id()) + x.lvstore_stack_tertiary = stranded_node.get_id() + x.write_to_db() + + return True + + def create_lvstore(snode: StorageNode, ndcs, npcs, distr_bs, distr_chunk_bs, page_size_in_blocks, max_size): db_controller = DBController() cluster = db_controller.get_cluster_by_id(snode.cluster_id) diff --git a/tests/unit/test_failure_domain.py b/tests/unit/test_failure_domain.py index 94eab3b191..3896ce2209 100644 --- a/tests/unit/test_failure_domain.py +++ b/tests/unit/test_failure_domain.py @@ -152,6 +152,98 @@ def test_enabled_falls_back_when_no_other_domain(self, MockDBCtrl): assert any("falling back" in m for m in cm.output) +# =========================================================================== +# 2b. splice_stranded_secondary +# +# get_secondary_nodes()'s greedy walk can close a pairing cycle over a strict +# subset of online nodes and strand the rest with zero candidates, even though +# a perfect pairing exists (observed 2026-08-03: 12 nodes / 3 domains formed +# an 11-node cycle, aborting activation for the 12th). splice_stranded_secondary +# repairs this by inserting the stranded node into an already-formed edge. +# =========================================================================== + +class TestSpliceStrandedSecondary(unittest.TestCase): + + def _mock_db(self, cluster, nodes): + mock_db = MagicMock() + by_id = {n.get_id(): n for n in nodes} + mock_db.get_cluster_by_id.return_value = cluster + mock_db.get_storage_nodes_by_cluster_id.return_value = nodes + mock_db.get_storage_node_by_id.side_effect = lambda nid: by_id.get(nid) + return mock_db + + @staticmethod + def _stub_writes(*nodes): + for n in nodes: + n.write_to_db = MagicMock() + + @patch("simplyblock_core.storage_node_ops.DBController") + def test_splices_into_existing_edge(self, MockDBCtrl): + import simplyblock_core.storage_node_ops as ops + # p -> x is an existing pairing from earlier in the activation pass. + p = _node("p", "10.0.0.1", failure_domain=0) + x = _node("x", "10.0.0.2", failure_domain=1) + p.secondary_node_id = x.get_id() + s = _node("s", "10.0.0.3", failure_domain=2) + MockDBCtrl.return_value = self._mock_db(_cluster(True), [p, x, s]) + self._stub_writes(p, x, s) + + assert ops.splice_stranded_secondary(s) is True + # p -> s -> x: p now points at s, s sits between p and x. + assert p.secondary_node_id == "s" + assert s.lvstore_stack_secondary == "p" + assert s.secondary_node_id == "x" + assert x.lvstore_stack_secondary == "s" + + @patch("simplyblock_core.storage_node_ops.DBController") + def test_prefers_edge_domain_disjoint_on_both_ends(self, MockDBCtrl): + import simplyblock_core.storage_node_ops as ops + s = _node("s", "10.0.0.9", failure_domain=0) + + # Same-domain edge (worse fit: 0/2 mismatch against s's domain). + bad_p = _node("bad_p", "10.0.0.1", failure_domain=0) + bad_x = _node("bad_x", "10.0.0.2", failure_domain=0) + bad_p.secondary_node_id = bad_x.get_id() + + # Domain-disjoint-on-both-ends edge (best fit: 2/2 mismatch). + good_p = _node("good_p", "10.0.0.3", failure_domain=1) + good_x = _node("good_x", "10.0.0.4", failure_domain=2) + good_p.secondary_node_id = good_x.get_id() + + nodes = [s, bad_p, bad_x, good_p, good_x] + MockDBCtrl.return_value = self._mock_db(_cluster(True), nodes) + self._stub_writes(*nodes) + + assert ops.splice_stranded_secondary(s) is True + assert good_p.secondary_node_id == "s" + assert s.secondary_node_id == "good_x" + assert bad_p.secondary_node_id == "bad_x" # untouched + + @patch("simplyblock_core.storage_node_ops.DBController") + def test_returns_false_when_no_edge_exists_yet(self, MockDBCtrl): + import simplyblock_core.storage_node_ops as ops + s = _node("s", "10.0.0.9", failure_domain=0) + other = _node("other", "10.0.0.1", failure_domain=1) # no pairing yet + MockDBCtrl.return_value = self._mock_db(_cluster(True), [s, other]) + self._stub_writes(s, other) + + assert ops.splice_stranded_secondary(s) is False + + @patch("simplyblock_core.storage_node_ops.DBController") + def test_skips_edge_not_host_disjoint_from_stranded(self, MockDBCtrl): + import simplyblock_core.storage_node_ops as ops + # x shares mgmt_ip with the stranded node -- splicing there would + # violate host-disjointness, so this edge must be skipped entirely. + s = _node("s", "10.0.0.5", failure_domain=0) + p = _node("p", "10.0.0.1", failure_domain=1) + x = _node("x", "10.0.0.5", failure_domain=2) + p.secondary_node_id = x.get_id() + MockDBCtrl.return_value = self._mock_db(_cluster(True), [s, p, x]) + self._stub_writes(s, p, x) + + assert ops.splice_stranded_secondary(s) is False + + # =========================================================================== # 3. get_secondary_nodes_2 (tertiary) # =========================================================================== @@ -199,6 +291,141 @@ def test_falls_back_when_only_shared_domains(self, MockDBCtrl): assert any("falling back" in m for m in cm.output) +# =========================================================================== +# 3b. splice_stranded_tertiary +# +# get_secondary_nodes_2() has the identical greedy-walk dead-end risk as +# get_secondary_nodes() (see TestSpliceStrandedSecondary above), reachable on +# any cluster with max_fault_tolerance >= 2 (e.g. a 2+2 layout). The extra +# wrinkle: a tertiary must be host-disjoint from both a primary and that +# primary's OWN secondary, so a valid splice must be re-checked against each +# side's current secondary_node_id, not just against each other. +# =========================================================================== + +class TestSpliceStrandedTertiary(unittest.TestCase): + + def _mock_db(self, cluster, nodes): + mock_db = MagicMock() + by_id = {n.get_id(): n for n in nodes} + mock_db.get_cluster_by_id.return_value = cluster + mock_db.get_storage_nodes_by_cluster_id.return_value = nodes + mock_db.get_storage_node_by_id.side_effect = lambda nid: by_id.get(nid) + return mock_db + + @staticmethod + def _stub_writes(*nodes): + for n in nodes: + n.write_to_db = MagicMock() + + @patch("simplyblock_core.storage_node_ops.DBController") + def test_splices_into_existing_tertiary_edge(self, MockDBCtrl): + import simplyblock_core.storage_node_ops as ops + # p -> x is an existing tertiary edge. s is stranded, with its own + # secondary (s_sec) on a distinct host from everyone else involved. + p_sec = _node("p_sec", "10.0.0.10", failure_domain=1) + p = _node("p", "10.0.0.1", failure_domain=0) + p.secondary_node_id = "p_sec" + x = _node("x", "10.0.0.2", failure_domain=1) + p.tertiary_node_id = "x" + + s_sec = _node("s_sec", "10.0.0.20", failure_domain=1) + s = _node("s", "10.0.0.3", failure_domain=2) + s.secondary_node_id = "s_sec" + + nodes = [p, p_sec, x, s, s_sec] + MockDBCtrl.return_value = self._mock_db(_cluster(True), nodes) + self._stub_writes(*nodes) + + assert ops.splice_stranded_tertiary(s) is True + # p -> s -> x: p now points at s, s sits between p and x. + assert p.tertiary_node_id == "s" + assert s.lvstore_stack_tertiary == "p" + assert s.tertiary_node_id == "x" + assert x.lvstore_stack_tertiary == "s" + + @patch("simplyblock_core.storage_node_ops.DBController") + def test_returns_false_when_no_edge_exists_yet(self, MockDBCtrl): + import simplyblock_core.storage_node_ops as ops + s = _node("s", "10.0.0.3", failure_domain=0) + other = _node("other", "10.0.0.1", failure_domain=1) # no tertiary edge yet + MockDBCtrl.return_value = self._mock_db(_cluster(True), [s, other]) + self._stub_writes(s, other) + + assert ops.splice_stranded_tertiary(s) is False + + @patch("simplyblock_core.storage_node_ops.DBController") + def test_rejects_edge_when_stranded_shares_host_with_primarys_secondary(self, MockDBCtrl): + import simplyblock_core.storage_node_ops as ops + # s sits on the same host as p's secondary (p_sec) -- s can never + # become p's tertiary, so the only existing edge must be rejected. + p_sec = _node("p_sec", "10.0.0.9", failure_domain=1) + p = _node("p", "10.0.0.1", failure_domain=0) + p.secondary_node_id = "p_sec" + x = _node("x", "10.0.0.2", failure_domain=1) + p.tertiary_node_id = "x" + + s = _node("s", "10.0.0.9", failure_domain=2) # same mgmt_ip as p_sec + s_sec = _node("s_sec", "10.0.0.20", failure_domain=1) + s.secondary_node_id = "s_sec" + + nodes = [p, p_sec, x, s, s_sec] + MockDBCtrl.return_value = self._mock_db(_cluster(True), nodes) + self._stub_writes(*nodes) + + assert ops.splice_stranded_tertiary(s) is False + + @patch("simplyblock_core.storage_node_ops.DBController") + def test_rejects_edge_when_x_shares_host_with_stranded_secondary(self, MockDBCtrl): + import simplyblock_core.storage_node_ops as ops + # x sits on the same host as s's secondary (s_sec) -- x can never + # become s's tertiary, so the only existing edge must be rejected. + p_sec = _node("p_sec", "10.0.0.9", failure_domain=1) + p = _node("p", "10.0.0.1", failure_domain=0) + p.secondary_node_id = "p_sec" + x = _node("x", "10.0.0.30", failure_domain=1) + p.tertiary_node_id = "x" + + s_sec = _node("s_sec", "10.0.0.30", failure_domain=1) # same mgmt_ip as x + s = _node("s", "10.0.0.3", failure_domain=2) + s.secondary_node_id = "s_sec" + + nodes = [p, p_sec, x, s, s_sec] + MockDBCtrl.return_value = self._mock_db(_cluster(True), nodes) + self._stub_writes(*nodes) + + assert ops.splice_stranded_tertiary(s) is False + + @patch("simplyblock_core.storage_node_ops.DBController") + def test_prefers_edge_domain_disjoint_on_both_ends(self, MockDBCtrl): + import simplyblock_core.storage_node_ops as ops + s_sec = _node("s_sec", "10.0.0.90", failure_domain=9) + s = _node("s", "10.0.0.9", failure_domain=0) + s.secondary_node_id = "s_sec" + + # Same-domain edge (worse fit: 0/2 mismatch against s's domain). + bad_p_sec = _node("bad_p_sec", "10.0.0.11", failure_domain=9) + bad_p = _node("bad_p", "10.0.0.1", failure_domain=0) + bad_p.secondary_node_id = "bad_p_sec" + bad_x = _node("bad_x", "10.0.0.2", failure_domain=0) + bad_p.tertiary_node_id = "bad_x" + + # Domain-disjoint-on-both-ends edge (best fit: 2/2 mismatch). + good_p_sec = _node("good_p_sec", "10.0.0.13", failure_domain=9) + good_p = _node("good_p", "10.0.0.3", failure_domain=1) + good_p.secondary_node_id = "good_p_sec" + good_x = _node("good_x", "10.0.0.4", failure_domain=2) + good_p.tertiary_node_id = "good_x" + + nodes = [s, s_sec, bad_p, bad_p_sec, bad_x, good_p, good_p_sec, good_x] + MockDBCtrl.return_value = self._mock_db(_cluster(True), nodes) + self._stub_writes(*nodes) + + assert ops.splice_stranded_tertiary(s) is True + assert good_p.tertiary_node_id == "s" + assert s.tertiary_node_id == "good_x" + assert bad_p.tertiary_node_id == "bad_x" # untouched + + # =========================================================================== # 4. get_sorted_ha_jms # =========================================================================== From 25746285d32b2ee008cb9fe622da2772b163fd0e Mon Sep 17 00:00:00 2001 From: wmousa Date: Mon, 3 Aug 2026 18:52:35 +0200 Subject: [PATCH 02/27] fix(ha): make node-shutdown FTT capacity check failure-domain aware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _check_ftt_allows_node_removal gates node shutdown/suspend purely on raw not-online node count (cap = npcs), independent of the operator's drain-gate. This is stricter than necessary with failure domains enabled: placement guarantees at most one erasure-coding chunk per domain once there are ndcs+npcs distinct domains, so losing up to npcs domains at once is already tolerated the same way losing up to npcs nodes is tolerated without FD — but this check still blocked a second node in the same domain, undermining the drain-coordinator's FD-aware concurrency (observed live: a same-domain concurrent drain was correctly waved through by the operator's gate, then independently rejected here with "FTT=1: cluster already has 1 not-online node(s)"). With FD enabled and the target node's domain assigned, the capacity check now counts distinct affected domains instead of raw node count: piling onto an already-affected domain is always free; a new domain is gated on distinct-domain count against npcs when there are enough domains for full one-chunk-per-domain isolation, or falls back to the plain node-count cap otherwise (mirrors the operator's fdDrainGate). FD disabled or an unassigned node falls back to the original node-count logic unchanged. The npcs=2/ft=1 primary-secondary pairing constraint is unaffected. --- simplyblock_core/storage_node_ops.py | 124 ++++++++++++++++++-------- tests/unit/test_ftt_protection.py | 128 ++++++++++++++++++++++++++- 2 files changed, 214 insertions(+), 38 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index a1d3ccdc08..53ed7a04dd 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -5293,10 +5293,56 @@ def _check_ftt_allows_node_removal(node_id, db_controller): if jm_replication_active: not_online_count += 1 - if npcs == 1: + # Capacity cap is npcs uniformly (the erasure code tolerates losing up to + # npcs of its ndcs+npcs chunks regardless of the *declared* ft, which only + # narrows npcs=2 down to a stricter per-pair constraint below). + capacity_cap = npcs + + fd_on = cluster.enable_failure_domain and snode.failure_domain >= 0 + blocked_by_capacity = False + capacity_reason = "" + + if fd_on: + # Placement guarantees at most one erasure-coding chunk per domain + # once there are >= ndcs+npcs distinct domains, so losing up to npcs + # *domains* at once is then tolerated the same way losing up to npcs + # *nodes* is tolerated without FD -- piling onto an already-affected + # domain costs nothing extra (mirrors that domain going down outright). + # Below that domain count, at least one domain necessarily carries + # more than one chunk, so a *new* domain falls back to the plain + # node-count cap instead of a second free domain slot. See the + # analogous fdDrainGate in the operator's nodedrain_controller.go. + domains_available = len({n.failure_domain for n in snodes if n.failure_domain >= 0}) + domains_needed = ndcs + npcs + active_domains = {n.failure_domain for n in not_online_nodes if n.failure_domain >= 0} + # jm_replication_active can't be attributed to a specific domain (the + # probe only says "some online node's journal is behind"), so treat + # it conservatively as always adding a new, unaccounted-for domain. + active_domain_count = len(active_domains) + (1 if jm_replication_active else 0) + my_domain = snode.failure_domain + + if my_domain not in active_domains: + if domains_needed > 0 and domains_available >= domains_needed: + if active_domain_count >= capacity_cap: + blocked_by_capacity = True + capacity_reason = ( + f"FTT={ft} (npcs={npcs}): cannot remove node, failure domain {my_domain} " + f"not yet active; {active_domain_count}/{capacity_cap} domains active" + f"{' (including in-progress journal replication)' if jm_replication_active else ''}" + ) + elif active_domain_count > 0 and not_online_count >= capacity_cap: + blocked_by_capacity = True + capacity_reason = ( + f"FTT={ft} (npcs={npcs}): insufficient failure domains " + f"({domains_available} available, {domains_needed} needed for full isolation) " + f"to remove node in failure domain {my_domain}; " + f"{not_online_count}/{capacity_cap} nodes active" + ) + elif npcs == 1: # FTT=1: no room at all if anything is already not online or journal replicating if not_online_count > 0: - return False, ( + blocked_by_capacity = True + capacity_reason = ( f"FTT=1 (npcs=1): cannot remove node, cluster already has " f"{len(not_online_nodes)} not-online node(s)" f"{' and journal replication in progress' if jm_replication_active else ''}" @@ -5306,53 +5352,59 @@ def _check_ftt_allows_node_removal(node_id, db_controller): if ft >= 2: # FTT=2: room for one not-online node, block if already have one+ if not_online_count >= 2: - return False, ( + blocked_by_capacity = True + capacity_reason = ( f"FTT=2 (npcs=2): cannot remove node, cluster already has " f"{len(not_online_nodes)} not-online node(s)" f"{' and journal replication in progress' if jm_replication_active else ''}" ) else: # npcs=2, ft=1: like FTT=2 for capacity, but additionally - # cannot remove both primary and its secondary + # cannot remove both primary and its secondary (checked below). if not_online_count >= 2: - return False, ( + blocked_by_capacity = True + capacity_reason = ( f"npcs=2/ft=1: cannot remove node, cluster already has " f"{len(not_online_nodes)} not-online node(s)" f"{' and journal replication in progress' if jm_replication_active else ''}" ) - # Check primary-secondary pair constraint: - # If the node being removed is a primary, check its secondary is online. - # If the node being removed is a secondary, check its primary is online. - for not_online_node in not_online_nodes: - # Is any not-online node the secondary of the node we're removing? - if snode.secondary_node_id == not_online_node.get_id(): - return False, ( - f"npcs=2/ft=1: cannot remove node {node_id}, " - f"its secondary {not_online_node.get_id()} is not online " - f"(status: {not_online_node.status})" - ) - if snode.tertiary_node_id == not_online_node.get_id(): - return False, ( - f"npcs=2/ft=1: cannot remove node {node_id}, " - f"its secondary {not_online_node.get_id()} is not online " - f"(status: {not_online_node.status})" - ) + if blocked_by_capacity: + return False, capacity_reason - # Is the node we're removing a secondary of any not-online primary? - for not_online_node in not_online_nodes: - if not_online_node.secondary_node_id == node_id: - return False, ( - f"npcs=2/ft=1: cannot remove node {node_id}, " - f"it is secondary of not-online primary {not_online_node.get_id()} " - f"(status: {not_online_node.status})" - ) - if not_online_node.tertiary_node_id == node_id: - return False, ( - f"npcs=2/ft=1: cannot remove node {node_id}, " - f"it is secondary of not-online primary {not_online_node.get_id()} " - f"(status: {not_online_node.status})" - ) + if npcs == 2 and ft == 1: + # npcs=2, ft=1: beyond the capacity cap above, cannot remove both a + # primary and its own secondary/tertiary at once -- a per-relationship + # constraint, orthogonal to failure domains. + for not_online_node in not_online_nodes: + # Is any not-online node the secondary of the node we're removing? + if snode.secondary_node_id == not_online_node.get_id(): + return False, ( + f"npcs=2/ft=1: cannot remove node {node_id}, " + f"its secondary {not_online_node.get_id()} is not online " + f"(status: {not_online_node.status})" + ) + if snode.tertiary_node_id == not_online_node.get_id(): + return False, ( + f"npcs=2/ft=1: cannot remove node {node_id}, " + f"its secondary {not_online_node.get_id()} is not online " + f"(status: {not_online_node.status})" + ) + + # Is the node we're removing a secondary of any not-online primary? + for not_online_node in not_online_nodes: + if not_online_node.secondary_node_id == node_id: + return False, ( + f"npcs=2/ft=1: cannot remove node {node_id}, " + f"it is secondary of not-online primary {not_online_node.get_id()} " + f"(status: {not_online_node.status})" + ) + if not_online_node.tertiary_node_id == node_id: + return False, ( + f"npcs=2/ft=1: cannot remove node {node_id}, " + f"it is secondary of not-online primary {not_online_node.get_id()} " + f"(status: {not_online_node.status})" + ) return True, "" diff --git a/tests/unit/test_ftt_protection.py b/tests/unit/test_ftt_protection.py index f63ffe162c..8ba8a598f3 100644 --- a/tests/unit/test_ftt_protection.py +++ b/tests/unit/test_ftt_protection.py @@ -23,7 +23,7 @@ # Helpers # --------------------------------------------------------------------------- -def _cluster(ha_type="ha", npcs=1, ndcs=2, ft=1, rebalancing=False): +def _cluster(ha_type="ha", npcs=1, ndcs=2, ft=1, rebalancing=False, enable_failure_domain=False): cl = Cluster() cl.uuid = "cluster-1" cl.ha_type = ha_type @@ -31,12 +31,14 @@ def _cluster(ha_type="ha", npcs=1, ndcs=2, ft=1, rebalancing=False): cl.distr_ndcs = ndcs cl.max_fault_tolerance = ft cl.is_re_balancing = rebalancing + cl.enable_failure_domain = enable_failure_domain cl.status = Cluster.STATUS_ACTIVE return cl def _node(node_id, status=StorageNode.STATUS_ONLINE, cluster_id="cluster-1", - secondary_id="", secondary_id_2="", jm_vuid=8881, lvstore="LVS_1"): + secondary_id="", secondary_id_2="", jm_vuid=8881, lvstore="LVS_1", + failure_domain=-1): n = MagicMock(spec=StorageNode) n.uuid = node_id n.get_id = MagicMock(return_value=node_id) @@ -47,6 +49,7 @@ def _node(node_id, status=StorageNode.STATUS_ONLINE, cluster_id="cluster-1", n.jm_vuid = jm_vuid n.lvstore = lvstore n.mgmt_ip = f"10.0.0.{hash(node_id) % 256}" + n.failure_domain = failure_domain # rpc_client mock: journal replication not active by default rpc = MagicMock() rpc.bdev_lvol_get_lvstores = MagicMock(return_value=[{"name": lvstore}]) @@ -573,6 +576,127 @@ def test_jm_replication_counts_but_pair_still_checked(self): self.assertFalse(allowed) +# --------------------------------------------------------------------------- +# Failure-domain-aware capacity check +# +# With FD enabled, the capacity cap (npcs) counts distinct affected failure +# domains instead of raw not-online nodes, mirroring the operator's +# fdDrainGate (nodedrain_controller.go) -- piling onto an already-affected +# domain is always free; a *new* domain is gated on distinct-domain count +# when there are >= ndcs+npcs domains, or on the plain node-count cap +# otherwise (under-provisioned). +# --------------------------------------------------------------------------- + +class TestFailureDomainAwareCapacity(unittest.TestCase): + + def test_piling_onto_active_domain_always_allowed(self): + """Two nodes already down in domain 1 (at cap already) -- removing a + THIRD node also in domain 1 must still be allowed.""" + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=1), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n4", failure_domain=2), + _node("n5", failure_domain=3), + _node("n6", failure_domain=4), + ] + db = _db(cl, nodes) + allowed, _ = _check_ftt_allows_node_removal("n1", db) + self.assertTrue(allowed) + + def test_well_provisioned_new_domain_blocked_at_cap(self): + """ndcs=2/npcs=2 needs 4 domains for full isolation; 4 are available. + Domains 1 and 2 already have not-online nodes (cap=npcs=2 domains + active) -- removing a node in a THIRD domain must be blocked.""" + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=3), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", status=StorageNode.STATUS_OFFLINE, failure_domain=2), + _node("n4", failure_domain=4), + _node("n5", failure_domain=1), + _node("n6", failure_domain=2), + ] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + self.assertIn("domain", reason) + + def test_well_provisioned_new_domain_within_cap_allowed(self): + """Same layout, but only ONE domain active so far -- opening a + second domain is still within the npcs=2 domain budget.""" + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=3), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", failure_domain=2), + _node("n4", failure_domain=4), + ] + db = _db(cl, nodes) + allowed, _ = _check_ftt_allows_node_removal("n1", db) + self.assertTrue(allowed) + + def test_under_provisioned_second_domain_within_node_budget_allowed(self): + """ndcs=2/npcs=2 needs 4 domains, but only 2 exist -- under-provisioned. + One node down in domain 1; removing a node in domain 2 (opening a + second domain) is still allowed while raw not-online count (1) is + under the npcs=2 node budget.""" + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=2), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", failure_domain=1), + _node("n4", failure_domain=2), + ] + db = _db(cl, nodes) + allowed, _ = _check_ftt_allows_node_removal("n1", db) + self.assertTrue(allowed) + + def test_under_provisioned_second_domain_blocked_at_node_budget(self): + """Same under-provisioned layout, but the node-count budget (npcs=2) + is already spent -- opening a second domain must now be blocked.""" + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=2), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n4", failure_domain=2), + ] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + self.assertIn("insufficient failure domains", reason) + + def test_fd_enabled_but_node_unassigned_falls_back_to_node_count(self): + """FD enabled cluster-wide, but this specific node has no domain + assignment (-1) -- must fall back to the plain node-count cap.""" + cl = _cluster(npcs=1, ndcs=2, ft=1, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=-1), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", failure_domain=1), + ] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + self.assertIn("not-online", reason) + + def test_fd_disabled_ignores_domains_uses_node_count(self): + """enable_failure_domain=False: domain tags present but irrelevant -- + behaves exactly like the non-FD npcs=1 case.""" + cl = _cluster(npcs=1, ndcs=2, ft=1, enable_failure_domain=False) + nodes = [ + _node("n1", failure_domain=1), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=2), + _node("n3", failure_domain=3), + ] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + self.assertIn("FTT=1", reason) + + # --------------------------------------------------------------------------- # Rebalancing additional scenarios # --------------------------------------------------------------------------- From 912eb47c9af8de258c54bd671fcb7372bcf75f69 Mon Sep 17 00:00:00 2001 From: wmousa Date: Tue, 4 Aug 2026 12:03:21 +0200 Subject: [PATCH 03/27] fix(cluster-activate): make secondary/tertiary pairing domain-order-independent get_secondary_nodes/get_secondary_nodes_2/splice_stranded_secondary/ splice_stranded_tertiary each fetch their own candidate list fresh from the DB and scan/score it in that order, independent of whatever order the caller processes primaries in. With failure domains enabled, this made the resulting primary/secondary/tertiary assignment sensitive to arbitrary node ordering: even when a fully domain-disjoint assignment exists (e.g. equal- sized domains), ~1 in 5 arbitrary orderings left some node with a same- domain secondary or tertiary (verified by simulation), and the live deployment hit exactly this. All four functions now sort their fetched node list by failure_domain before scanning, which makes equal-sized domains fully order-independent (0 conflicts across 50 arbitrary orderings, verified). _cluster_activate's own pairing loop also sorts its processing order the same way: once domain sizes are uneven and splice-repair is required, the repair works off whatever partial assignment already exists, so the caller's processing order still mattered even with the candidate-scan fix alone. With both in place, unequal-domain conflict counts become deterministic instead of order-dependent. Both sorts are no-ops when failure domains are disabled. --- simplyblock_core/storage_node_ops.py | 21 +++++ tests/unit/test_failure_domain.py | 120 +++++++++++++++++++++++++++ 2 files changed, 141 insertions(+) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 53ed7a04dd..3a5811c746 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -9880,6 +9880,18 @@ def get_secondary_nodes(current_node: StorageNode, exclude_ids=None, removed_nod db_controller = DBController() cluster = db_controller.get_cluster_by_id(current_node.cluster_id) all_nodes = db_controller.get_storage_nodes_by_cluster_id(current_node.cluster_id) + # Group by failure domain (stable sort, preserves DB order within each + # domain) before scanning candidates. The "first valid candidate after my + # own position" logic below skips same-domain nodes as forbidden, so on an + # arbitrary/interleaved node order it can still land back on a same-domain + # pick once every other domain's nodes are already claimed -- purely an + # artifact of iteration order, not availability (verified by simulation: + # ~1 in 5 arbitrary orderings produces an avoidable same-domain pick even + # when a fully domain-disjoint assignment exists). Grouping first removes + # that sensitivity: every node's forward scan cleanly skips past the rest + # of its own domain into the next one. A no-op when FD is disabled (all + # nodes share the same failure_domain, so the sort is order-preserving). + all_nodes = sorted(all_nodes, key=lambda n: n.failure_domain) if len(all_nodes) == 2: for node in all_nodes: if node.get_id() != current_node.get_id() and node.get_id() not in exclude_ids: @@ -9963,6 +9975,9 @@ def splice_stranded_secondary(stranded_node) -> bool: """ db_controller = DBController() all_nodes = db_controller.get_storage_nodes_by_cluster_id(stranded_node.cluster_id) + # Deterministic tie-breaking among equally domain-scored edges -- see + # get_secondary_nodes for why this sort matters. + all_nodes = sorted(all_nodes, key=lambda n: n.failure_domain) edges = [n for n in all_nodes if n.secondary_node_id and n.get_id() != stranded_node.get_id()] def _host_disjoint(p, x): @@ -10035,6 +10050,9 @@ def get_secondary_nodes_2(current_node: StorageNode, exclude_ids=None, exclude_m db_controller = DBController() cluster = db_controller.get_cluster_by_id(current_node.cluster_id) all_nodes = db_controller.get_storage_nodes_by_cluster_id(current_node.cluster_id) + # See get_secondary_nodes for why this sort matters: it removes the + # pairing algorithm's sensitivity to arbitrary/interleaved node order. + all_nodes = sorted(all_nodes, key=lambda n: n.failure_domain) if len(all_nodes) == 2: for node in all_nodes: if node.get_id() != current_node.get_id() and node.get_id() not in exclude_ids: @@ -10119,6 +10137,9 @@ def splice_stranded_tertiary(stranded_node) -> bool: """ db_controller = DBController() all_nodes = db_controller.get_storage_nodes_by_cluster_id(stranded_node.cluster_id) + # Deterministic tie-breaking among equally domain-scored edges -- see + # get_secondary_nodes for why this sort matters. + all_nodes = sorted(all_nodes, key=lambda n: n.failure_domain) by_id = {n.get_id(): n for n in all_nodes} stranded_sec = by_id.get(stranded_node.secondary_node_id) if stranded_node.secondary_node_id else None diff --git a/tests/unit/test_failure_domain.py b/tests/unit/test_failure_domain.py index 3896ce2209..e16d08bb07 100644 --- a/tests/unit/test_failure_domain.py +++ b/tests/unit/test_failure_domain.py @@ -24,6 +24,7 @@ All external dependencies (FDB, RPC) are mocked. """ +import random import unittest from unittest.mock import MagicMock, patch @@ -872,5 +873,124 @@ def test_tertiary_excludes_secondary_label(self, MockDBCtrl): assert "c2" not in result # primary's label +# =========================================================================== +# 9. Domain-grouped ordering eliminates pairing order-sensitivity +# +# Two independent things had to be fixed for the pairing loop to stop being +# order-sensitive: +# +# 1. get_secondary_nodes/get_secondary_nodes_2/splice_stranded_secondary/ +# splice_stranded_tertiary each fetch their own candidate list fresh via +# db_controller.get_storage_nodes_by_cluster_id() and scan/score it in +# that order -- so their domain-disjointness used to depend on whatever +# order the DB happened to return nodes in, regardless of caller. All +# four now sort their fetched node list by failure_domain first. +# 2. Even with (1) fixed, _cluster_activate's own pairing loop still +# determines which primary is processed first, and once a domain-size +# imbalance forces splice-repair, the repair works off whatever partial +# assignment already exists -- so the CALLER's processing order still +# changes the outcome. _cluster_activate now also sorts its local +# `snodes` by failure_domain before the loop. +# +# Verified here by running the full secondary+tertiary pairing sequence with +# the DB mock returning nodes in an arbitrary (shuffled) order -- proving (1) +# -- while the caller-side loop mirrors _cluster_activate's own domain sort +# -- proving (1)+(2) together give a fully order-independent result. +# =========================================================================== + +class TestDomainGroupedCandidateScanOrderIndependence(unittest.TestCase): + + def _mock_db(self, cluster, nodes): + mock_db = MagicMock() + by_id = {n.get_id(): n for n in nodes} + mock_db.get_cluster_by_id.return_value = cluster + mock_db.get_storage_nodes_by_cluster_id.return_value = nodes + mock_db.get_storage_node_by_id.side_effect = lambda nid: by_id.get(nid) + return mock_db + + @staticmethod + def _build_nodes(domain_sizes): + nodes = [] + idx = 0 + for fd, size in enumerate(domain_sizes, start=1): + for _ in range(size): + idx += 1 + nodes.append(_node(f"n{idx}", f"10.0.0.{idx}", failure_domain=fd)) + return nodes + + def _run_pairing(self, db_order_nodes, cluster): + """Mirrors _cluster_activate's secondary+tertiary pairing loop + (cluster_ops.py), including its own failure_domain sort of the + processing order. `db_order_nodes` is deliberately left in an + arbitrary order to represent whatever get_storage_nodes_by_cluster_id + naturally returns -- proving get_secondary_nodes/_2's own internal + sort is what keeps candidate scanning domain-grouped regardless.""" + import simplyblock_core.storage_node_ops as ops + by_id = {n.get_id(): n for n in db_order_nodes} + for n in db_order_nodes: + n.write_to_db = MagicMock() + + processing_order = sorted(db_order_nodes, key=lambda n: n.failure_domain) + + with patch("simplyblock_core.storage_node_ops.DBController", + return_value=self._mock_db(cluster, db_order_nodes)): + for snode in processing_order: + secs = ops.get_secondary_nodes(snode) + if secs: + snode.secondary_node_id = secs[0] + by_id[secs[0]].lvstore_stack_secondary = snode.get_id() + else: + assert ops.splice_stranded_secondary(snode), f"{snode.get_id()} stranded on secondary" + + used_tertiary = [] + for snode in processing_order: + sec = by_id[snode.secondary_node_id] + t2 = ops.get_secondary_nodes_2( + snode, + exclude_ids=[snode.secondary_node_id] + used_tertiary, + exclude_mgmt_ips=[sec.mgmt_ip], + exclude_failure_domains=[sec.failure_domain], + exclude_physical_labels=[sec.physical_label], + ) + if t2: + snode.tertiary_node_id = t2[0] + by_id[t2[0]].lvstore_stack_tertiary = snode.get_id() + used_tertiary.append(t2[0]) + else: + assert ops.splice_stranded_tertiary(snode), f"{snode.get_id()} stranded on tertiary" + used_tertiary.append(snode.tertiary_node_id) + + conflicts = 0 + for n in db_order_nodes: + sec = by_id[n.secondary_node_id] + ter = by_id[n.tertiary_node_id] + if len({n.failure_domain, sec.failure_domain, ter.failure_domain}) != 3: + conflicts += 1 + return conflicts + + def test_equal_domains_conflict_free_across_many_arbitrary_db_orders(self): + cluster = _cluster(True, distr_npcs=2) + cluster.distr_ndcs = 2 + for seed in range(50): + nodes = self._build_nodes([4, 4, 4]) + random.Random(seed).shuffle(nodes) # arbitrary "DB return" order + conflicts = self._run_pairing(nodes, cluster) + assert conflicts == 0, f"seed={seed} produced {conflicts} conflicts" + + def test_unequal_domains_conflict_count_is_deterministic(self): + # 4/5/4: a full zero-conflict assignment is mathematically impossible + # (domains aren't equal size), but the combined sort (candidate scan + # + processing order) makes the resulting conflict count the same + # regardless of the arbitrary DB return order. + cluster = _cluster(True, distr_npcs=2) + cluster.distr_ndcs = 2 + counts = set() + for seed in range(20): + nodes = self._build_nodes([4, 5, 4]) + random.Random(seed).shuffle(nodes) + counts.add(self._run_pairing(nodes, cluster)) + assert len(counts) == 1, f"expected one consistent conflict count, got {counts}" + + if __name__ == "__main__": unittest.main() From 0b5ad2961adadb6d4a144015e282bf196876e1ba Mon Sep 17 00:00:00 2001 From: wmousa Date: Wed, 5 Aug 2026 13:56:37 +0200 Subject: [PATCH 04/27] fix(ha): correct failure-domain risk budget for node-shutdown capacity check The previous rule treated piling additional nodes onto an already-affected failure domain as always free, only falling back to a raw node-count cap when opening a brand-new domain under-provisioned. Cross-checked against the backend team's confirmed tolerance (2 FD: one whole FD down OR one node in each FD, nothing else; 3 FD: one whole FD only; 4 FD: two whole FDs), that rule incorrectly allowed unsafe combinations such as one node in FD1 plus two nodes in FD2 on a 2-FD cluster. Each domain's worst-case contribution to a stripe's chunk loss is now capped at chunks_per_domain = ceil((ndcs+npcs) / domains_available). A domain already at or above that count has maxed its risk contribution, so further nodes in the same domain are free; otherwise the summed capped risk across all affected domains plus the node being removed must stay within npcs. This collapses to the existing "npcs whole domains free" behavior once there are >= ndcs+npcs domains, and reproduces the confirmed 2/3/4-FD tolerance exactly -- verified both by direct simulation of the formula and by driving the real function through every scheme/domain-count combination in tests/unit/test_ftt_protection.py. --- simplyblock_core/storage_node_ops.py | 70 +++++------ tests/unit/test_ftt_protection.py | 168 +++++++++++++++++++++++++-- 2 files changed, 197 insertions(+), 41 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 3a5811c746..cce011441d 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -5293,51 +5293,53 @@ def _check_ftt_allows_node_removal(node_id, db_controller): if jm_replication_active: not_online_count += 1 - # Capacity cap is npcs uniformly (the erasure code tolerates losing up to - # npcs of its ndcs+npcs chunks regardless of the *declared* ft, which only - # narrows npcs=2 down to a stricter per-pair constraint below). - capacity_cap = npcs - fd_on = cluster.enable_failure_domain and snode.failure_domain >= 0 blocked_by_capacity = False capacity_reason = "" if fd_on: - # Placement guarantees at most one erasure-coding chunk per domain - # once there are >= ndcs+npcs distinct domains, so losing up to npcs - # *domains* at once is then tolerated the same way losing up to npcs - # *nodes* is tolerated without FD -- piling onto an already-affected - # domain costs nothing extra (mirrors that domain going down outright). - # Below that domain count, at least one domain necessarily carries - # more than one chunk, so a *new* domain falls back to the plain - # node-count cap instead of a second free domain slot. See the - # analogous fdDrainGate in the operator's nodedrain_controller.go. + # Placement spreads a stripe's ndcs+npcs chunks as evenly as possible + # across the domains that actually exist. With fewer domains than + # chunks, at least one domain holds ceil((ndcs+npcs)/domains) chunks + # -- that many nodes can go down within a SINGLE domain for free + # (mirrors that domain's worst-case chunk contribution going down + # outright, already priced in), but once a domain hits that many + # down, it has maxed its contribution to the npcs risk budget and a + # DIFFERENT domain can only add a node if the combined risk across + # every affected domain (each capped at chunks_per_domain) still + # leaves room. This reduces to the familiar "up to npcs whole domains + # are free" rule when there are >= ndcs+npcs domains (chunks_per_domain + # == 1). See the analogous fdDrainGate in nodedrain_controller.go. domains_available = len({n.failure_domain for n in snodes if n.failure_domain >= 0}) domains_needed = ndcs + npcs - active_domains = {n.failure_domain for n in not_online_nodes if n.failure_domain >= 0} - # jm_replication_active can't be attributed to a specific domain (the - # probe only says "some online node's journal is behind"), so treat - # it conservatively as always adding a new, unaccounted-for domain. - active_domain_count = len(active_domains) + (1 if jm_replication_active else 0) - my_domain = snode.failure_domain + chunks_per_domain = -(-domains_needed // domains_available) if domains_available > 0 else domains_needed - if my_domain not in active_domains: - if domains_needed > 0 and domains_available >= domains_needed: - if active_domain_count >= capacity_cap: - blocked_by_capacity = True - capacity_reason = ( - f"FTT={ft} (npcs={npcs}): cannot remove node, failure domain {my_domain} " - f"not yet active; {active_domain_count}/{capacity_cap} domains active" - f"{' (including in-progress journal replication)' if jm_replication_active else ''}" - ) - elif active_domain_count > 0 and not_online_count >= capacity_cap: + domain_down_counts: dict[int, int] = {} + for node in not_online_nodes: + if node.failure_domain >= 0: + domain_down_counts[node.failure_domain] = domain_down_counts.get(node.failure_domain, 0) + 1 + + my_domain = snode.failure_domain + my_domain_down = domain_down_counts.get(my_domain, 0) + + if my_domain_down < chunks_per_domain: + current_risk = sum(min(c, chunks_per_domain) for c in domain_down_counts.values()) + # jm_replication_active can't be attributed to a specific domain + # (the probe only says "some online node's journal is behind"), + # so treat it conservatively as always adding a fresh risk unit. + if jm_replication_active: + current_risk += 1 + if current_risk + 1 > npcs: blocked_by_capacity = True capacity_reason = ( - f"FTT={ft} (npcs={npcs}): insufficient failure domains " - f"({domains_available} available, {domains_needed} needed for full isolation) " - f"to remove node in failure domain {my_domain}; " - f"{not_online_count}/{capacity_cap} nodes active" + f"FTT={ft} (npcs={npcs}): cannot remove node in failure domain {my_domain}; " + f"{current_risk}/{npcs} failure-domain risk budget already committed " + f"({domains_available} domain(s) available, {chunks_per_domain} chunk(s)/domain worst case)" + f"{' (including in-progress journal replication)' if jm_replication_active else ''}" ) + # else: this domain already holds >= chunks_per_domain down nodes -- + # it has maxed its contribution to the risk budget, so one more node + # in the SAME domain adds no additional risk. elif npcs == 1: # FTT=1: no room at all if anything is already not online or journal replicating if not_online_count > 0: diff --git a/tests/unit/test_ftt_protection.py b/tests/unit/test_ftt_protection.py index 8ba8a598f3..65c95cbb76 100644 --- a/tests/unit/test_ftt_protection.py +++ b/tests/unit/test_ftt_protection.py @@ -579,12 +579,22 @@ def test_jm_replication_counts_but_pair_still_checked(self): # --------------------------------------------------------------------------- # Failure-domain-aware capacity check # -# With FD enabled, the capacity cap (npcs) counts distinct affected failure -# domains instead of raw not-online nodes, mirroring the operator's -# fdDrainGate (nodedrain_controller.go) -- piling onto an already-affected -# domain is always free; a *new* domain is gated on distinct-domain count -# when there are >= ndcs+npcs domains, or on the plain node-count cap -# otherwise (under-provisioned). +# With FD enabled, the risk budget is npcs, spent per domain at a rate of +# min(nodes_down_in_that_domain, chunks_per_domain), where chunks_per_domain +# = ceil((ndcs+npcs)/domains_available). A domain that already has +# chunks_per_domain nodes down has maxed its contribution -- further nodes in +# THAT SAME domain are free -- but a node in a domain that hasn't maxed out +# yet is only allowed if the combined risk across every affected domain still +# leaves room in the npcs budget. This mirrors the operator's fdDrainGate +# (nodedrain_controller.go) and reduces to "up to npcs whole domains are +# free" when there are >= ndcs+npcs domains (chunks_per_domain == 1). +# +# Confirmed against the backend team's stated requirements (2026-08, Dmitrii +# Iakovlev): for a 2+2 layout, +# - 2 failure domains: safe combos are "1 whole FD down" (any node count +# within it) OR "1 node in each of the 2 FDs" -- nothing beyond that. +# - 3 failure domains: only 1 FD may be fully down, not 2. +# - 4 failure domains: 2 FDs may be fully down (the well-provisioned case). # --------------------------------------------------------------------------- class TestFailureDomainAwareCapacity(unittest.TestCase): @@ -666,7 +676,151 @@ def test_under_provisioned_second_domain_blocked_at_node_budget(self): db = _db(cl, nodes) allowed, reason = _check_ftt_allows_node_removal("n1", db) self.assertFalse(allowed) - self.assertIn("insufficient failure domains", reason) + self.assertIn("risk budget", reason) + + # --- Dmitrii's confirmed scenarios: 2 FD / 2+2 --------------------------- + # chunks_per_domain = ceil(4/2) = 2. Safe: "1 whole FD" or "1 node per FD", + # nothing more -- verified against the specific combinations he ruled out. + + def test_2fd_one_whole_fd_down_is_safe(self): + """Domain 1 already has 3-of-4 nodes down; the 4th is still allowed + (piling within a single domain, which may go fully down).""" + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=1), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n4", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n5", failure_domain=2), _node("n6", failure_domain=2), + _node("n7", failure_domain=2), _node("n8", failure_domain=2), + ] + db = _db(cl, nodes) + allowed, _ = _check_ftt_allows_node_removal("n1", db) + self.assertTrue(allowed) + + def test_2fd_one_node_per_fd_is_safe(self): + """1 node down in domain 1; removing 1 node in domain 2 is safe.""" + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=2), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", failure_domain=1), + _node("n4", failure_domain=2), + ] + db = _db(cl, nodes) + allowed, _ = _check_ftt_allows_node_removal("n1", db) + self.assertTrue(allowed) + + def test_2fd_whole_fd_plus_other_fd_node_is_unsafe(self): + """Domain 1 fully down (4 nodes) -- a node in domain 2 must now be + blocked (not one of the two safe combinations).""" + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=2), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n4", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n5", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n6", failure_domain=2), _node("n7", failure_domain=2), _node("n8", failure_domain=2), + ] + db = _db(cl, nodes) + allowed, _ = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + + def test_2fd_one_per_fd_plus_second_node_in_same_fd_is_unsafe(self): + """1 node down in EACH of domains 1 and 2 already (the safe combo) -- + piling a SECOND node onto domain 2 must now be blocked, even though + domain 2 is already 'active'. This is the exact gap the old + unconditional-piling logic missed.""" + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=2), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", status=StorageNode.STATUS_OFFLINE, failure_domain=2), + _node("n4", failure_domain=1), + ] + db = _db(cl, nodes) + allowed, _ = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + + def test_2fd_one_per_fd_plus_second_node_in_other_fd_is_unsafe(self): + """Same setup, but piling the extra node onto domain 1 (the first, + 'already active' domain) instead of domain 2 -- also unsafe.""" + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=1), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", status=StorageNode.STATUS_OFFLINE, failure_domain=2), + _node("n4", failure_domain=2), + ] + db = _db(cl, nodes) + allowed, _ = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + + # --- Dmitrii's confirmed scenarios: 3 FD / 2+2 --------------------------- + # chunks_per_domain = ceil(4/3) = 2 -- same per-domain cap as 2 FD, but + # spread across 3 domains. Confirmed: 1 FD fully down is safe, 2 FDs is not. + + def test_3fd_one_whole_fd_down_is_safe(self): + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=1), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n4", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n5", failure_domain=2), _node("n6", failure_domain=3), + ] + db = _db(cl, nodes) + allowed, _ = _check_ftt_allows_node_removal("n1", db) + self.assertTrue(allowed) + + def test_3fd_two_whole_fds_down_is_unsafe(self): + """Domain 1 fully down already (2+ nodes, maxing its chunks_per_domain + budget) -- a node in domain 2 must be blocked.""" + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=2), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n4", failure_domain=2), _node("n5", failure_domain=3), + ] + db = _db(cl, nodes) + allowed, _ = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + + # --- Dmitrii's confirmed scenarios: 4 FD / 2+2 (well-provisioned) -------- + # chunks_per_domain = ceil(4/4) = 1. Confirmed: 2 whole FDs down is safe. + + def test_4fd_two_whole_fds_down_is_safe(self): + """Domain 1 already has 3-of-4 nodes down (maxed, chunks_per_domain=1 + means it only ever contributes 1 to the budget regardless of count); + removing a node in domain 2 -- opening the second domain -- is still + within the npcs=2 budget.""" + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=2), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n4", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n5", failure_domain=2), _node("n6", failure_domain=3), _node("n7", failure_domain=4), + ] + db = _db(cl, nodes) + allowed, _ = _check_ftt_allows_node_removal("n1", db) + self.assertTrue(allowed) + + def test_4fd_three_whole_fds_down_is_unsafe(self): + """Domains 1 and 2 already have a node down each (budget=2/2 spent); + removing a node in a THIRD domain (3) must be blocked.""" + cl = _cluster(npcs=2, ndcs=2, ft=2, enable_failure_domain=True) + nodes = [ + _node("n1", failure_domain=3), + _node("n2", status=StorageNode.STATUS_OFFLINE, failure_domain=1), + _node("n3", status=StorageNode.STATUS_OFFLINE, failure_domain=2), + _node("n4", failure_domain=4), + ] + db = _db(cl, nodes) + allowed, _ = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) def test_fd_enabled_but_node_unassigned_falls_back_to_node_count(self): """FD enabled cluster-wide, but this specific node has no domain From b05bcca002d80177185f3ff5c333a6bb6cc5c06b Mon Sep 17 00:00:00 2001 From: wmousa Date: Wed, 5 Aug 2026 18:10:25 +0200 Subject: [PATCH 05/27] fix(cluster-activate): require npcs+2 domains, not npcs+1, at fresh activation The bare correctness minimum for the interleaved rotation layout is npcs+1 distinct domains (2 for npcs=1, 3 for npcs=2) -- below that even the initial static placement is wrong: at exactly 2 domains the tertiary role mathematically always lands back in the primary's own domain, since "2 steps ahead" in a period-2 round-robin wraps to where it started (verified directly against rotation_layout: 8/8 tertiary placements landed same-domain at 2 domains, 0/12 at 3+). But a minimum-correct STATIC layout has zero spare hosts per domain, and the moment a single node is added or removed, the relocation logic (_pick_replica_relocation_node) has no spare candidate left to reassign the stranded role to. Verified directly: removing one node from a bare-minimum npcs=1/2-domain or npcs=2/3-domain layout strands another node's secondary/tertiary with _pick_replica_relocation_node returning None -- blocking the removal outright, not just degrading placement quality. This also matches the backend team's confirmed stance that a 2-FD layout can never absorb a second independent failure once one domain is down, so it's excluded at any npcs level. Fresh activation now hard-requires npcs+2 distinct domains (3 for npcs=1, 4 for npcs=2) -- one domain of spare capacity beyond the bare correctness floor, so a later single add/remove has somewhere to place the relocated role. Extracted as fd_activation_domain_count_violation() in planner.py (alongside fd_balance_violation, same pattern) since _cluster_activate itself has no unit-test mocking infrastructure and was otherwise untestable; 7 new tests cover the boundary directly. --- simplyblock_core/cluster_ops.py | 24 +++++++---- .../controllers/cluster_expansion/planner.py | 41 +++++++++++++++++++ tests/unit/test_fd_topology_policy.py | 35 ++++++++++++++++ 3 files changed, 92 insertions(+), 8 deletions(-) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 6e493a5eec..f4cd88ff47 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -976,14 +976,16 @@ def _cluster_activate(cl_id, force=False, force_lvstore_create=False) -> None: set_cluster_status(cl_id, ols_status) raise - # Failure-domain coverage check (best-effort: warn, don't block). To - # survive losing a whole failure domain we need at least npcs+1 distinct - # domains; with fewer, placement falls back to host-disjoint and a domain - # outage may exceed the cluster's fault tolerance. + # Failure-domain coverage check (best-effort: warn, don't block). A 2-FD + # layout can never absorb a second independent failure once one domain + # is down, so the hard minimum below (enforced at fresh activation) is + # npcs+2, not npcs+1 -- this warning uses the same number so a + # reactivation that's short of it gets the same signal without being + # blocked (recovering a drifted layout must not turn into an outage). fd_desired_layout: t.Dict[str, t.Tuple[str, str]] = {} if cluster.enable_failure_domain: distinct_domains = {node.failure_domain for node in online_nodes if node.failure_domain >= 0} - min_domains = cluster.distr_npcs + 1 + min_domains = cluster.distr_npcs + 2 if len(distinct_domains) < min_domains: logger.warning( "Failure-domain feature is enabled but only %d distinct failure " @@ -1023,9 +1025,15 @@ def _fd_fail(msg: str) -> None: f"a host must sit entirely in one domain") fd_host_counts = Counter(host_fd.values()) - if len(fd_host_counts) < 2: - _fd_fail("failure domains are enabled but all hosts are in a " - "single domain; at least two domains are required") + # See fd_activation_domain_count_violation's docstring: npcs+2 + # domains, not just the bare rotation-correctness minimum, so a + # later single add/remove has a spare candidate instead of + # stranding another node's secondary/tertiary with none at all. + # This also subsumes the plain "at least two domains" floor. + domain_count_violation = fd_planner.fd_activation_domain_count_violation( + cluster.distr_npcs, len(fd_host_counts)) + if domain_count_violation: + _fd_fail(domain_count_violation) if len(set(fd_host_counts.values())) != 1: _fd_fail( f"failure domains must hold an EQUAL number of hosts at " diff --git a/simplyblock_core/controllers/cluster_expansion/planner.py b/simplyblock_core/controllers/cluster_expansion/planner.py index 324a2b79de..4f4adf9ac1 100644 --- a/simplyblock_core/controllers/cluster_expansion/planner.py +++ b/simplyblock_core/controllers/cluster_expansion/planner.py @@ -503,6 +503,47 @@ def fd_balance_violation( return None +def fd_activation_domain_count_violation( + npcs: int, distinct_domain_count: int, +) -> Optional[str]: + """Validate the number of distinct failure domains for fresh activation. + + A 2-FD layout can never absorb a second independent failure once one + domain is fully down (confirmed with the backend team), so it is not + supported at any npcs level. + + The bare *correctness* minimum for the rotation layout itself is + npcs+1 (e.g. 2 domains for npcs=1, 3 for npcs=2 -- below that even the + initial static placement is wrong: at exactly 2 domains the tertiary + role mathematically always lands back in the primary's own domain, + since "2 steps ahead" in a period-2 round-robin wraps to where it + started; verified directly against rotation_layout()). But a + minimum-correct STATIC layout has zero spare hosts per domain, and the + moment a single node is added or removed, the relocation logic + (_pick_replica_relocation_node) has no spare candidate left to + reassign the stranded role to -- verified directly: removing one node + from a bare-minimum npcs=1/2-domain or npcs=2/3-domain layout strands + another node's secondary/tertiary with no replacement at all, blocking + the removal outright rather than just degrading placement quality. + + Requiring npcs+2 domains (3 for npcs=1, 4 for npcs=2, which also rules + out exactly 2 for both) keeps one domain of spare capacity beyond the + bare correctness floor, so a single add/remove has somewhere to place + the relocated role instead of failing immediately. Returns a + human-readable reason on violation, ``None`` when the count is + acceptable. + """ + min_domains = npcs + 2 + if distinct_domain_count < min_domains: + return ( + f"failure domains are enabled with npcs={npcs}, which requires at " + f"least {min_domains} distinct failure domains (2 domains is not " + f"supported at any npcs level); currently have " + f"{distinct_domain_count}. Add hosts in additional domains, or " + f"disable failure domains, then activate.") + return None + + # --------------------------------------------------------------------------- # Persistence helpers for ``Cluster.expand_state``. # diff --git a/tests/unit/test_fd_topology_policy.py b/tests/unit/test_fd_topology_policy.py index f8540d977a..46478d0c3f 100644 --- a/tests/unit/test_fd_topology_policy.py +++ b/tests/unit/test_fd_topology_policy.py @@ -178,6 +178,41 @@ def test_unset_domain_ignored(self): self.assertIsNone(planner.fd_balance_violation({-1: 7, 0: 2, 1: 2})) +# --------------------------------------------------------------------------- +# planner.fd_activation_domain_count_violation +# --------------------------------------------------------------------------- + +class TestFdActivationDomainCount(unittest.TestCase): + + def test_npcs1_two_domains_violates(self): + self.assertIsNotNone( + planner.fd_activation_domain_count_violation(1, 2)) + + def test_npcs1_three_domains_ok(self): + self.assertIsNone( + planner.fd_activation_domain_count_violation(1, 3)) + + def test_npcs1_one_domain_violates(self): + self.assertIsNotNone( + planner.fd_activation_domain_count_violation(1, 1)) + + def test_npcs2_two_domains_violates(self): + self.assertIsNotNone( + planner.fd_activation_domain_count_violation(2, 2)) + + def test_npcs2_three_domains_violates(self): + self.assertIsNotNone( + planner.fd_activation_domain_count_violation(2, 3)) + + def test_npcs2_four_domains_ok(self): + self.assertIsNone( + planner.fd_activation_domain_count_violation(2, 4)) + + def test_npcs2_more_than_four_domains_ok(self): + self.assertIsNone( + planner.fd_activation_domain_count_violation(2, 6)) + + # --------------------------------------------------------------------------- # preconditions: add / remove / current admission # --------------------------------------------------------------------------- From f9515a73fd6544e3f36e88cacd13d4c432087b15 Mon Sep 17 00:00:00 2001 From: wmousa Date: Fri, 7 Aug 2026 17:38:04 +0200 Subject: [PATCH 06/27] fix(node-removal): splice a stranded replica into an existing pairing when no free cross-domain candidate exists get_secondary_nodes/get_secondary_nodes_2 only ever offer UNCLAIMED nodes (each node hosts at most one secondary/tertiary at a time). A node removal frees exactly one node cluster-wide -- whoever hosted the removed node's own role -- so _pick_replica_relocation_node has exactly one candidate to work with. If that one candidate lands in the wrong failure domain (or nothing is free at all), the direct search had nothing else to offer even though a valid rearrangement exists elsewhere in the cluster. Verified directly: two removals in a row (each individually fine) can chain into exactly this dead end -- the second removal's repair needs a new cross-domain home, the only free node is same-domain, and the search gave up. Confirmed the same 9-node/3-domain topology that hit this now resolves via the new fallback. Adds _find_splice_target_for_relocation, generalizing splice_stranded_secondary/splice_stranded_tertiary's fix for the identical dead end at cluster-activation time (splice into an already-formed pairing P->X instead of requiring an idle node) to the removal-repair path, with an exclude list for the node being removed. _pick_replica_relocation_node now falls back to it whenever the direct search comes up empty. Unlike the activation-time splice helpers -- which only ever run before any physical LVS exists -- this can be asked to splice into a pairing that already has real data on both ends, so _relocate_one_replica gained _relocate_replica_between to execute it: evict the existing occupant onto the node being relocated (tear down + rebuild), then claim the freed slot. Both legs follow the same idempotent, commit-pointers-then-build pattern _relocate_one_replica already uses, so a crash mid-splice resumes cleanly on retry. --- simplyblock_core/storage_node_ops.py | 154 +++++++++++++- tests/unit/test_node_removal.py | 294 ++++++++++++++++++++++++++- 2 files changed, 438 insertions(+), 10 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index cce011441d..25887a2b84 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -3560,7 +3560,23 @@ def _pick_replica_relocation_node(primary, removed_node: StorageNode, role, db_c enforced HARD here (not best-effort): if the primary's OTHER non-leader role does not already live in a different domain than the primary, the replacement must — otherwise a full-domain outage would leave the LVS - with zero surviving paths. Returning None makes + with zero surviving paths. + + ``get_secondary_nodes``/``get_secondary_nodes_2`` only ever offer + UNCLAIMED nodes (each node hosts at most one secondary/tertiary at a + time — ``lvstore_stack_secondary``/``_tertiary`` is a single field, not + a list). A removal frees exactly one node system-wide (whoever hosted + ``removed_node``'s own role); if that one lands in the wrong domain — + or nothing is free at all — the direct search has nothing else to + offer even though a valid rearrangement exists elsewhere in the + cluster (2026-08-07, chained-removal incident: two removals in a row + stranded a third node's secondary with zero free cross-domain + candidates, while an existing pairing two hops away could have + absorbed it). Falls back to splicing ``primary`` into an already-formed + pairing (see ``_find_splice_target_for_relocation``) — exactly the fix + ``splice_stranded_secondary``/``splice_stranded_tertiary`` already apply + to the identical dead end at cluster-activation time. Only returning + None here (both searches exhausted) makes ``_check_replica_relocation_feasible`` refuse the removal up front. """ exclude_ids = [removed_node.get_id()] @@ -3581,12 +3597,9 @@ def _pick_replica_relocation_node(primary, removed_node: StorageNode, role, db_c pass cands = get_secondary_nodes_2( primary, exclude_ids=exclude_ids, exclude_mgmt_ips=exclude_mgmt_ips) - if not cands: - return None cluster = db_controller.get_cluster_by_id(primary.cluster_id) - if (getattr(cluster, "enable_failure_domain", False) - and primary.failure_domain >= 0): + if cands and getattr(cluster, "enable_failure_domain", False) and primary.failure_domain >= 0: other_cross = False if other_id and other_id != removed_node.get_id(): try: @@ -3606,8 +3619,84 @@ def _pick_replica_relocation_node(primary, removed_node: StorageNode, role, db_c if (cand.failure_domain >= 0 and cand.failure_domain != primary.failure_domain): return cand_id - return None - return cands[0] + else: + return cands[0] + elif cands: + return cands[0] + + splice = _find_splice_target_for_relocation( + primary, role, db_controller, exclude_ids=exclude_ids + [primary.get_id()]) + return splice[1] if splice else None + + +def _find_splice_target_for_relocation(stranded_primary, role, db_controller, exclude_ids=()): + """Find an already-formed pairing ``P -> X`` (``P. == X``) + elsewhere in the cluster to splice ``stranded_primary`` into: + ``P -> stranded_primary -> X``. Read-only — callers decide whether and + how to execute the resulting move (see ``_relocate_one_replica``). + + Generalizes ``splice_stranded_secondary``/``splice_stranded_tertiary``'s + edge search (same scoring: prefer both ends domain-disjoint from the + stranded node, then relax) with an ``exclude_ids`` list, so the + node-removal repair path can rule out the node being removed and any + other already-claimed id. Unlike the activation-time splice helpers — + which only ever run before any physical LVS exists — this can be asked + to splice into a pairing that already has real data on both ends; + executing that move (not just picking the edge) is the caller's job. + + Returns ``(p_id, x_id)`` or ``None`` if no valid edge exists. + """ + field = "secondary_node_id" if role == "secondary" else "tertiary_node_id" + all_nodes = db_controller.get_storage_nodes_by_cluster_id(stranded_primary.cluster_id) + all_nodes = sorted(all_nodes, key=lambda n: n.failure_domain) + by_id = {n.get_id(): n for n in all_nodes} + exclude = set(exclude_ids) | {stranded_primary.get_id()} + + stranded_sec = None + if role == "tertiary" and stranded_primary.secondary_node_id: + stranded_sec = by_id.get(stranded_primary.secondary_node_id) + + def _online(*nodes): + return all(n.status == StorageNode.STATUS_ONLINE for n in nodes) + + def _valid_tertiary(node, node_sec, candidate): + if candidate.get_id() == node.get_id(): + return False + if candidate.mgmt_ip == node.mgmt_ip: + return False + if node_sec and candidate.mgmt_ip == node_sec.mgmt_ip: + return False + return True + + def _domain_mismatch_score(*nodes): + if stranded_primary.failure_domain < 0: + return 0 + return sum(1 for n in nodes if n.failure_domain != stranded_primary.failure_domain) + + edges = [n for n in all_nodes if getattr(n, field) and n.get_id() not in exclude] + + best, best_score = None, -1 + for p in edges: + x_id = getattr(p, field) + if x_id in exclude: + continue + x = by_id.get(x_id) + if not x or not _online(p, x): + continue + if role == "secondary": + if p.mgmt_ip == stranded_primary.mgmt_ip or x.mgmt_ip == stranded_primary.mgmt_ip: + continue + else: + p_sec = by_id.get(p.secondary_node_id) if p.secondary_node_id else None + if not _valid_tertiary(p, p_sec, stranded_primary): + continue + if not _valid_tertiary(stranded_primary, stranded_sec, x): + continue + score = _domain_mismatch_score(p, x) + if score > best_score: + best_score, best = score, (p.get_id(), x.get_id()) + + return best def node_removal_orchestrate(node_id, force_remove=False): @@ -3796,6 +3885,21 @@ def _relocate_one_replica(removed_node: StorageNode, primary_id, role): logger.error( f"[REMOVAL] no relocation target for {role} replica of {primary_id}") return False + + new_node = db_controller.get_storage_node_by_id(new_id) + occupant_id = getattr(new_node, backref) + if occupant_id and occupant_id not in (primary_id, removed_node.get_id()): + # _pick_replica_relocation_node fell back to a splice candidate: + # new_id is currently busy hosting occupant_id's replica. Evict + # that occupant onto `primary` (the node whose replica we're + # relocating) before claiming new_id for primary's own role — + # see _find_splice_target_for_relocation's docstring. + if not _relocate_replica_between(occupant_id, new_id, primary_id, role, db_controller): + logger.error( + f"[REMOVAL] failed to splice {primary_id} into the pairing " + f"occupying {new_id} (occupant {occupant_id})") + return False + primary = db_controller.get_storage_node_by_id(primary_id) setattr(primary, field, new_id) primary.write_to_db() @@ -3819,6 +3923,42 @@ def _relocate_one_replica(removed_node: StorageNode, primary_id, role): return True +def _relocate_replica_between(occupant_primary_id, old_host_id, new_host_id, role, db_controller): + """Physically move ``occupant_primary_id``'s ``role`` replica off + ``old_host_id`` onto ``new_host_id``, updating its forward pointer. + + Used by the splice fallback in ``_relocate_one_replica``: before an + already-busy node can be claimed for the primary being relocated, its + current occupant must move onto that primary's node instead (see + ``_find_splice_target_for_relocation``'s docstring for why an + already-formed pairing, not an idle node, is what's available). + + Idempotent: skips the pointer flip (and the teardown it implies) if a + prior attempt already committed it; ``recreate_lvstore_on_non_leader`` + is retried unconditionally either way, matching ``_relocate_one_replica``'s + own idempotency pattern. Returns True if ``occupant_primary_id`` no + longer exists — nothing left to relocate. + """ + field = "secondary_node_id" if role == "secondary" else "tertiary_node_id" + try: + occupant_primary = db_controller.get_storage_node_by_id(occupant_primary_id) + except KeyError: + return True + + if getattr(occupant_primary, field) != new_host_id: + old_host = db_controller.get_storage_node_by_id(old_host_id) + cluster = db_controller.get_cluster_by_id(occupant_primary.cluster_id) + if old_host.status == StorageNode.STATUS_ONLINE: + _delete_replica_on_peer(old_host, occupant_primary, cluster) + occupant_primary = db_controller.get_storage_node_by_id(occupant_primary_id) + setattr(occupant_primary, field, new_host_id) + occupant_primary.write_to_db() + + new_host = db_controller.get_storage_node_by_id(new_host_id) + occupant_primary = db_controller.get_storage_node_by_id(occupant_primary_id) + return bool(recreate_lvstore_on_non_leader(new_host, occupant_primary, occupant_primary)) + + def _clear_replica_backref(removed_node: StorageNode, backref): db_controller = DBController() removed_node = db_controller.get_storage_node_by_id(removed_node.get_id()) diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 3a0b78840a..f7461edeca 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -30,7 +30,8 @@ # Fixtures # --------------------------------------------------------------------------- -def _cluster(ha_type="ha", npcs=1, ndcs=2, ft=1, mode="docker"): +def _cluster(ha_type="ha", npcs=1, ndcs=2, ft=1, mode="docker", + enable_failure_domain=False): cl = Cluster() cl.uuid = "cluster-1" cl.ha_type = ha_type @@ -40,12 +41,14 @@ def _cluster(ha_type="ha", npcs=1, ndcs=2, ft=1, mode="docker"): cl.mode = mode cl.status = Cluster.STATUS_ACTIVE cl.nqn = "nqn.2023-01.io.simplyblock:cluster-1" + cl.enable_failure_domain = enable_failure_domain return cl def _node(node_id, status=StorageNode.STATUS_ONLINE, lvstore="", secondary_id="", tertiary_id="", - stack_secondary="", stack_tertiary="", n_devices=0, with_jm=False): + stack_secondary="", stack_tertiary="", n_devices=0, with_jm=False, + failure_domain=-1, mgmt_ip=None): n = MagicMock(spec=StorageNode) n.uuid = node_id n.get_id = MagicMock(return_value=node_id) @@ -59,7 +62,8 @@ def _node(node_id, status=StorageNode.STATUS_ONLINE, lvstore="", n.tertiary_node_id = tertiary_id n.lvstore_stack_secondary = stack_secondary n.lvstore_stack_tertiary = stack_tertiary - n.mgmt_ip = f"10.0.0.{abs(hash(node_id)) % 250 + 1}" + n.failure_domain = failure_domain + n.mgmt_ip = mgmt_ip or f"10.0.0.{abs(hash(node_id)) % 250 + 1}" n.write_to_db = MagicMock() n.rpc_client = MagicMock(return_value=MagicMock()) n.hublvol_nqn_for_lvstore = MagicMock(return_value=f"nqn:hub:{lvstore}") @@ -260,6 +264,160 @@ def test_pick_secondary_uses_get_secondary_nodes(self): self.assertIn("n9", kwargs["exclude_ids"]) +# --------------------------------------------------------------------------- +# _find_splice_target_for_relocation — the removal-repair fallback used when +# get_secondary_nodes/_2 offer no free (unclaimed) cross-domain candidate. +# +# Regression coverage for the 2026-08-07 chained-removal incident: two +# removals in a row can strand a third node's secondary with zero free +# cross-domain candidates, even though an existing pairing two hops away +# could absorb it. Mirrors splice_stranded_secondary/_tertiary's edge search +# (used for the identical dead end at activation time), generalized with an +# exclude list for the removal path. +# --------------------------------------------------------------------------- + +class TestFindSpliceTargetForRelocation(unittest.TestCase): + + def test_splices_into_existing_secondary_edge(self): + cl = _cluster(enable_failure_domain=True) + p = _node("p", secondary_id="x", failure_domain=0, mgmt_ip="10.0.0.1") + x = _node("x", failure_domain=1, mgmt_ip="10.0.0.2") + stranded = _node("s", failure_domain=2, mgmt_ip="10.0.0.3") + db = FakeDB(cl, [p, x, stranded]) + got = storage_node_ops._find_splice_target_for_relocation(stranded, "secondary", db) + self.assertEqual(got, ("p", "x")) + + def test_prefers_edge_domain_disjoint_on_both_ends(self): + cl = _cluster(enable_failure_domain=True) + stranded = _node("s", failure_domain=0, mgmt_ip="10.0.0.9") + bad_p = _node("bad_p", secondary_id="bad_x", failure_domain=0, mgmt_ip="10.0.0.1") + bad_x = _node("bad_x", failure_domain=0, mgmt_ip="10.0.0.2") + good_p = _node("good_p", secondary_id="good_x", failure_domain=1, mgmt_ip="10.0.0.3") + good_x = _node("good_x", failure_domain=2, mgmt_ip="10.0.0.4") + db = FakeDB(cl, [stranded, bad_p, bad_x, good_p, good_x]) + got = storage_node_ops._find_splice_target_for_relocation(stranded, "secondary", db) + self.assertEqual(got, ("good_p", "good_x")) + + def test_excludes_ids_passed_by_caller(self): + cl = _cluster(enable_failure_domain=True) + p = _node("p", secondary_id="x", failure_domain=0, mgmt_ip="10.0.0.1") + x = _node("x", failure_domain=1, mgmt_ip="10.0.0.2") + stranded = _node("s", failure_domain=2, mgmt_ip="10.0.0.3") + db = FakeDB(cl, [p, x, stranded]) + got = storage_node_ops._find_splice_target_for_relocation( + stranded, "secondary", db, exclude_ids=["x"]) + self.assertIsNone(got) + + def test_no_edge_exists_returns_none(self): + cl = _cluster(enable_failure_domain=True) + stranded = _node("s", failure_domain=0, mgmt_ip="10.0.0.1") + other = _node("other", failure_domain=1, mgmt_ip="10.0.0.2") + db = FakeDB(cl, [stranded, other]) + got = storage_node_ops._find_splice_target_for_relocation(stranded, "secondary", db) + self.assertIsNone(got) + + def test_skips_edge_with_offline_endpoint(self): + cl = _cluster(enable_failure_domain=True) + p = _node("p", secondary_id="x", failure_domain=0, mgmt_ip="10.0.0.1") + x = _node("x", failure_domain=1, mgmt_ip="10.0.0.2", + status=StorageNode.STATUS_OFFLINE) + stranded = _node("s", failure_domain=2, mgmt_ip="10.0.0.3") + db = FakeDB(cl, [p, x, stranded]) + got = storage_node_ops._find_splice_target_for_relocation(stranded, "secondary", db) + self.assertIsNone(got) + + def test_skips_edge_not_host_disjoint_from_stranded(self): + stranded = _node("s", failure_domain=0, mgmt_ip="10.0.0.5") + p = _node("p", secondary_id="x", failure_domain=1, mgmt_ip="10.0.0.1") + x = _node("x", failure_domain=2, mgmt_ip="10.0.0.5") # shares stranded's host + cl = _cluster(enable_failure_domain=True) + db = FakeDB(cl, [stranded, p, x]) + got = storage_node_ops._find_splice_target_for_relocation(stranded, "secondary", db) + self.assertIsNone(got) + + def test_tertiary_edge_respects_secondary_host_disjointness(self): + cl = _cluster(enable_failure_domain=True) + stranded = _node("s", failure_domain=0, secondary_id="s_sec", mgmt_ip="10.0.0.9") + s_sec = _node("s_sec", failure_domain=1, mgmt_ip="10.0.0.50") + p = _node("p", tertiary_id="x", failure_domain=0, mgmt_ip="10.0.0.60") + x = _node("x", failure_domain=1, mgmt_ip="10.0.0.61") + db = FakeDB(cl, [stranded, s_sec, p, x]) + got = storage_node_ops._find_splice_target_for_relocation(stranded, "tertiary", db) + self.assertEqual(got, ("p", "x")) + + +# --------------------------------------------------------------------------- +# _pick_replica_relocation_node — falls back to the splice finder above when +# the direct free-candidate search comes up empty (no candidates at all, or +# none that satisfy the hard cross-domain requirement). +# --------------------------------------------------------------------------- + +class TestPickReplicaRelocationSpliceFallback(unittest.TestCase): + + def test_falls_back_to_splice_when_no_free_candidate_at_all(self): + cl = _cluster(enable_failure_domain=True) + primary = _node("p1", secondary_id="n1", failure_domain=2, mgmt_ip="10.0.0.9") + removed = _node("n1", failure_domain=1, mgmt_ip="10.0.0.99") + edge_p = _node("edge_p", secondary_id="edge_x", failure_domain=0, mgmt_ip="10.0.0.1") + edge_x = _node("edge_x", failure_domain=1, mgmt_ip="10.0.0.2") + db = FakeDB(cl, [primary, removed, edge_p, edge_x]) + with patch.object(storage_node_ops, "get_secondary_nodes", return_value=[]): + got = storage_node_ops._pick_replica_relocation_node( + primary, removed, "secondary", db) + self.assertEqual(got, "edge_x") + + def test_falls_back_to_splice_when_only_candidate_is_same_domain(self): + cl = _cluster(enable_failure_domain=True) + primary = _node("p1", secondary_id="n1", failure_domain=2, mgmt_ip="10.0.0.9") + removed = _node("n1", failure_domain=1, mgmt_ip="10.0.0.99") + same_domain_cand = _node("same_domain_cand", failure_domain=2, mgmt_ip="10.0.0.8") + edge_p = _node("edge_p", secondary_id="edge_x", failure_domain=0, mgmt_ip="10.0.0.1") + edge_x = _node("edge_x", failure_domain=1, mgmt_ip="10.0.0.2") + db = FakeDB(cl, [primary, removed, same_domain_cand, edge_p, edge_x]) + with patch.object(storage_node_ops, "get_secondary_nodes", + return_value=["same_domain_cand"]): + got = storage_node_ops._pick_replica_relocation_node( + primary, removed, "secondary", db) + self.assertEqual(got, "edge_x") + + def test_returns_none_when_no_free_and_no_splice_candidate(self): + cl = _cluster(enable_failure_domain=True) + primary = _node("p1", secondary_id="n1", failure_domain=0, mgmt_ip="10.0.0.9") + removed = _node("n1", failure_domain=1, mgmt_ip="10.0.0.99") + db = FakeDB(cl, [primary, removed]) + with patch.object(storage_node_ops, "get_secondary_nodes", return_value=[]): + got = storage_node_ops._pick_replica_relocation_node( + primary, removed, "secondary", db) + self.assertIsNone(got) + + def test_does_not_use_splice_when_free_cross_domain_candidate_exists(self): + cl = _cluster(enable_failure_domain=True) + primary = _node("p1", secondary_id="n1", failure_domain=0, mgmt_ip="10.0.0.9") + removed = _node("n1", failure_domain=1, mgmt_ip="10.0.0.99") + free1 = _node("free1", failure_domain=1, mgmt_ip="10.0.0.1") + db = FakeDB(cl, [primary, removed, free1]) + with patch.object(storage_node_ops, "get_secondary_nodes", return_value=["free1"]), \ + patch.object(storage_node_ops, "_find_splice_target_for_relocation") as finder: + got = storage_node_ops._pick_replica_relocation_node( + primary, removed, "secondary", db) + self.assertEqual(got, "free1") + finder.assert_not_called() + + def test_fd_disabled_never_needs_splice_when_candidate_exists(self): + # Non-FD clusters take the unconditional cands[0] path -- splice is + # only relevant once cands is genuinely empty. + cl = _cluster(enable_failure_domain=False) + primary = _node("p1", secondary_id="n1", mgmt_ip="10.0.0.9") + removed = _node("n1", mgmt_ip="10.0.0.99") + db = FakeDB(cl, [primary, removed]) + with patch.object(storage_node_ops, "get_secondary_nodes", return_value=["free1"]), \ + patch.object(storage_node_ops, "_find_splice_target_for_relocation") as finder: + got = storage_node_ops._pick_replica_relocation_node( + primary, removed, "secondary", db) + self.assertEqual(got, "free1") + finder.assert_not_called() + + # --------------------------------------------------------------------------- # Case A — teardown of own primary's replicas # --------------------------------------------------------------------------- @@ -411,6 +569,136 @@ def test_relocate_missing_primary_just_clears(self): self.assertEqual(removed.lvstore_stack_secondary, "") +# --------------------------------------------------------------------------- +# Case B, splice fallback — _pick_replica_relocation_node returned a BUSY +# node (a splice candidate, per _find_splice_target_for_relocation) instead +# of a free one. _relocate_one_replica must evict that node's current +# occupant onto the stranded primary before claiming the slot for itself. +# --------------------------------------------------------------------------- + +class TestRelocateOneReplicaSpliceExecution(unittest.TestCase): + + def test_relocate_via_splice_evicts_occupant_first(self): + cl = _cluster() + removed = _node("n1", stack_secondary="stranded") + stranded = _node("stranded", secondary_id="n1", lvstore="LVS_stranded") + occupant = _node("occupant", secondary_id="x", lvstore="LVS_occupant") + x = _node("x", stack_secondary="occupant") # x is busy, not free + db = FakeDB(cl, [removed, stranded, occupant, x]) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + return_value="x"), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + return_value=True) as rec, \ + patch.object(storage_node_ops, "_delete_replica_on_peer") as drp: + ret = storage_node_ops._relocate_one_replica(removed, "stranded", "secondary") + + self.assertTrue(ret) + drp.assert_called_once() # occupant's old replica torn down off x + self.assertEqual(occupant.secondary_node_id, "stranded") # occupant re-homed onto stranded + self.assertEqual(stranded.secondary_node_id, "x") # stranded takes over x's freed slot + self.assertEqual(x.lvstore_stack_secondary, "stranded") + self.assertEqual(rec.call_count, 2) # occupant's rebuild + stranded's own rebuild + self.assertEqual(removed.lvstore_stack_secondary, "") + + def test_relocate_via_splice_occupant_rebuild_failure_keeps_forward_pointers(self): + cl = _cluster() + removed = _node("n1", stack_secondary="stranded") + stranded = _node("stranded", secondary_id="n1", lvstore="LVS_stranded") + occupant = _node("occupant", secondary_id="x", lvstore="LVS_occupant") + x = _node("x", stack_secondary="occupant") + db = FakeDB(cl, [removed, stranded, occupant, x]) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + return_value="x"), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + return_value=False), \ + patch.object(storage_node_ops, "_delete_replica_on_peer"): + ret = storage_node_ops._relocate_one_replica(removed, "stranded", "secondary") + + self.assertFalse(ret) + # Forward pointer for occupant's move is committed even though the + # physical rebuild failed -- matches _relocate_one_replica's own + # idempotent-retry pattern: a retry resumes from the committed intent + # rather than re-picking (see test_relocate_resume_reuses_committed_target). + self.assertEqual(occupant.secondary_node_id, "stranded") + # The outer splice claim (stranded -> x) never got committed, since + # _relocate_replica_between reported failure first. + self.assertEqual(stranded.secondary_node_id, "n1") + self.assertEqual(removed.lvstore_stack_secondary, "stranded") + + def test_relocate_via_splice_own_rebuild_failure_after_occupant_moved(self): + # occupant's move succeeds (evicted + rebuilt on stranded), but + # stranded's own rebuild on the freed slot x fails. + cl = _cluster() + removed = _node("n1", stack_secondary="stranded") + stranded = _node("stranded", secondary_id="n1", lvstore="LVS_stranded") + occupant = _node("occupant", secondary_id="x", lvstore="LVS_occupant") + x = _node("x", stack_secondary="occupant") + db = FakeDB(cl, [removed, stranded, occupant, x]) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + return_value="x"), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + side_effect=[True, False]), \ + patch.object(storage_node_ops, "_delete_replica_on_peer") as drp: + ret = storage_node_ops._relocate_one_replica(removed, "stranded", "secondary") + + self.assertFalse(ret) + drp.assert_called_once() + self.assertEqual(occupant.secondary_node_id, "stranded") + # Forward pointers for stranded's own claim ARE committed (pre-build, + # same idempotent pattern) even though the rebuild on x failed. + self.assertEqual(stranded.secondary_node_id, "x") + self.assertEqual(x.lvstore_stack_secondary, "stranded") + self.assertEqual(removed.lvstore_stack_secondary, "stranded") + + def test_relocate_via_splice_tertiary_role(self): + cl = _cluster() + removed = _node("n1", stack_tertiary="stranded") + stranded = _node("stranded", tertiary_id="n1", lvstore="LVS_stranded") + occupant = _node("occupant", tertiary_id="x", lvstore="LVS_occupant") + x = _node("x", stack_tertiary="occupant") + db = FakeDB(cl, [removed, stranded, occupant, x]) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + return_value="x"), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + return_value=True), \ + patch.object(storage_node_ops, "_delete_replica_on_peer") as drp: + ret = storage_node_ops._relocate_one_replica(removed, "stranded", "tertiary") + + self.assertTrue(ret) + drp.assert_called_once() + self.assertEqual(occupant.tertiary_node_id, "stranded") + self.assertEqual(stranded.tertiary_node_id, "x") + self.assertEqual(x.lvstore_stack_tertiary, "stranded") + self.assertEqual(removed.lvstore_stack_tertiary, "") + + def test_relocate_free_target_never_triggers_splice_eviction(self): + # Regression guard: when the picked target is genuinely free (no + # backref set), _relocate_one_replica must behave exactly as before + # -- no eviction, no extra recreate_lvstore_on_non_leader call. + cl = _cluster() + removed = _node("n1", stack_secondary="p1") + primary = _node("p1", secondary_id="n1", lvstore="LVS_p1") + free_node = _node("n3") # stack_secondary="" -- genuinely free + db = FakeDB(cl, [removed, primary, free_node]) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + return_value="n3"), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + return_value=True) as rec, \ + patch.object(storage_node_ops, "_delete_replica_on_peer") as drp: + ret = storage_node_ops._relocate_one_replica(removed, "p1", "secondary") + + self.assertTrue(ret) + drp.assert_not_called() + rec.assert_called_once() + self.assertEqual(primary.secondary_node_id, "n3") + self.assertEqual(free_node.lvstore_stack_secondary, "p1") + + # --------------------------------------------------------------------------- # Device decommission completion gate # --------------------------------------------------------------------------- From c215e168d72b97785bf4d57c73af26c1abac0e9a Mon Sep 17 00:00:00 2001 From: wmousa Date: Fri, 7 Aug 2026 23:18:40 +0200 Subject: [PATCH 07/27] fix(node-removal): create-before-destroy in the splice fallback _relocate_replica_between tore down the occupant's existing, healthy replica BEFORE building its replacement -- between those two steps the occupant had zero surviving copies. Under FTT1 (no tertiary) that's a real gap: a cluster only tolerates one node down at a time, and that budget belongs to the node actually being removed, not to whatever unrelated, healthy node the splice happens to touch. Confirmed live (2026-08-07): removing a node correctly triggered the splice fallback, but the rebuild step hit a hublvol attach failure and RAISED an exception instead of returning False. That propagated uncaught, the task retried repeatedly, and the occupant's only copy sat torn down the whole time -- the cluster's health monitor eventually suspended it. Reorders to create-before-destroy: build the replacement on the new host first (old copy stays live and serving throughout); only tear down the old copy once the new one is confirmed. A raised exception from the rebuild is now caught and treated the same as a returned False -- both leave the old copy untouched and safe to retry. The teardown step is guarded by the old host's own back-reference (not the occupant's forward pointer), so a crash between the two commits still resumes the teardown on the next pass instead of leaking a stale replica. --- simplyblock_core/storage_node_ops.py | 55 ++++++++++++++++----- tests/unit/test_node_removal.py | 72 ++++++++++++++++++++++++---- 2 files changed, 106 insertions(+), 21 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 25887a2b84..5649dcb260 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -3933,30 +3933,61 @@ def _relocate_replica_between(occupant_primary_id, old_host_id, new_host_id, rol ``_find_splice_target_for_relocation``'s docstring for why an already-formed pairing, not an idle node, is what's available). - Idempotent: skips the pointer flip (and the teardown it implies) if a - prior attempt already committed it; ``recreate_lvstore_on_non_leader`` - is retried unconditionally either way, matching ``_relocate_one_replica``'s - own idempotency pattern. Returns True if ``occupant_primary_id`` no - longer exists — nothing left to relocate. + Create-before-destroy: the new replica is built on ``new_host_id`` + BEFORE the old one on ``old_host_id`` is torn down, so + ``occupant_primary`` never has zero surviving copies -- critical on + FTT1 (no tertiary): a cluster only tolerates one node down at a time, + and that budget belongs to the node actually being removed, not to + whatever healthy node this splice happens to touch. (2026-08-07 + incident: the old destroy-then-build order tore down the occupant's + only copy up front; a hublvol attach failure on the rebuild then + retried for minutes with that copy already gone.) A raised exception + from the rebuild is treated the same as a returned False -- both leave + the old copy untouched and safe to retry. + + Idempotent and retry-safe: "already built" is read from the occupant's + forward pointer, so a retry after a confirmed build skips straight to + the teardown check without re-running the rebuild. The teardown itself + is guarded separately by ``old_host``'s own back-reference (not the + occupant's forward pointer), so a crash between the two commits still + resumes the teardown on the next pass instead of leaking a stale + replica on ``old_host`` forever. + + Returns True if ``occupant_primary_id`` no longer exists — nothing left + to relocate. """ field = "secondary_node_id" if role == "secondary" else "tertiary_node_id" + backref = "lvstore_stack_secondary" if role == "secondary" else "lvstore_stack_tertiary" try: occupant_primary = db_controller.get_storage_node_by_id(occupant_primary_id) except KeyError: return True if getattr(occupant_primary, field) != new_host_id: - old_host = db_controller.get_storage_node_by_id(old_host_id) - cluster = db_controller.get_cluster_by_id(occupant_primary.cluster_id) - if old_host.status == StorageNode.STATUS_ONLINE: - _delete_replica_on_peer(old_host, occupant_primary, cluster) + new_host = db_controller.get_storage_node_by_id(new_host_id) + try: + built = recreate_lvstore_on_non_leader(new_host, occupant_primary, occupant_primary) + except Exception as e: + logger.error( + f"[REMOVAL] splice: failed to build {role} replica of " + f"{occupant_primary_id} on {new_host_id}, old copy on " + f"{old_host_id} left untouched: {e}") + return False + if not built: + return False occupant_primary = db_controller.get_storage_node_by_id(occupant_primary_id) setattr(occupant_primary, field, new_host_id) occupant_primary.write_to_db() - new_host = db_controller.get_storage_node_by_id(new_host_id) - occupant_primary = db_controller.get_storage_node_by_id(occupant_primary_id) - return bool(recreate_lvstore_on_non_leader(new_host, occupant_primary, occupant_primary)) + old_host = db_controller.get_storage_node_by_id(old_host_id) + if getattr(old_host, backref) == occupant_primary_id: + cluster = db_controller.get_cluster_by_id(occupant_primary.cluster_id) + if old_host.status == StorageNode.STATUS_ONLINE: + _delete_replica_on_peer(old_host, occupant_primary, cluster) + old_host = db_controller.get_storage_node_by_id(old_host_id) + setattr(old_host, backref, "") + old_host.write_to_db() + return True def _clear_replica_backref(removed_node: StorageNode, backref): diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index f7461edeca..4b02b09111 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -601,7 +601,11 @@ def test_relocate_via_splice_evicts_occupant_first(self): self.assertEqual(rec.call_count, 2) # occupant's rebuild + stranded's own rebuild self.assertEqual(removed.lvstore_stack_secondary, "") - def test_relocate_via_splice_occupant_rebuild_failure_keeps_forward_pointers(self): + def test_relocate_via_splice_occupant_rebuild_failure_leaves_old_copy_untouched(self): + # Create-before-destroy: a failed rebuild on the stranded node must + # NOT tear down or repoint the occupant's still-intact old copy on x + # -- that copy is occupant's ONLY surviving replica under FTT1, so a + # failed build must change nothing about it (2026-08-07 incident). cl = _cluster() removed = _node("n1", stack_secondary="stranded") stranded = _node("stranded", secondary_id="n1", lvstore="LVS_stranded") @@ -613,20 +617,70 @@ def test_relocate_via_splice_occupant_rebuild_failure_keeps_forward_pointers(sel return_value="x"), \ patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", return_value=False), \ - patch.object(storage_node_ops, "_delete_replica_on_peer"): + patch.object(storage_node_ops, "_delete_replica_on_peer") as drp: ret = storage_node_ops._relocate_one_replica(removed, "stranded", "secondary") self.assertFalse(ret) - # Forward pointer for occupant's move is committed even though the - # physical rebuild failed -- matches _relocate_one_replica's own - # idempotent-retry pattern: a retry resumes from the committed intent - # rather than re-picking (see test_relocate_resume_reuses_committed_target). - self.assertEqual(occupant.secondary_node_id, "stranded") - # The outer splice claim (stranded -> x) never got committed, since - # _relocate_replica_between reported failure first. + drp.assert_not_called() # old copy on x never torn down + self.assertEqual(occupant.secondary_node_id, "x") # unchanged -- still protected + self.assertEqual(x.lvstore_stack_secondary, "occupant") # unchanged + # The outer splice claim (stranded -> x) never got committed either. self.assertEqual(stranded.secondary_node_id, "n1") self.assertEqual(removed.lvstore_stack_secondary, "stranded") + def test_relocate_via_splice_occupant_rebuild_raises_treated_as_failure(self): + # The 2026-08-07 incident's actual failure mode: recreate_lvstore_on_non_leader + # RAISED (a hublvol attach error) instead of returning False. Must be + # caught and handled identically to a returned False -- old copy on x + # stays untouched either way. + cl = _cluster() + removed = _node("n1", stack_secondary="stranded") + stranded = _node("stranded", secondary_id="n1", lvstore="LVS_stranded") + occupant = _node("occupant", secondary_id="x", lvstore="LVS_occupant") + x = _node("x", stack_secondary="occupant") + db = FakeDB(cl, [removed, stranded, occupant, x]) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + return_value="x"), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + side_effect=Exception("connect_to_hublvol failed for LVS_18")), \ + patch.object(storage_node_ops, "_delete_replica_on_peer") as drp: + ret = storage_node_ops._relocate_one_replica(removed, "stranded", "secondary") + + self.assertFalse(ret) + drp.assert_not_called() + self.assertEqual(occupant.secondary_node_id, "x") + self.assertEqual(x.lvstore_stack_secondary, "occupant") + self.assertEqual(stranded.secondary_node_id, "n1") + + def test_relocate_via_splice_resumes_teardown_after_crash_between_writes(self): + # occupant's move was already built + committed by a PRIOR attempt + # (forward pointer already points at stranded), but the process + # crashed before the old copy on x was torn down -- x's backref is + # still stale. A retry must skip re-building (already done) and go + # straight to finishing the teardown, without erroring. + cl = _cluster() + removed = _node("n1", stack_secondary="stranded") + stranded = _node("stranded", secondary_id="n1", lvstore="LVS_stranded") + occupant = _node("occupant", secondary_id="stranded", lvstore="LVS_occupant") + x = _node("x", stack_secondary="occupant") # stale -- not yet cleared + db = FakeDB(cl, [removed, stranded, occupant, x]) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + return_value="x"), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + return_value=True) as rec, \ + patch.object(storage_node_ops, "_delete_replica_on_peer") as drp: + ret = storage_node_ops._relocate_one_replica(removed, "stranded", "secondary") + + self.assertTrue(ret) + drp.assert_called_once() # the deferred teardown finally runs + self.assertEqual(stranded.secondary_node_id, "x") + self.assertEqual(x.lvstore_stack_secondary, "stranded") + # rebuild still runs once for stranded's own claim on x -- the + # occupant's rebuild was already done, so it must NOT run again. + self.assertEqual(rec.call_count, 1) + def test_relocate_via_splice_own_rebuild_failure_after_occupant_moved(self): # occupant's move succeeds (evicted + rebuilt on stranded), but # stranded's own rebuild on the freed slot x fails. From 4c243f224416c991442e1270569e3ba94a32aaa0 Mon Sep 17 00:00:00 2001 From: wmousa Date: Mon, 10 Aug 2026 23:45:10 +0200 Subject: [PATCH 08/27] fix(node-removal): stop a transient RPC/DNS blip from killing phase 5, and stop masking an incomplete phase 5 on retry Two related bugs found live-testing FD-aware node removal's second (splice-fallback) path on a real cluster: 1. _connect_to_remote_jm_devs' fallback bdev-existence poll called rpc_client.get_bdevs() unguarded, right after the primary connect_device() failure had already been correctly degraded (logged, not raised). A transient DNS/RPC blip against the connecting peer's own SPDK-proxy hostname hit that second call too, but this one propagated -- raising RPCException out of _decommission_node_devices and killing the whole node-removal task. Now wrapped in a bounded retry (3 attempts, 1s apart, tenacity Retrying/RetryError, matching the existing pattern in tasks_runner_lvol_migration.py) so a blip that clears within a few seconds is caught transparently; only once that's exhausted does it degrade to "this JM not connected" (self-heals later via the periodic health-check service's topology-diff sweep) instead of raising. 2. node_removal_orchestrate's top-of-function guard treated `status == REMOVED` as "fully done" and returned True immediately. But phase 4 flips that status *before* phase 5 (device/JM decommission) runs -- so if phase 5 raised (as in bug 1, before the retry existed) after phase 4 had already committed, every resumed attempt hit this guard and reported "Node removed" without phase 5 ever actually completing. Now only phases 1/3a/3b/4 are skipped on resume; phase 5 always (re)runs -- it's already idempotent, so this is a no-op once it has genuinely finished. Observed live: an RPC connection error mid phase-5 JM reassignment left a peer's LVS un-rebuilt on its new host (bdev_lvol_get_lvstores "No such device") while the task still reported done, and the cluster cycled IN_ACTIVATION <-> SUSPENDED. Adds: - TestNodeRemovalOrchestrateResumesPhase5 (3 tests) - TestConnectToRemoteJmDevsDegradesOnRpcException (4 tests, incl. one verifying the bounded retry actually recovers a blip that clears within budget, and one verifying it still degrades gracefully once exhausted) Co-Authored-By: Claude Sonnet 5 --- simplyblock_core/storage_node_ops.py | 124 ++++++++++++------ tests/unit/test_node_removal.py | 185 ++++++++++++++++++++++++++- 2 files changed, 265 insertions(+), 44 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 5649dcb260..d3d329f112 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -2,6 +2,7 @@ import copy import datetime import json +import logging import math import platform import socket @@ -19,6 +20,7 @@ import docker from docker.types import LogConfig from pydantic import SecretStr +from tenacity import RetryError, Retrying, before_sleep_log, retry_if_exception_type, stop_after_attempt, wait_fixed from simplyblock_core import constants, scripts, distr_controller, cluster_ops from simplyblock_core import utils @@ -2144,17 +2146,42 @@ def _connect_to_remote_jm_devs(this_node: StorageNode, jm_ids=None, only_node_id # 5s for one that can never appear. During whole-cluster recovery the # 10x0.5s wait ran per dead peer JM (~30 of them), adding minutes to # every restart attempt (2026-07-13). - for _ in range(1 if connect_failed else 10): - if remote_device.remote_bdev and rpc_client.get_bdevs(remote_device.remote_bdev): - break - if rpc_client.get_bdevs(expected_bdev): - remote_device.remote_bdev = expected_bdev - break - time.sleep(0.5) - if not remote_device.remote_bdev and org_dev.get_id() in existing_remote_jm_devices: - existing_remote_device = existing_remote_jm_devices[org_dev.get_id()] - if existing_remote_device.remote_bdev and rpc_client.get_bdevs(existing_remote_device.remote_bdev): - remote_device.remote_bdev = existing_remote_device.remote_bdev + def _poll_for_remote_jm_bdev(): + for _ in range(1 if connect_failed else 10): + if remote_device.remote_bdev and rpc_client.get_bdevs(remote_device.remote_bdev): + return + if rpc_client.get_bdevs(expected_bdev): + remote_device.remote_bdev = expected_bdev + return + time.sleep(0.5) + if not remote_device.remote_bdev and org_dev.get_id() in existing_remote_jm_devices: + existing_remote_device = existing_remote_jm_devices[org_dev.get_id()] + if existing_remote_device.remote_bdev and rpc_client.get_bdevs(existing_remote_device.remote_bdev): + remote_device.remote_bdev = existing_remote_device.remote_bdev + + try: + # Bounded retry: a transient RPC/DNS blip against this_node's + # own proxy (the same one connect_device just hit above) is + # given a few seconds to clear before giving up. Same + # degrade-not-crash rationale as the connect_device catch + # above -- only RPCException (the transport-level failure) is + # retried; anything else propagates immediately. + Retrying( + stop=stop_after_attempt(3), + wait=wait_fixed(1), + retry=retry_if_exception_type(RPCException), + before_sleep=before_sleep_log(logger, logging.WARNING), + )(_poll_for_remote_jm_bdev) + except RetryError as e: + # Still failing after 3 attempts -- degrade to "this JM not + # connected" instead of aborting the whole node-removal / + # restart operation (2026-08-10 incident: this exact call + # raised uncaught and killed a node-removal task mid phase 5, + # leaving a peer's lvstore un-rebuilt while the task still + # reported "done"). + logger.warning( + f'get_bdevs kept failing while polling for {expected_bdev} ' + f'on {this_node.get_id()} after 3 attempts: {e}') if not remote_device.remote_bdev: logger.error(f"Failed to connect to remote JM device {org_dev.alceml_name}") continue @@ -3716,8 +3743,18 @@ def node_removal_orchestrate(node_id, force_remove=False): logger.error(f"node_removal_orchestrate: node {node_id} not found") return False - if snode.status == StorageNode.STATUS_REMOVED: - return True + # Phase 4 (below) flips status to REMOVED *before* phase 5 (device/JM + # decommission) runs -- so "status == REMOVED" means phases 1/3a/3b/4 + # committed, NOT that removal is fully done. A bare `return True` here + # would let a transient failure inside phase 5 (e.g. an RPC error + # against a peer) get permanently masked: the retry re-enters, hits this + # guard, and reports "done" forever without phase 5 ever completing + # (2026-08-10 incident: a mid-phase-5 RPC error left a peer's lvstore + # un-rebuilt while the task reported "Node removed"). Only phases + # 1/3a/3b/4 are skipped below when already_removed; phase 5 always + # runs and is itself idempotent (skips devices/JM already migrated), so + # resuming it here is a no-op once it has genuinely finished. + already_removed = snode.status == StorageNode.STATUS_REMOVED # Node removal is a recognised restart-phase owner: phase 3b relocates # replicas onto an ONLINE target and sets a restart phase there, which @@ -3730,40 +3767,43 @@ def node_removal_orchestrate(node_id, force_remove=False): prev_cluster_status = cluster.status cluster_ops.set_cluster_status(cluster.get_id(), Cluster.STATUS_IN_SHRINK) try: - # Phase 1 — shut the node down (graceful). Skipped on re-entry. - if snode.status in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED]: - logger.info(f"[REMOVAL] {node_id}: phase 1 — shutdown") - ret = shutdown_storage_node(node_id, force=force_remove) - if isinstance(ret, tuple): - ret, reason = ret - if not ret: - logger.error(f"[REMOVAL] {node_id}: shutdown failed: {reason}") + if not already_removed: + # Phase 1 — shut the node down (graceful). Skipped on re-entry. + if snode.status in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_SUSPENDED]: + logger.info(f"[REMOVAL] {node_id}: phase 1 — shutdown") + ret = shutdown_storage_node(node_id, force=force_remove) + if isinstance(ret, tuple): + ret, reason = ret + if not ret: + logger.error(f"[REMOVAL] {node_id}: shutdown failed: {reason}") + return False + elif not ret: + logger.error(f"[REMOVAL] {node_id}: shutdown failed") return False - elif not ret: - logger.error(f"[REMOVAL] {node_id}: shutdown failed") - return False - snode = db_controller.get_storage_node_by_id(node_id) + snode = db_controller.get_storage_node_by_id(node_id) - # Phase 3a — tear down the (empty) secondary/tertiary replicas of THIS - # node's own primary LVS, on the peers that host them (Case A). - logger.info(f"[REMOVAL] {node_id}: phase 3a — tear down own replicas") - if not _teardown_replicas_of_primary(snode): - return False + # Phase 3a — tear down the (empty) secondary/tertiary replicas of THIS + # node's own primary LVS, on the peers that host them (Case A). + logger.info(f"[REMOVAL] {node_id}: phase 3a — tear down own replicas") + if not _teardown_replicas_of_primary(snode): + return False - # Phase 3b — relocate replicas this node hosts for OTHER primaries (Case B). - logger.info(f"[REMOVAL] {node_id}: phase 3b — relocate hosted replicas") - if not _relocate_replicas_hosted_on(snode): - return False + # Phase 3b — relocate replicas this node hosts for OTHER primaries (Case B). + logger.info(f"[REMOVAL] {node_id}: phase 3b — relocate hosted replicas") + if not _relocate_replicas_hosted_on(snode): + return False - # Phase 5 — finalize (swarm leave, gpt cleanup) and flip to removed. - logger.info(f"[REMOVAL] {node_id}: phase 4 — finalize") - _finalize_node_removal(snode) - set_node_status(node_id, StorageNode.STATUS_REMOVED, caused_by="remove") - snode = db_controller.get_storage_node_by_id(node_id) - # storage_events.snode_status_change( - # snode, StorageNode.STATUS_REMOVED, StorageNode.STATUS_IN_REMOVAL, caused_by="remove") + # Phase 4 — finalize (swarm leave, gpt cleanup) and flip to removed. + logger.info(f"[REMOVAL] {node_id}: phase 4 — finalize") + _finalize_node_removal(snode) + set_node_status(node_id, StorageNode.STATUS_REMOVED, caused_by="remove") + snode = db_controller.get_storage_node_by_id(node_id) + # storage_events.snode_status_change( + # snode, StorageNode.STATUS_REMOVED, StorageNode.STATUS_IN_REMOVAL, caused_by="remove") - # Phase 4 — remove + fail devices, then wait for failure-migration to finish. + # Phase 5 — remove + fail devices, then wait for failure-migration to + # finish. Always attempted, even on resume after status already + # flipped to REMOVED -- see the already_removed comment above. logger.info(f"[REMOVAL] {node_id}: phase 5 — devices remove/fail/migrate") if not _decommission_node_devices(snode): return False diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 4b02b09111..0f202452f0 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -17,13 +17,13 @@ """ import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import DEFAULT, MagicMock, patch from simplyblock_core import storage_node_ops from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice from simplyblock_core.models.cluster import Cluster -from simplyblock_core.rpc_client import RPCConnectionError +from simplyblock_core.rpc_client import RPCConnectionError, RPCException # --------------------------------------------------------------------------- @@ -804,6 +804,187 @@ def test_complete_when_all_migrated(self): dc.device_remove.assert_not_called() +# --------------------------------------------------------------------------- +# node_removal_orchestrate — phase-5 resume gap +# +# Phase 4 flips node status to REMOVED *before* phase 5 (device/JM +# decommission) runs. "status == REMOVED" therefore means phases 1/3a/3b/4 +# committed, NOT that removal is fully done -- a resumed attempt must still +# (re)run phase 5 rather than short-circuiting to "done" (2026-08-10 +# incident: an RPC error mid phase 5 left a peer's lvstore un-rebuilt while +# the task still reported "Node removed"). +# --------------------------------------------------------------------------- + +class TestNodeRemovalOrchestrateResumesPhase5(unittest.TestCase): + + def _patch_all(self): + return patch.multiple( + storage_node_ops, + DBController=DEFAULT, + cluster_ops=DEFAULT, + shutdown_storage_node=DEFAULT, + _teardown_replicas_of_primary=DEFAULT, + _relocate_replicas_hosted_on=DEFAULT, + _finalize_node_removal=DEFAULT, + set_node_status=DEFAULT, + _decommission_node_devices=DEFAULT, + ) + + def test_already_removed_skips_phases_1_to_4_but_reruns_phase5(self): + cl = _cluster() + node = _node("n1", status=StorageNode.STATUS_REMOVED) + db = FakeDB(cl, [node]) + with self._patch_all() as mocks: + mocks["DBController"].return_value = db + mocks["_decommission_node_devices"].return_value = True + ret = storage_node_ops.node_removal_orchestrate("n1") + + self.assertTrue(ret) + mocks["shutdown_storage_node"].assert_not_called() + mocks["_teardown_replicas_of_primary"].assert_not_called() + mocks["_relocate_replicas_hosted_on"].assert_not_called() + mocks["_finalize_node_removal"].assert_not_called() + mocks["set_node_status"].assert_not_called() + mocks["_decommission_node_devices"].assert_called_once_with(node) + + def test_already_removed_reports_incomplete_if_phase5_fails_again(self): + # The regression this guards: a prior attempt raised mid phase 5 + # after the status flip had already committed. The retry must + # actually retry phase 5, not silently report done because status + # already reads REMOVED. + cl = _cluster() + node = _node("n1", status=StorageNode.STATUS_REMOVED) + db = FakeDB(cl, [node]) + with self._patch_all() as mocks: + mocks["DBController"].return_value = db + mocks["_decommission_node_devices"].return_value = False + ret = storage_node_ops.node_removal_orchestrate("n1") + + self.assertFalse(ret) + mocks["_decommission_node_devices"].assert_called_once_with(node) + + def test_fresh_removal_still_runs_all_phases_then_phase5(self): + # Regression guard the other way: a from-scratch removal (status + # still ONLINE) must not skip phases 1/3a/3b/4. + cl = _cluster() + node = _node("n1", status=StorageNode.STATUS_ONLINE) + db = FakeDB(cl, [node]) + with self._patch_all() as mocks: + mocks["DBController"].return_value = db + mocks["shutdown_storage_node"].return_value = True + mocks["_teardown_replicas_of_primary"].return_value = True + mocks["_relocate_replicas_hosted_on"].return_value = True + mocks["_decommission_node_devices"].return_value = True + ret = storage_node_ops.node_removal_orchestrate("n1") + + self.assertTrue(ret) + mocks["shutdown_storage_node"].assert_called_once() + mocks["_teardown_replicas_of_primary"].assert_called_once() + mocks["_relocate_replicas_hosted_on"].assert_called_once() + mocks["_finalize_node_removal"].assert_called_once() + mocks["set_node_status"].assert_called_once_with( + "n1", StorageNode.STATUS_REMOVED, caused_by="remove") + mocks["_decommission_node_devices"].assert_called_once() + + +# --------------------------------------------------------------------------- +# _connect_to_remote_jm_devs — bounded retry + degrade-not-crash on a +# transient RPC/DNS failure during the fallback bdev-existence poll +# +# The primary connect_device() failure already degrades gracefully (logs +# "Failed to connect to ...", sets connect_failed=True). The get_bdevs() +# poll called right after it, against the same rpc_client, hits the +# identical transport and gets a bounded retry (3 attempts, 1s apart) to +# ride out a DNS blip; only once that's exhausted does it degrade to "this +# JM not connected" instead of raising -- 2026-08-10 incident: this exact +# call raised RPCException uncaught and killed a node-removal task mid +# phase 5. +# --------------------------------------------------------------------------- + +class TestConnectToRemoteJmDevsDegradesOnRpcException(unittest.TestCase): + + def _owner_setup(self, this_node_id="this-node"): + jm_dev = JMDevice() + jm_dev.uuid = "jm-owner" + jm_dev.jm_bdev = "jm_owner_bdev" + jm_dev.status = NVMeDevice.STATUS_ONLINE + + owner_node = MagicMock(spec=StorageNode) + owner_node.get_id = MagicMock(return_value="owner-node") + owner_node.status = StorageNode.STATUS_ONLINE + owner_node.jm_device = jm_dev + + this_node = MagicMock(spec=StorageNode) + this_node.get_id = MagicMock(return_value=this_node_id) + this_node.jm_ids = [] + this_node.lvstore_stack_secondary = "" + this_node.lvstore_stack_tertiary = "" + this_node.remote_jm_devices = [] + rpc_client = MagicMock() + this_node.rpc_client = MagicMock(return_value=rpc_client) + + db = MagicMock() + db.get_jm_device_by_id.return_value = jm_dev + db.get_storage_nodes.return_value = [owner_node] + + return this_node, rpc_client, db + + def test_get_bdevs_rpc_exception_exhausts_retries_and_does_not_raise(self): + # Persistent failure (all 3 bounded-retry attempts fail): must + # still degrade, not raise -- this is the exact call chain that + # took down a live node-removal task before the retry was added. + this_node, rpc_client, db = self._owner_setup() + rpc_client.get_bdevs.side_effect = RPCException("connection error") + + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "connect_device", + side_effect=RPCException("connection error")): + result = storage_node_ops._connect_to_remote_jm_devs( + this_node, jm_ids=["jm-owner"]) + + self.assertEqual(result, []) + # 3 bounded-retry attempts, one get_bdevs call each (remote_bdev + # is empty so the first branch short-circuits without calling). + self.assertEqual(rpc_client.get_bdevs.call_count, 3) + + def test_transient_failure_recovers_on_retry(self): + # A blip that clears within the retry budget must be caught, not + # just tolerated -- the whole point of adding the bounded retry + # instead of degrading on the very first failure. + this_node, rpc_client, db = self._owner_setup() + rpc_client.get_bdevs.side_effect = [ + RPCException("connection error"), + RPCException("connection error"), + {"name": "remote_jm_owner_bdevn1"}, + ] + + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "connect_device", + side_effect=RPCException("connection error")), \ + patch.object(storage_node_ops.time, "sleep"): + result = storage_node_ops._connect_to_remote_jm_devs( + this_node, jm_ids=["jm-owner"]) + + self.assertEqual(len(result), 1) + self.assertEqual(result[0].remote_bdev, "remote_jm_owner_bdevn1") + self.assertEqual(rpc_client.get_bdevs.call_count, 3) + + def test_transient_failure_does_not_block_a_clean_connect(self): + # Once the blip has cleared, the same code path must still succeed + # normally -- the new guard must not swallow a real success too. + this_node, rpc_client, db = self._owner_setup() + rpc_client.get_bdevs.return_value = {"name": "remote_jm_owner_bdevn1"} + + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "connect_device", + return_value="remote_jm_owner_bdevn1"): + result = storage_node_ops._connect_to_remote_jm_devs( + this_node, jm_ids=["jm-owner"]) + + self.assertEqual(len(result), 1) + self.assertEqual(result[0].remote_bdev, "remote_jm_owner_bdevn1") + + class TestShrinkStatusDoesNotDeadlockRemoval(unittest.TestCase): """``node_removal_orchestrate`` holds ``Cluster.STATUS_IN_SHRINK`` for the duration of an attempt, so that the restart phases its replica relocation From d6bd660be8a011d6e43a3b48ba04fe2149f66183 Mon Sep 17 00:00:00 2001 From: wmousa Date: Tue, 11 Aug 2026 16:55:38 +0200 Subject: [PATCH 09/27] fix(node-removal): skip already-removed peers with stale jm_ids in phase 5 get_storage_nodes_by_cluster_id returns every node regardless of status, including ones already REMOVED. The JM-device peer-reassignment loop in _decommission_node_devices never filtered on that: an earlier-removed node can still carry the currently-removed node's JM id in its own jm_ids (never cleared on ITS OWN removal), so a later removal's phase 5 would try to "fix" that dead node's JM connections using its own rpc_client -- which points at a pod that no longer exists and can never resolve, let alone connect. 2026-08-11 incident: removing node A, then later removing node B (whose JM device A used to reference) sent phase 5 chasing A's permanently-dead hostname (NameResolutionError -> uncaught RPCException, same class of failure as the bounded-retry fix targets, but that retry can't save a hostname that will never resolve). B's own devices never reached failed/failed_and_migrated because the crash happened before the device loop further down in the same function, and the task still reported "done" on the next attempt (status already REMOVED short-circuits phase 1-4, per the earlier phase-5-resume fix) -- leaving B's devices stuck at unavailable with no task left to retry them. Skip node.status == STATUS_REMOVED outright in that loop -- a removed node's own bookkeeping is dead weight, not something to reconnect. --- simplyblock_core/storage_node_ops.py | 11 +++++++++++ tests/unit/test_node_removal.py | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index d3d329f112..6e73193347 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4054,6 +4054,17 @@ def _decommission_node_devices(removed_node: StorageNode): device_controller.remove_jm_device(removed_node.jm_device.get_id(), force=True) # look for other nodes who use this JM and replace it for node in db_controller.get_storage_nodes_by_cluster_id(removed_node.cluster_id): + # get_storage_nodes_by_cluster_id returns every node regardless of + # status, including ones already REMOVED. An already-removed node + # can still carry the just-removed node's JM id in its own stale + # jm_ids (never cleared on ITS removal) -- without this guard we'd + # try to "fix" that dead node's JM connections using its own + # rpc_client, which points at a pod that no longer exists and can + # never resolve/connect (2026-08-11 incident: a prior removal's + # leftover jm_ids on 7b8hf sent a later removal's phase 5 chasing + # a permanently-dead hostname). + if node.status == StorageNode.STATUS_REMOVED: + continue if node.jm_ids and removed_node.jm_device.get_id() in node.jm_ids: node.jm_ids.remove(removed_node.jm_device.get_id()) jm_ids = get_sorted_ha_jms(node) diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 0f202452f0..7105154ec9 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -803,6 +803,32 @@ def test_complete_when_all_migrated(self): self.assertTrue(ret) dc.device_remove.assert_not_called() + def test_skips_already_removed_peer_with_stale_jm_ids(self): + # 2026-08-11 incident: an earlier-removed node can still carry the + # currently-removed node's JM id in its own stale jm_ids (never + # cleared on ITS OWN removal) -- get_storage_nodes_by_cluster_id + # returns every node regardless of status, including removed ones. + # Must be skipped outright, not "fixed" via its own (permanently + # dead) rpc_client. + cl = _cluster() + removed = _node("n1", n_devices=0, with_jm=True) + removed.jm_ids = [] + stale_peer = _node("stale-peer", status=StorageNode.STATUS_REMOVED) + stale_peer.jm_ids = [removed.jm_device.get_id()] + db = FakeDB(cl, [removed, stale_peer]) + dc = MagicMock() + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "device_controller", dc), \ + patch.object(storage_node_ops, "_connect_to_remote_jm_devs") as connect_mock: + ret = storage_node_ops._decommission_node_devices(removed) + + self.assertTrue(ret) + dc.remove_jm_device.assert_called_once() + connect_mock.assert_not_called() + # The stale peer's own bookkeeping is left alone -- it's dead, not "fixed". + self.assertEqual(stale_peer.jm_ids, [removed.jm_device.get_id()]) + stale_peer.write_to_db.assert_not_called() + # --------------------------------------------------------------------------- # node_removal_orchestrate — phase-5 resume gap From 129b12e5b438be3d842f3df07aab143cf57d51ab Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 12 Aug 2026 22:07:02 +0300 Subject: [PATCH 10/27] fix(storage_node): disconnect hublvol from secondary and tertiary nodes during teardown --- simplyblock_core/storage_node_ops.py | 42 +++++++++++++++++++++------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 6e73193347..bbd222cb6f 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -3861,16 +3861,16 @@ def _delete_replica_on_peer(peer, primary, cluster): lvstore = primary.lvstore if not lvstore: return - try: - nqn = peer.hublvol_nqn_for_lvstore(cluster.nqn, lvstore) - if rpc_client.subsystem_get(nqn): - rpc_client.subsystem_delete(nqn) - except RPCException as e: - logger.warning(f"hublvol subsystem teardown for {lvstore} on {peer.get_id()} failed: {e}") - try: - rpc_client.bdev_lvol_delete_hublvol(lvstore) - except RPCException as e: - logger.warning(f"hublvol bdev teardown for {lvstore} on {peer.get_id()} failed: {e}") + # try: + # nqn = peer.hublvol_nqn_for_lvstore(cluster.nqn, lvstore) + # if rpc_client.subsystem_get(nqn): + # rpc_client.subsystem_delete(nqn) + # except RPCException as e: + # logger.warning(f"hublvol subsystem teardown for {lvstore} on {peer.get_id()} failed: {e}") + # try: + # rpc_client.bdev_lvol_delete_hublvol(lvstore) + # except RPCException as e: + # logger.warning(f"hublvol bdev teardown for {lvstore} on {peer.get_id()} failed: {e}") try: # deepcopy: _remove_bdev_stack stamps bdev['status']; don't mutate the # primary's stored stack definition. @@ -6050,6 +6050,28 @@ def shutdown_storage_node(node_id, force=False, keep_auto_restart=False, "Loop 2: peer-side detach pass raised %s (continuing to kill)", e) + if snode.hublvol: + # Disconnect hublvol from secondary + if snode.secondary_node_id: + sec_node = db_controller.get_storage_node_by_id(snode.secondary_node_id) + if sec_node.status == StorageNode.STATUS_ONLINE: + logger.info("Disconnecting hublvol from %s", sec_node.get_id()) + try: + sec_node.rpc_client().bdev_nvme_detach_controller(snode.hublvol.bdev_name) + except Exception as e: + logger.warning("Disconnecting hublvol failed: %s", e) + + # Disconnect hublvol from tertiary + if snode.tertiary_node_id: + ter_node = db_controller.get_storage_node_by_id(snode.tertiary_node_id) + if ter_node.status == StorageNode.STATUS_ONLINE: + logger.info("Disconnecting hublvol from %s", ter_node.get_id()) + try: + ter_node.rpc_client().bdev_nvme_detach_controller(snode.hublvol.bdev_name) + except Exception as e: + logger.warning("Disconnecting hublvol failed: %s", e) + + # Step 5: hard-kill SPDK. Same code path as the existing --force # shutdown — peers see the TCP drop and host multipath retries on # surviving paths. Any IO inside SPDK at this instant is lost; From 174430ed8b4a8f79f663d7d7a25ea984f33c3140 Mon Sep 17 00:00:00 2001 From: wmousa Date: Wed, 12 Aug 2026 22:53:37 +0200 Subject: [PATCH 11/27] fix(node-removal): cascade-vacate a splice target's pre-existing occupant _relocate_replica_between claimed new_host's lvstore_stack_secondary/ _tertiary slot for the evicted occupant but never wrote it -- and even if it had, that slot was already occupied: every node in a full ring already hosts exactly one other node's replica before any removal starts, so a splice claiming it for a new occupant would silently drop the pre-existing one (single-value field, can't hold both). Confirmed live (2026-08-12): after a splice, sn list showed two different primaries both pointing their secondary_node_id at the same host. The physical build was fine -- SPDK doesn't mind a node hosting a second lvstore -- but the pre-existing relationship's own back-reference was never touched, leaving that replica untracked for any future failover, and missing from lvstore_ports. Fix, in _relocate_replica_between: * Actually write new_host's backref + lvstore_ports entry for the newly-placed replica (previously never written at all). * Before claiming the slot, check whether it's already occupied by an unrelated primary. If so, relocate that occupant first -- recursively, via this same function -- onto a fresh target. This is a rotation, not a retry: _seen guards against a cycle, but the rotation is otherwise always finite since each hop heads toward the one slot the original removal freed. Fails closed (refuses the whole splice) if the displaced occupant has nowhere to go, rather than overloading the slot. Also: _delete_replica_on_peer's callers now prune the torn-down replica's stale lvstore_ports entry (_prune_stale_lvstore_ports) -- previously left in place indefinitely after a node removal, even though (unlike the restart-reconnect path that intentionally keeps it for port reuse) it was never coming back. Added regression tests for both the secondary and tertiary (FTT2) cases: the cascade correctly vacating a pre-existing occupant, and failing closed when that occupant has no relocation target. --- simplyblock_core/storage_node_ops.py | 95 ++++++++++++++++++- tests/unit/test_node_removal.py | 137 +++++++++++++++++++++++++++ 2 files changed, 230 insertions(+), 2 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index bbd222cb6f..08bc611397 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -3839,6 +3839,7 @@ def _teardown_replicas_of_primary(removed_node: StorageNode): if peer.status == StorageNode.STATUS_ONLINE: _delete_replica_on_peer(peer, removed_node, cluster) + _prune_stale_lvstore_ports(peer_id, removed_node.lvstore, db_controller) # Clear the peer's back-reference if it still points at us. peer = db_controller.get_storage_node_by_id(peer_id) @@ -3879,6 +3880,25 @@ def _delete_replica_on_peer(peer, primary, cluster): logger.warning(f"replica bdev-stack teardown for {lvstore} on {peer.get_id()} failed: {e}") +def _prune_stale_lvstore_ports(node_id, lvstore, db_controller): + """Drop ``lvstore``'s port reservation from ``node_id``'s + ``lvstore_ports`` after its replica has been torn down there for good. + + Unlike ``teardown_non_leader_lvstore``'s donor-reconnect path (which + deliberately keeps the entry so a returning node reuses its old ports), + callers of this helper -- node removal, splice eviction -- know the + replica isn't coming back to this node. A stale entry there only + misrepresents `sn list`'s "LVS Ports" column (2026-08-12: found live via + bdev_lvol_get_lvstores disagreeing with the DB after a node removal). + Re-fetches fresh to avoid clobbering unrelated concurrent edits.""" + if not lvstore: + return + node = db_controller.get_storage_node_by_id(node_id) + if node.lvstore_ports and lvstore in node.lvstore_ports: + del node.lvstore_ports[lvstore] + node.write_to_db() + + def _relocate_replicas_hosted_on(removed_node: StorageNode): """Case B: ``removed_node`` holds a secondary and/or tertiary replica for other primaries. Re-host each on a fresh, anti-affinity-valid node so the @@ -3963,9 +3983,10 @@ def _relocate_one_replica(removed_node: StorageNode, primary_id, role): return True -def _relocate_replica_between(occupant_primary_id, old_host_id, new_host_id, role, db_controller): +def _relocate_replica_between(occupant_primary_id, old_host_id, new_host_id, role, db_controller, _seen=None): """Physically move ``occupant_primary_id``'s ``role`` replica off - ``old_host_id`` onto ``new_host_id``, updating its forward pointer. + ``old_host_id`` onto ``new_host_id``, updating its forward pointer AND + ``new_host_id``'s back-reference. Used by the splice fallback in ``_relocate_one_replica``: before an already-busy node can be claimed for the primary being relocated, its @@ -3973,6 +3994,24 @@ def _relocate_replica_between(occupant_primary_id, old_host_id, new_host_id, rol ``_find_splice_target_for_relocation``'s docstring for why an already-formed pairing, not an idle node, is what's available). + ``new_host_id`` itself may ALREADY be hosting a different primary's + replica via this same single-value ``lvstore_stack_secondary`` / + ``lvstore_stack_tertiary`` slot: every node in a full ring hosts exactly + one other node's replica before any removal starts, and that + relationship has nothing to do with whichever edge the splice search + happened to pick. (2026-08-12 incident: a splice claimed a node whose + slot already held an unrelated pre-existing occupant. The physical + build succeeded -- SPDK doesn't mind hosting a second lvstore -- but the + slot could not record it, silently untracking that replica for any + future failover, and leaving `sn list`'s "LVS Ports" column short by + one entry.) When the slot is occupied, the existing occupant is + relocated first -- recursively, via this same function -- onto a fresh + target, before ``occupant_primary``'s replica claims the freed slot. + This is a rotation, not a retry loop: ``_seen`` accumulates visited + ``new_host_id``s across the recursion purely as a cycle backstop against + a topology bug; the rotation itself is always finite, since each hop + heads toward the one slot the original removal freed. + Create-before-destroy: the new replica is built on ``new_host_id`` BEFORE the old one on ``old_host_id`` is torn down, so ``occupant_primary`` never has zero surviving copies -- critical on @@ -3998,6 +4037,7 @@ def _relocate_replica_between(occupant_primary_id, old_host_id, new_host_id, rol """ field = "secondary_node_id" if role == "secondary" else "tertiary_node_id" backref = "lvstore_stack_secondary" if role == "secondary" else "lvstore_stack_tertiary" + seen = _seen if _seen is not None else set() try: occupant_primary = db_controller.get_storage_node_by_id(occupant_primary_id) except KeyError: @@ -4005,6 +4045,41 @@ def _relocate_replica_between(occupant_primary_id, old_host_id, new_host_id, rol if getattr(occupant_primary, field) != new_host_id: new_host = db_controller.get_storage_node_by_id(new_host_id) + + existing_occupant_id = getattr(new_host, backref) + if existing_occupant_id and existing_occupant_id != occupant_primary_id: + if new_host_id in seen: + logger.error( + f"[REMOVAL] splice: cycle detected while vacating " + f"{new_host_id}'s existing {role} occupant " + f"{existing_occupant_id} to make room for " + f"{occupant_primary_id}; refusing") + return False + seen.add(new_host_id) + try: + existing_occupant = db_controller.get_storage_node_by_id(existing_occupant_id) + except KeyError: + existing_occupant = None + vacate_target = ( + _pick_replica_relocation_node(existing_occupant, new_host, role, db_controller) + if existing_occupant else None) + if not vacate_target or vacate_target == occupant_primary_id: + logger.error( + f"[REMOVAL] splice: no relocation target to vacate " + f"{new_host_id}'s existing {role} occupant " + f"{existing_occupant_id}; cannot free the slot for " + f"{occupant_primary_id}") + return False + if not _relocate_replica_between( + existing_occupant_id, new_host_id, vacate_target, role, + db_controller, _seen=seen): + logger.error( + f"[REMOVAL] splice: failed to vacate {new_host_id}'s " + f"existing {role} occupant {existing_occupant_id} onto " + f"{vacate_target}") + return False + new_host = db_controller.get_storage_node_by_id(new_host_id) + try: built = recreate_lvstore_on_non_leader(new_host, occupant_primary, occupant_primary) except Exception as e: @@ -4019,11 +4094,27 @@ def _relocate_replica_between(occupant_primary_id, old_host_id, new_host_id, rol setattr(occupant_primary, field, new_host_id) occupant_primary.write_to_db() + # Record the new host's side of the relationship too. Re-fetch: the + # build above (recreate_lvstore_on_non_leader) persists its own + # lvstore_ports snapshot for new_host, derived only from its + # (now-vacated) lvstore_stack_secondary/_tertiary -- it has no way to + # know about occupant_primary, so this entry has to be added here. + new_host = db_controller.get_storage_node_by_id(new_host_id) + setattr(new_host, backref, occupant_primary_id) + if not new_host.lvstore_ports: + new_host.lvstore_ports = {} + new_host.lvstore_ports[occupant_primary.lvstore] = { + "lvol_subsys_port": occupant_primary.lvol_subsys_port, + "hublvol_port": occupant_primary.hublvol.nvmf_port if occupant_primary.hublvol else 0, + } + new_host.write_to_db() + old_host = db_controller.get_storage_node_by_id(old_host_id) if getattr(old_host, backref) == occupant_primary_id: cluster = db_controller.get_cluster_by_id(occupant_primary.cluster_id) if old_host.status == StorageNode.STATUS_ONLINE: _delete_replica_on_peer(old_host, occupant_primary, cluster) + _prune_stale_lvstore_ports(old_host_id, occupant_primary.lvstore, db_controller) old_host = db_controller.get_storage_node_by_id(old_host_id) setattr(old_host, backref, "") old_host.write_to_db() diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 7105154ec9..4d435471c3 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -707,6 +707,97 @@ def test_relocate_via_splice_own_rebuild_failure_after_occupant_moved(self): self.assertEqual(x.lvstore_stack_secondary, "stranded") self.assertEqual(removed.lvstore_stack_secondary, "stranded") + def test_relocate_via_splice_vacates_strandeds_preexisting_occupant_first(self): + # 2026-08-12 live incident: `stranded` already hosts an unrelated + # occupant `z` via its own back-reference BEFORE this removal starts + # -- every node in a full ring already hosts someone. Splicing + # `occupant` onto `stranded` without first moving `z` elsewhere + # would silently drop z's back-reference (a single-value field + # can't hold both z and occupant): live symptom was `sn list` + # showing two different primaries both claiming the same secondary + # node, and a physically-live replica invisible to lvstore_ports. + # `z` must be relocated first -- onto a genuinely free node -- before + # `occupant` claims the freed slot. + cl = _cluster() + removed = _node("n1", stack_secondary="stranded") + stranded = _node("stranded", secondary_id="n1", lvstore="LVS_stranded", + stack_secondary="z") # already hosting z + occupant = _node("occupant", secondary_id="x", lvstore="LVS_occupant") + x = _node("x", stack_secondary="occupant") + z = _node("z", secondary_id="stranded", lvstore="LVS_z") + free_node = _node("free", lvstore="LVS_free") # genuinely unclaimed + db = FakeDB(cl, [removed, stranded, occupant, x, z, free_node]) + + def pick_side_effect(primary, exclude_node, role, db_controller): + if primary.get_id() == "stranded": + self.assertEqual(exclude_node.get_id(), "n1") + return "x" + if primary.get_id() == "z": + self.assertEqual(exclude_node.get_id(), "stranded") + return "free" + raise AssertionError(f"unexpected pick for {primary.get_id()}") + + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + side_effect=pick_side_effect), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + return_value=True) as rec, \ + patch.object(storage_node_ops, "_delete_replica_on_peer") as drp: + ret = storage_node_ops._relocate_one_replica(removed, "stranded", "secondary") + + self.assertTrue(ret) + # z relocated off stranded onto the genuinely free node. + self.assertEqual(z.secondary_node_id, "free") + self.assertEqual(free_node.lvstore_stack_secondary, "z") + # stranded's slot no longer holds the stale "z" -- it now correctly + # reflects occupant, the relationship this splice actually created. + self.assertEqual(stranded.lvstore_stack_secondary, "occupant") + # occupant evicted off x onto the now-vacated stranded. + self.assertEqual(occupant.secondary_node_id, "stranded") + # stranded's own claim on x (the originally-picked splice target). + self.assertEqual(stranded.secondary_node_id, "x") + self.assertEqual(x.lvstore_stack_secondary, "stranded") + self.assertEqual(removed.lvstore_stack_secondary, "") + self.assertEqual(rec.call_count, 3) # z's + occupant's + stranded's own rebuild + self.assertEqual(drp.call_count, 2) # old z copy off stranded, old occupant copy off x + + def test_relocate_via_splice_refuses_when_preexisting_occupant_has_no_target(self): + # Same setup, but z has nowhere to go. Must fail closed -- refuse the + # whole splice rather than overload stranded's single-value slot. + cl = _cluster() + removed = _node("n1", stack_secondary="stranded") + stranded = _node("stranded", secondary_id="n1", lvstore="LVS_stranded", + stack_secondary="z") + occupant = _node("occupant", secondary_id="x", lvstore="LVS_occupant") + x = _node("x", stack_secondary="occupant") + z = _node("z", secondary_id="stranded", lvstore="LVS_z") + db = FakeDB(cl, [removed, stranded, occupant, x, z]) + + def pick_side_effect(primary, exclude_node, role, db_controller): + if primary.get_id() == "stranded": + return "x" + if primary.get_id() == "z": + return None # nothing free anywhere for z + raise AssertionError(f"unexpected pick for {primary.get_id()}") + + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + side_effect=pick_side_effect), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + return_value=True) as rec, \ + patch.object(storage_node_ops, "_delete_replica_on_peer") as drp: + ret = storage_node_ops._relocate_one_replica(removed, "stranded", "secondary") + + self.assertFalse(ret) + drp.assert_not_called() + rec.assert_not_called() + # Nothing committed -- z, occupant, x, stranded, removed all untouched. + self.assertEqual(z.secondary_node_id, "stranded") + self.assertEqual(stranded.lvstore_stack_secondary, "z") + self.assertEqual(occupant.secondary_node_id, "x") + self.assertEqual(stranded.secondary_node_id, "n1") + self.assertEqual(removed.lvstore_stack_secondary, "stranded") + def test_relocate_via_splice_tertiary_role(self): cl = _cluster() removed = _node("n1", stack_tertiary="stranded") @@ -729,6 +820,52 @@ def test_relocate_via_splice_tertiary_role(self): self.assertEqual(x.lvstore_stack_tertiary, "stranded") self.assertEqual(removed.lvstore_stack_tertiary, "") + def test_relocate_via_splice_tertiary_vacates_strandeds_preexisting_occupant_first(self): + # FTT2 (dual fault tolerance) variant of + # test_relocate_via_splice_vacates_strandeds_preexisting_occupant_first: + # secondary and tertiary live in separate fields on every node, so a + # node hosting one primary's secondary AND a different primary's + # tertiary at once is fine -- that was never the collision. The + # collision is within a single field, and the cascade fix is + # parameterized by role throughout, so this exercises the same path + # for lvstore_stack_tertiary specifically. + cl = _cluster() + removed = _node("n1", stack_tertiary="stranded") + stranded = _node("stranded", tertiary_id="n1", lvstore="LVS_stranded", + stack_tertiary="z") # already hosting z's tertiary + occupant = _node("occupant", tertiary_id="x", lvstore="LVS_occupant") + x = _node("x", stack_tertiary="occupant") + z = _node("z", tertiary_id="stranded", lvstore="LVS_z") + free_node = _node("free", lvstore="LVS_free") + db = FakeDB(cl, [removed, stranded, occupant, x, z, free_node]) + + def pick_side_effect(primary, exclude_node, role, db_controller): + self.assertEqual(role, "tertiary") + if primary.get_id() == "stranded": + return "x" + if primary.get_id() == "z": + return "free" + raise AssertionError(f"unexpected pick for {primary.get_id()}") + + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + side_effect=pick_side_effect), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + return_value=True) as rec, \ + patch.object(storage_node_ops, "_delete_replica_on_peer") as drp: + ret = storage_node_ops._relocate_one_replica(removed, "stranded", "tertiary") + + self.assertTrue(ret) + self.assertEqual(z.tertiary_node_id, "free") + self.assertEqual(free_node.lvstore_stack_tertiary, "z") + self.assertEqual(stranded.lvstore_stack_tertiary, "occupant") + self.assertEqual(occupant.tertiary_node_id, "stranded") + self.assertEqual(stranded.tertiary_node_id, "x") + self.assertEqual(x.lvstore_stack_tertiary, "stranded") + self.assertEqual(removed.lvstore_stack_tertiary, "") + self.assertEqual(rec.call_count, 3) + self.assertEqual(drp.call_count, 2) + def test_relocate_free_target_never_triggers_splice_eviction(self): # Regression guard: when the picked target is genuinely free (no # backref set), _relocate_one_replica must behave exactly as before From acf3ca118ad133d7abec081fd836e9b15601ac24 Mon Sep 17 00:00:00 2001 From: wmousa Date: Thu, 13 Aug 2026 13:10:13 +0200 Subject: [PATCH 12/27] fix(node-removal): clear the removed node's own stale lvstore_ports Case A/B relocation (_teardown_replicas_of_primary, _relocate_replicas_hosted_on) clears every forward/back-reference field on the removed node as each relationship moves elsewhere, but neither touches lvstore_ports -- it isn't part of any relocation, just a port-reuse cache for the node's own restarts (recreate_lvstore_on_non_leader). Confirmed live (2026-08-13): after removing a node, sn list kept showing its old LVS Ports entries indefinitely even though its SPDK process was gone and its status read removed -- there's no restart to reuse those ports for, since removal is terminal. _finalize_node_removal now clears it right before the node flips to REMOVED, alongside its other best-effort cleanup. --- simplyblock_core/storage_node_ops.py | 14 ++++++++++ tests/unit/test_node_removal.py | 39 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 08bc611397..0cfe9e6512 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4210,6 +4210,20 @@ def _finalize_node_removal(removed_node: StorageNode): removed_node = db_controller.get_storage_node_by_id(removed_node.get_id()) cluster = db_controller.get_cluster_by_id(removed_node.cluster_id) + # Case A/B (_teardown_replicas_of_primary, _relocate_replicas_hosted_on) + # already cleared this node's forward/back-reference fields as they + # relocated each side of the relationship elsewhere. lvstore_ports is + # the one piece of bookkeeping neither touches -- it isn't part of any + # relocation, just a port-reuse cache for THIS node's own restarts (see + # recreate_lvstore_on_non_leader) -- and by the time this function runs + # there won't be one: the node is about to flip to REMOVED for good. + # Left uncleared, `sn list`'s "LVS Ports" column keeps showing entries + # for a node with no SPDK process left to back them (2026-08-13, found + # live after a removal). + if removed_node.lvstore_ports: + removed_node.lvstore_ports = {} + removed_node.write_to_db() + if cluster.mode == "docker": logger.info("Leaving swarm...") try: diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 4d435471c3..c422856469 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -1050,6 +1050,45 @@ def test_fresh_removal_still_runs_all_phases_then_phase5(self): mocks["_decommission_node_devices"].assert_called_once() +# --------------------------------------------------------------------------- +# _finalize_node_removal — clearing the removed node's OWN stale bookkeeping +# +# Case A/B relocation clears every forward/back-reference field as each +# relationship is moved elsewhere, but neither touches lvstore_ports -- it +# isn't part of any relocation, just a port-reuse cache for this node's own +# restarts. Left uncleared, `sn list`'s "LVS Ports" column keeps showing +# entries for a node with no SPDK process left to back them (2026-08-13, +# found live after a removal). +# --------------------------------------------------------------------------- + +class TestFinalizeNodeRemovalClearsLvstorePorts(unittest.TestCase): + + def _run(self, removed, cluster_mode="kubernetes", node_api_up=False): + cl = _cluster(mode=cluster_mode) + db = FakeDB(cl, [removed]) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops.health_controller, "_check_node_api", + return_value=node_api_up): + storage_node_ops._finalize_node_removal(removed) + return removed + + def test_clears_stale_lvstore_ports(self): + removed = _node("n1", lvstore="LVS_1") + removed.lvstore_ports = {"LVS_1": {"lvol_subsys_port": 4440, "hublvol_port": 4441}, + "LVS_9": {"lvol_subsys_port": 4450, "hublvol_port": 4451}} + removed = self._run(removed) + self.assertEqual(removed.lvstore_ports, {}) + removed.write_to_db.assert_called() + + def test_no_op_when_already_empty(self): + # Don't write to the DB at all when there's nothing to clear. + removed = _node("n1", lvstore="LVS_1") + removed.lvstore_ports = {} + removed = self._run(removed) + self.assertEqual(removed.lvstore_ports, {}) + removed.write_to_db.assert_not_called() + + # --------------------------------------------------------------------------- # _connect_to_remote_jm_devs — bounded retry + degrade-not-crash on a # transient RPC/DNS failure during the fallback bdev-existence poll From 43d6ebcafd3f946ef5d3574e8187c8108453d50c Mon Sep 17 00:00:00 2001 From: wmousa Date: Thu, 13 Aug 2026 14:15:37 +0200 Subject: [PATCH 13/27] fix(web): return 400, not 500, when node remove/delete is refused The DELETE /storage-nodes/{id} endpoint raised a bare ValueError when remove_storage_node()/delete_storage_node() returned False. With no registered handler for ValueError, FastAPI turns that into an unhandled-exception 500. Confirmed live (2026-08-13): the operator's drain reconciler suspends a node, then calls this same DELETE to remove it. When sbcli refused the removal (failure-domain balance would drop below the allowed spread), the 500 response landed on the operator's *retryable* status list (webapi/errorclass.go) instead of the non-retryable one -- so it just requeued and retried the identical, permanently-doomed DELETE forever, rather than reaching resumeAndFail and un-suspending the node. The node sat suspended indefinitely with no self-healing path. 400 is correctly classified as non-retryable on the operator side, so this alone fixes the stuck-suspended symptom without any operator change. (remove_storage_node's precondition gates still only signal via return False rather than a specific exception carrying the actual reason string they already log -- that's a bigger refactor, not done here.) --- .../api/v2/cluster/storage_node/__init__.py | 14 ++++++++-- .../web/api/v2/test_storage_node_endpoints.py | 26 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/simplyblock_web/api/v2/cluster/storage_node/__init__.py b/simplyblock_web/api/v2/cluster/storage_node/__init__.py index 1f68785187..f25ca6ab10 100644 --- a/simplyblock_web/api/v2/cluster/storage_node/__init__.py +++ b/simplyblock_web/api/v2/cluster/storage_node/__init__.py @@ -115,18 +115,28 @@ def get(cluster: Cluster, storage_node: StorageNode): @instance_api.delete('/', name='clusters:storage-nodes:delete') def delete( cluster: Cluster, storage_node: StorageNode, force_remove: bool = False, force_migrate: bool = False, force_delete: bool = False) -> Response: + # remove_storage_node's precondition gates (FTT, failure-domain balance, + # replica-relocation feasibility, ...) reject via `return False` rather + # than a specific exception (see the reason string it logs via + # logger.error) -- tracked as a bigger refactor, not done here. But an + # unhandled ValueError with no registered handler becomes an HTTP 500, + # and 500 is on the operator's *retryable* list (webapi/errorclass.go) + # -- so a permanently-infeasible removal (e.g. would unbalance failure + # domains) was retried forever instead of the operator resuming the + # node it had already suspended and failing cleanly (2026-08-13 + # incident). 400 is correctly classified as non-retryable there. none_or_false = storage_node_ops.remove_storage_node( storage_node.get_id(), force_remove=force_remove, force_migrate=force_migrate ) if none_or_false == False: # noqa - raise ValueError('Failed to remove storage node') + raise HTTPException(400, 'Failed to remove storage node') if force_delete: none_or_false = storage_node_ops.delete_storage_node( storage_node.get_id(), force=force_delete ) if none_or_false == False: # noqa - raise ValueError('Failed to delete storage node') + raise HTTPException(400, 'Failed to delete storage node') return Response(status_code=204) diff --git a/tests/unit/web/api/v2/test_storage_node_endpoints.py b/tests/unit/web/api/v2/test_storage_node_endpoints.py index 859ca945f3..4e0c8c4ce9 100644 --- a/tests/unit/web/api/v2/test_storage_node_endpoints.py +++ b/tests/unit/web/api/v2/test_storage_node_endpoints.py @@ -112,6 +112,32 @@ def test_force_delete_also_deletes_node(self, client, storage_node, storage_node storage_node_ops.delete_storage_node.assert_called_once_with( STORAGE_NODE_ID, force=True) + def test_refused_removal_returns_400_not_500(self, client, storage_node, storage_node_ops): + # remove_storage_node's precondition gates (FTT, failure-domain + # balance, replica-relocation feasibility, ...) signal refusal via + # `return False`. An unhandled exception with no registered FastAPI + # handler becomes a 500 -- and 500 is on the operator's *retryable* + # list, so a permanently-infeasible removal (e.g. would leave a + # failure domain unbalanced) got retried forever instead of the + # operator resuming the node it had already suspended and failing + # cleanly (2026-08-13 incident). Must be a 400: non-retryable there. + storage_node_ops.remove_storage_node.return_value = False + + response = client.delete(f'{BASE}/{STORAGE_NODE_ID}/') + + assert response.status_code == 400 + storage_node_ops.delete_storage_node.assert_not_called() + + def test_refused_delete_after_successful_remove_returns_400( + self, client, storage_node, storage_node_ops): + storage_node_ops.remove_storage_node.return_value = 'task-uuid-1' + storage_node_ops.delete_storage_node.return_value = False + + response = client.delete( + f'{BASE}/{STORAGE_NODE_ID}/', params={'force_delete': True}) + + assert response.status_code == 400 + class TestStorageNodeLifecycle: From 465d68372f717560607a3996dcd4434ad1193ce2 Mon Sep 17 00:00:00 2001 From: wmousa Date: Fri, 14 Aug 2026 14:32:40 +0200 Subject: [PATCH 14/27] fix(node-removal): also refresh peers that inherit a dead JM via hosting _decommission_node_devices' phase-5 JM cleanup only checked a peer's OWN jm_ids (its redundancy set for its own JM) to decide whether to fix it. But _connect_to_remote_jm_devs populates remote_jm_devices from a SECOND source too: whichever primary a node hosts as secondary/tertiary pulls in that primary's jm_ids as well, so the hublvol journal stays consistent with the primary being replicated. A peer reachable only through that second path never touches its own jm_ids at all, so the jm_ids-only guard never even looked at it -- its remote_jm_devices entry for the dead JM was left stale forever. Confirmed live (2026-08-14): after two node removals -- the first a plain relocation, the second triggering the splice-cascade fallback -- two peers had the second removed node's JM lingering in remote_jm_devices while their own jm_ids were already clean. The first removal never reshuffled who-hosts-whom, so it never exposed the gap; the splice's reshuffling did. Adds an elif branch: when a peer's remote_jm_devices contains the dead JM but its own jm_ids doesn't, refresh via _connect_to_remote_jm_devs (no replacement pick needed, unlike the jm_ids branch -- not a fixed-size redundancy slot, just a stale connection that a plain refresh naturally drops). --- simplyblock_core/storage_node_ops.py | 21 ++++++++++ tests/unit/test_node_removal.py | 57 +++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 0cfe9e6512..3129a1a3a3 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4175,6 +4175,27 @@ def _decommission_node_devices(removed_node: StorageNode): node.write_to_db() else: logger.error(f"no jm_id found for {node.get_id()}") + elif any(d.uuid == removed_node.jm_device.get_id() for d in (node.remote_jm_devices or [])): + # node.jm_ids is this node's OWN redundancy set for its OWN JM + # -- but _connect_to_remote_jm_devs also connects to a SECOND + # source: whichever primary this node hosts as secondary/ + # tertiary pulls in THAT primary's jm_ids too (so the hublvol + # journal stays consistent with the primary it's replicating). + # A node reachable only via that second path never touches + # node.jm_ids at all, so the branch above never even looks at + # it, and its remote_jm_devices entry for the dead JM is left + # stale forever (2026-08-14 incident: exposed by a splice + # reshuffling who hosts whom -- a plain removal that never + # changes any hosting relationship never surfaces this gap, + # which is why the FIRST of two removals in the same test + # showed no symptom and the second, spliced one did). No + # "replacement" pick needed here, unlike the jm_ids branch -- + # this isn't a fixed-size redundancy slot, just a stale + # connection to drop; a plain refresh naturally excludes the + # now-removed JM since it can no longer be reached through + # either source. + node.remote_jm_devices = _connect_to_remote_jm_devs(node, node.jm_ids) + node.write_to_db() removed_node = db_controller.get_storage_node_by_id(removed_node.get_id()) for dev in removed_node.nvme_devices: diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index c422856469..30a8e3a186 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -21,7 +21,7 @@ from simplyblock_core import storage_node_ops from simplyblock_core.models.storage_node import StorageNode -from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice +from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice, RemoteJMDevice from simplyblock_core.models.cluster import Cluster from simplyblock_core.rpc_client import RPCConnectionError, RPCException @@ -966,6 +966,61 @@ def test_skips_already_removed_peer_with_stale_jm_ids(self): self.assertEqual(stale_peer.jm_ids, [removed.jm_device.get_id()]) stale_peer.write_to_db.assert_not_called() + def test_refreshes_peer_holding_dead_jm_only_via_hosted_primary(self): + # 2026-08-14 incident: _connect_to_remote_jm_devs populates + # remote_jm_devices from TWO sources -- a node's own jm_ids (its + # redundancy set for its own JM), AND, separately, whichever + # primary it hosts as secondary/tertiary pulls in THAT primary's + # jm_ids too (lvstore_stack_secondary/_tertiary). A peer reachable + # only through the second path never touches its own jm_ids at + # all, so the jm_ids-only guard above never even looks at it, and + # its remote_jm_devices entry for the dead JM is left stale + # forever. A plain removal that never reshuffles who-hosts-whom + # never surfaces this (peer's remote_jm_devices happens to already + # be right) -- it takes a splice reshuffle to expose it. + cl = _cluster() + removed = _node("n1", n_devices=0, with_jm=True) + removed.jm_ids = [] + peer = _node("peer1", stack_secondary="some-primary") + peer.jm_ids = [] # clean -- the dead JM was never in ITS OWN set + stale_remote = RemoteJMDevice() + stale_remote.uuid = removed.jm_device.get_id() + peer.remote_jm_devices = [stale_remote] + db = FakeDB(cl, [removed, peer]) + dc = MagicMock() + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "device_controller", dc), \ + patch.object(storage_node_ops, "_connect_to_remote_jm_devs", + return_value=[]) as connect_mock: + ret = storage_node_ops._decommission_node_devices(removed) + + self.assertTrue(ret) + dc.remove_jm_device.assert_called_once() + connect_mock.assert_called_once_with(peer, peer.jm_ids) + self.assertEqual(peer.remote_jm_devices, []) + peer.write_to_db.assert_called_once() + + def test_does_not_refresh_peer_with_no_dead_jm_reference_at_all(self): + # Regression guard for the new elif's condition itself: a peer with + # neither the dead JM in its own jm_ids NOR in remote_jm_devices + # must be left completely untouched. + cl = _cluster() + removed = _node("n1", n_devices=0, with_jm=True) + removed.jm_ids = [] + peer = _node("peer1") + peer.jm_ids = [] + peer.remote_jm_devices = [] + db = FakeDB(cl, [removed, peer]) + dc = MagicMock() + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "device_controller", dc), \ + patch.object(storage_node_ops, "_connect_to_remote_jm_devs") as connect_mock: + ret = storage_node_ops._decommission_node_devices(removed) + + self.assertTrue(ret) + connect_mock.assert_not_called() + peer.write_to_db.assert_not_called() + # --------------------------------------------------------------------------- # node_removal_orchestrate — phase-5 resume gap From 43d3a67492873aeab0c8555a80e38aef3cdad193 Mon Sep 17 00:00:00 2001 From: wmousa Date: Fri, 14 Aug 2026 19:05:12 +0200 Subject: [PATCH 15/27] fix(node-removal): detach evicted peer's hublvol controller, not just its bdev stack _delete_replica_on_peer tore down the evicted peer's bdev stack for the primary's replica but never detached the peer's NVMe-oF controller consuming the primary's hublvol - that connection stays live the whole time the peer holds the replica, for fast failover. Left dangling, it can later be found wedged in a non-enabled state if the same peer is re-selected to host a replica of the same primary again before its next restart, and HublvolReconnectCoordinator's detach-and-wait-gone then times out, aborting the rebuild. Live sequence that hit this (2026-08-14): a splice eviction during one node removal moved a primary's replica off ffznh onto another peer, leaving ffznh's controller to that primary's hublvol connected but idle. A later, unrelated node removal re-selected ffznh to host that same primary's replica again; the stale controller's detach-wait-gone timed out, which killed SPDK on ffznh and marked it offline. The peer's own hublvol subsystem (its own exposed endpoint for this replica, dormant with no consumers) stays commented out per f5a052f3 - only the consumer-side controller detach is being added here, mirroring the existing pattern in teardown_non_leader_lvstore. Rewrote TestDeleteReplicaOnPeer, which pre-dates this session and still asserted the subsystem_get/subsystem_delete/bdev_lvol_delete_hublvol calls f5a052f3 commented out; it now covers the new detach call instead. --- simplyblock_core/storage_node_ops.py | 21 +++++++++++ tests/unit/test_node_removal.py | 53 ++++++++++++++-------------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 3129a1a3a3..bb7f4926c3 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -3862,6 +3862,9 @@ def _delete_replica_on_peer(peer, primary, cluster): lvstore = primary.lvstore if not lvstore: return + # peer's own hublvol subsystem for this replica has no consumers -- only + # a promoted primary's hublvol is ever attached-to -- so it sits dormant + # and is cleaned up naturally on peer's next restart; left alone here. # try: # nqn = peer.hublvol_nqn_for_lvstore(cluster.nqn, lvstore) # if rpc_client.subsystem_get(nqn): @@ -3872,6 +3875,24 @@ def _delete_replica_on_peer(peer, primary, cluster): # rpc_client.bdev_lvol_delete_hublvol(lvstore) # except RPCException as e: # logger.warning(f"hublvol bdev teardown for {lvstore} on {peer.get_id()} failed: {e}") + + # UNLIKE the subsystem above, peer's NVMe-oF controller connecting TO + # primary's hublvol *is* a live consumer connection (kept attached the + # whole time peer held this replica, for fast failover) and must be + # detached here -- mirrors teardown_non_leader_lvstore's step 2. Leaving + # it dangling is not harmless: if peer is later re-selected to host a + # replica of this same primary again before its next restart, the stale + # controller can be found wedged in a non-enabled state, and the + # reconcile's detach-and-wait-gone can then time out and abort the + # rebuild (2026-08-14: this exact sequence took ffznh's SPDK down -- + # this connection was left behind by an earlier splice eviction and + # ffznh was never restarted before being re-selected as 5bc9k's + # secondary again). + if primary.hublvol and primary.hublvol.bdev_name: + try: + rpc_client.bdev_nvme_detach_controller(primary.hublvol.bdev_name) + except RPCException as e: + logger.warning(f"hublvol controller detach for {lvstore} on {peer.get_id()} failed: {e}") try: # deepcopy: _remove_bdev_stack stamps bdev['status']; don't mutate the # primary's stored stack definition. diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 30a8e3a186..41bedc0d53 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -446,57 +446,58 @@ def test_clears_bookkeeping_both_sides(self): # _delete_replica_on_peer — the peer-side hublvol+bdev teardown _teardown_ # replicas_of_primary and _relocate_replica_between both call. # -# Regression coverage for a real bug found live (2026-08-10): the hublvol -# subsystem check called rpc_client.subsystem_list(nqn), but subsystem_list() -# takes no arguments at all (it lists everything, unfiltered) -- every call -# raised TypeError, silently swallowed by the surrounding best-effort -# try/except, so the hublvol subsystem was never actually torn down on any -# peer during node removal. subsystem_get(nqn) is the existing, correct, -# server-side-filtered method for this. +# peer's own hublvol subsystem/bdev teardown (subsystem_get/subsystem_delete/ +# bdev_lvol_delete_hublvol) is deliberately commented out as of f5a052f3 -- +# that subsystem has no consumers and is harmless to leave until peer's next +# restart -- so these tests do NOT assert those calls. +# +# Regression coverage for a real bug found live (2026-08-14): peer's NVMe-oF +# controller connecting TO primary's hublvol (kept live the whole time peer +# held this replica) was never detached on eviction. Left dangling, it can +# later be found wedged in a non-enabled state when peer is re-selected to +# host a replica again, and the reconcile's detach-and-wait-gone can then +# time out and abort the rebuild -- this exact sequence took ffznh's SPDK +# down after a splice eviction left this connection behind. # --------------------------------------------------------------------------- class TestDeleteReplicaOnPeer(unittest.TestCase): - def test_deletes_subsystem_when_present(self): + def test_detaches_hublvol_controller_when_present(self): cl = _cluster() primary = _node("p1", lvstore="LVS_1") - peer = _node("peer1", lvstore="LVS_1") # hublvol_nqn_for_lvstore's mocked return value bakes in this lvstore + primary.hublvol = MagicMock(bdev_name="LVS_1/hublvol") + peer = _node("peer1", lvstore="LVS_1") rpc = peer.rpc_client() - rpc.subsystem_get.return_value = {"nqn": "nqn:hub:LVS_1"} storage_node_ops._delete_replica_on_peer(peer, primary, cl) - rpc.subsystem_get.assert_called_once_with("nqn:hub:LVS_1") - rpc.subsystem_delete.assert_called_once_with("nqn:hub:LVS_1") + rpc.bdev_nvme_detach_controller.assert_called_once_with("LVS_1/hublvol") - def test_skips_delete_when_subsystem_absent(self): + def test_skips_hublvol_detach_when_primary_has_no_hublvol(self): cl = _cluster() primary = _node("p1", lvstore="LVS_1") - peer = _node("peer1", lvstore="LVS_1") # hublvol_nqn_for_lvstore's mocked return value bakes in this lvstore + primary.hublvol = None + peer = _node("peer1", lvstore="LVS_1") rpc = peer.rpc_client() - rpc.subsystem_get.return_value = None storage_node_ops._delete_replica_on_peer(peer, primary, cl) - rpc.subsystem_get.assert_called_once_with("nqn:hub:LVS_1") - rpc.subsystem_delete.assert_not_called() + rpc.bdev_nvme_detach_controller.assert_not_called() - def test_subsystem_get_failure_is_caught_not_raised(self): + def test_hublvol_detach_failure_is_caught_not_raised(self): # Best-effort: an RPC failure here must not propagate and block removal. cl = _cluster() primary = _node("p1", lvstore="LVS_1") - peer = _node("peer1", lvstore="LVS_1") # hublvol_nqn_for_lvstore's mocked return value bakes in this lvstore + primary.hublvol = MagicMock(bdev_name="LVS_1/hublvol") + peer = _node("peer1", lvstore="LVS_1") rpc = peer.rpc_client() - rpc.subsystem_get.side_effect = RPCConnectionError("connection error") + rpc.bdev_nvme_detach_controller.side_effect = RPCConnectionError("connection error") storage_node_ops._delete_replica_on_peer(peer, primary, cl) # must not raise - rpc.subsystem_delete.assert_not_called() - # Teardown of the other artifacts still proceeds despite this failure. - rpc.bdev_lvol_delete_hublvol.assert_called_once_with("LVS_1") def test_no_op_when_primary_has_no_lvstore(self): cl = _cluster() primary = _node("p1", lvstore="") - peer = _node("peer1", lvstore="LVS_1") # hublvol_nqn_for_lvstore's mocked return value bakes in this lvstore + primary.hublvol = MagicMock(bdev_name="LVS_1/hublvol") + peer = _node("peer1", lvstore="LVS_1") rpc = peer.rpc_client() storage_node_ops._delete_replica_on_peer(peer, primary, cl) - rpc.subsystem_get.assert_not_called() - rpc.bdev_lvol_delete_hublvol.assert_not_called() + rpc.bdev_nvme_detach_controller.assert_not_called() # --------------------------------------------------------------------------- From 055125d53e96af2e3f2a3236c261a8bf9ca97beb Mon Sep 17 00:00:00 2001 From: wmousa Date: Mon, 17 Aug 2026 12:28:19 +0200 Subject: [PATCH 16/27] fix(node-removal): don't destroy the shared lvstore when only relocating a non-leader replica _delete_replica_on_peer called _remove_bdev_stack without remove_distr_only=True, so it always took the bdev_lvol_delete_lvstore branch -- destroying the shared on-disk blobstore metadata, not just the peer's local examine copy. teardown_non_leader_lvstore (the single-node-expansion sibling) already gets this right and spells out why in its docstring: the lvstore must never be deleted on a non-leader, since bdev_lvol_delete_lvstore wipes metadata shared by every replica. That's correct for _teardown_replicas_of_primary (Case A): primary IS the node being removed, so destroying its lvstore there is intentional -- nothing will ever read it again once its own devices are decommissioned. It's wrong for _relocate_replica_between's splice/relocation eviction: occupant_primary there is a SURVIVING node whose replica is only being moved to a new host. old_host held nothing but a non-leader examine copy of a still-live lvstore, and destroying it destroys the shared blobstore metadata out from under the primary and any other surviving replica. Live sequence that hit this (2026-08-16): a splice eviction moved vdr27's LVS_1 replica off 66gqf via this exact path, corrupting LVS_1's on-disk metadata. It surfaced later as "unsupported version on super block" (blobstore.c:bs_super_validate) when vdr27 tried to reload LVS_1 on restart -- by then indistinguishable from the leadership-loss symptom under investigation, and only traced back here afterward via log analysis. Added a destroy_lvstore parameter (default True, preserving Case A's existing/correct behavior) and pass destroy_lvstore=False from the splice eviction call site, threading through to _remove_bdev_stack(remove_distr_only=not destroy_lvstore) -- the same mechanism teardown_non_leader_lvstore already uses. --- simplyblock_core/storage_node_ops.py | 35 +++++++++++++++++++--- tests/unit/test_node_removal.py | 44 +++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index bb7f4926c3..54fbdf93fa 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -3854,10 +3854,31 @@ def _teardown_replicas_of_primary(removed_node: StorageNode): return True -def _delete_replica_on_peer(peer, primary, cluster): +def _delete_replica_on_peer(peer, primary, cluster, destroy_lvstore=True): """Best-effort teardown of ``primary``'s replica lvstore (+ hublvol) on the online ``peer``. RPC failures are logged, not fatal: the peer is healthy and - a lingering empty bdev is harmless, while blocking removal on it is not.""" + a lingering empty bdev is harmless, while blocking removal on it is not. + + ``destroy_lvstore``: whether the shared on-disk blobstore backing this + lvstore may be destroyed once the local bdev stack comes down. + + - ``True`` (default -- Case A, node removal): ``primary`` IS the node + being removed; nothing else will ever read this lvstore again once + its own devices are decommissioned, so destroying it here is correct. + - ``False`` (splice/relocation eviction): ``primary`` is a SURVIVING + node whose replica is only being *moved* off ``peer`` onto a new + host -- the lvstore itself stays alive and in use elsewhere. + ``peer`` only ever held a non-leader examine copy, so + ``bdev_lvol_delete_lvstore`` here would destroy the shared blobstore + metadata out from under the still-live primary and any other + surviving replica (2026-08-16: this is what actually corrupted + LVS_1's on-disk metadata during a splice eviction, surfacing later + as `bs_super_validate: unsupported version on super block` when the + primary tried to reload it on restart -- caught only after the fact + via log analysis, not before). Pass ``False`` here; only the local + raid/distrib examine bdevs get hot-removed, matching + ``teardown_non_leader_lvstore``'s existing, correct pattern for the + identical non-leader-eviction case.""" rpc_client = peer.rpc_client() lvstore = primary.lvstore if not lvstore: @@ -3896,7 +3917,8 @@ def _delete_replica_on_peer(peer, primary, cluster): try: # deepcopy: _remove_bdev_stack stamps bdev['status']; don't mutate the # primary's stored stack definition. - _remove_bdev_stack(copy.deepcopy(primary.lvstore_stack), rpc_client) + _remove_bdev_stack(copy.deepcopy(primary.lvstore_stack), rpc_client, + remove_distr_only=not destroy_lvstore) except RPCException as e: logger.warning(f"replica bdev-stack teardown for {lvstore} on {peer.get_id()} failed: {e}") @@ -4134,7 +4156,12 @@ def _relocate_replica_between(occupant_primary_id, old_host_id, new_host_id, rol if getattr(old_host, backref) == occupant_primary_id: cluster = db_controller.get_cluster_by_id(occupant_primary.cluster_id) if old_host.status == StorageNode.STATUS_ONLINE: - _delete_replica_on_peer(old_host, occupant_primary, cluster) + # occupant_primary survives this relocation (only its host is + # moving) -- must NOT destroy the shared lvstore, only vacate + # old_host's local examine copy. See _delete_replica_on_peer's + # destroy_lvstore docstring. + _delete_replica_on_peer(old_host, occupant_primary, cluster, + destroy_lvstore=False) _prune_stale_lvstore_ports(old_host_id, occupant_primary.lvstore, db_controller) old_host = db_controller.get_storage_node_by_id(old_host_id) setattr(old_host, backref, "") diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 41bedc0d53..59b16f133d 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -440,6 +440,10 @@ def test_clears_bookkeeping_both_sides(self): self.assertEqual(sec.lvstore_stack_secondary, "") self.assertEqual(tert.lvstore_stack_tertiary, "") self.assertEqual(drp.call_count, 2) + # Case A: removed IS the node going away -- destroying its lvstore + # here is correct (default destroy_lvstore=True, not overridden). + drp.assert_any_call(sec, removed, cl) + drp.assert_any_call(tert, removed, cl) # --------------------------------------------------------------------------- @@ -499,6 +503,40 @@ def test_no_op_when_primary_has_no_lvstore(self): storage_node_ops._delete_replica_on_peer(peer, primary, cl) rpc.bdev_nvme_detach_controller.assert_not_called() + # ----------------------------------------------------------------- + # destroy_lvstore -- regression coverage for a real bug found live + # (2026-08-16): the splice/relocation eviction path called this with + # the default (destroy) behavior, which calls bdev_lvol_delete_lvstore + # on a peer holding only a non-leader examine copy -- destroying the + # SHARED on-disk blobstore metadata out from under the still-live + # primary elsewhere. This corrupted LVS_1's on-disk metadata during a + # splice eviction, surfacing later as a superblock validation failure + # when the primary tried to reload it on restart. + # ----------------------------------------------------------------- + + def test_destroy_lvstore_default_true_deletes_shared_blobstore(self): + # Case A (node removal): primary IS the node going away, so + # destroying its lvstore here is correct. + cl = _cluster() + primary = _node("p1", lvstore="LVS_1") + peer = _node("peer1", lvstore="LVS_1") + rpc = peer.rpc_client() + storage_node_ops._delete_replica_on_peer(peer, primary, cl) + rpc.bdev_lvol_delete_lvstore.assert_called_once_with("LVS_1") + + def test_destroy_lvstore_false_never_deletes_shared_blobstore(self): + # Splice/relocation eviction: primary survives, only its host on + # this peer is moving -- the shared blobstore must NOT be touched, + # only the local raid/distrib examine bdevs hot-removed. + cl = _cluster() + primary = _node("p1", lvstore="LVS_1") + peer = _node("peer1", lvstore="LVS_1") + rpc = peer.rpc_client() + storage_node_ops._delete_replica_on_peer(peer, primary, cl, destroy_lvstore=False) + rpc.bdev_lvol_delete_lvstore.assert_not_called() + rpc.bdev_raid_delete.assert_called_once_with("raid_1") + rpc.bdev_distrib_delete.assert_called_once_with("distrib_1") + # --------------------------------------------------------------------------- # Case B — relocate a hosted replica @@ -595,7 +633,11 @@ def test_relocate_via_splice_evicts_occupant_first(self): ret = storage_node_ops._relocate_one_replica(removed, "stranded", "secondary") self.assertTrue(ret) - drp.assert_called_once() # occupant's old replica torn down off x + # occupant's old replica torn down off x -- must NOT destroy the + # shared lvstore: occupant survives this relocation, only its host + # moves (2026-08-16: destroying it here corrupted the on-disk + # blobstore for a still-live primary). + drp.assert_called_once_with(x, occupant, cl, destroy_lvstore=False) self.assertEqual(occupant.secondary_node_id, "stranded") # occupant re-homed onto stranded self.assertEqual(stranded.secondary_node_id, "x") # stranded takes over x's freed slot self.assertEqual(x.lvstore_stack_secondary, "stranded") From d7603c25d4aaac06bc8d1c3efaf19f55a93bc661 Mon Sep 17 00:00:00 2001 From: wmousa Date: Mon, 17 Aug 2026 18:01:47 +0200 Subject: [PATCH 17/27] fix(web): make SimplyblockCollector inherit prometheus_client's Collector Surfaced by the rebase onto origin/main: mypy flagged SimplyblockCollector as not satisfying the CollectorRegistry.register() parameter type, which is typed against prometheus_client's Collector protocol/base class. --- simplyblock_web/api/v2/metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/simplyblock_web/api/v2/metrics.py b/simplyblock_web/api/v2/metrics.py index 98a8afa020..74ff2d1884 100644 --- a/simplyblock_web/api/v2/metrics.py +++ b/simplyblock_web/api/v2/metrics.py @@ -20,6 +20,7 @@ from fastapi.responses import Response from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, generate_latest from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily, Metric +from prometheus_client.registry import Collector from simplyblock_core.db_controller import DBController from simplyblock_core.models.stats import CpuStats, ReactorStats, ThreadStats @@ -257,7 +258,7 @@ def _cpu_families(entries: Iterable[tuple[list[str], CpuStats]]) -> Iterator[Met yield family -class SimplyblockCollector: +class SimplyblockCollector(Collector): """Builds the full metric set from FoundationDB on every scrape.""" def collect(self) -> Iterator[Metric]: From 1fcbd9dc1833826253698fc95e9ea4dc650e2de1 Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Mon, 17 Aug 2026 20:26:08 +0300 Subject: [PATCH 18/27] fix(storage_node_ops): update node shutdown logic to exclude pending removal status Do not cancel fail device migration task on node shutdown --- simplyblock_core/services/tasks_runner_migration.py | 6 ++++++ simplyblock_core/storage_node_ops.py | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/simplyblock_core/services/tasks_runner_migration.py b/simplyblock_core/services/tasks_runner_migration.py index a457f914a5..405dc63b8c 100644 --- a/simplyblock_core/services/tasks_runner_migration.py +++ b/simplyblock_core/services/tasks_runner_migration.py @@ -86,6 +86,12 @@ def task_runner(task): _migration_retry_allowed(task, unavailable) return False + if snode.status == StorageNode.STATUS_REMOVED: + task.status = JobSchedule.STATUS_DONE + task.function_result = f"Node found in r: {task.node_id}" + task.write_to_db(db.kv_store) + return True + cluster = db.get_cluster_by_id(task.cluster_id) if cluster.status not in Cluster.OPERABLE_STATUSES: task.function_result = "cluster is not active, retrying" diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 54fbdf93fa..b640d50952 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -6170,7 +6170,6 @@ def shutdown_storage_node(node_id, force=False, keep_auto_restart=False, continue if task.function_name in [ JobSchedule.FN_DEV_MIG, - JobSchedule.FN_FAILED_DEV_MIG, JobSchedule.FN_NEW_DEV_MIG, ]: task.canceled = True From 57d6ace628c69d3b8cef067ec8b4b3cb3ad09d09 Mon Sep 17 00:00:00 2001 From: wmousa Date: Tue, 18 Aug 2026 14:54:34 +0200 Subject: [PATCH 19/27] fix(node-removal): repoint lvol.nodes when a replica relocates, not just the storage-node bookkeeping _relocate_one_replica (normal Case B relocation) and _relocate_replica_between (the splice/rotation fallback) both moved a peer's secondary/tertiary replica from one host to another correctly at the storage-node level -- updating secondary_node_id/tertiary_node_id and the lvstore_stack_secondary/_tertiary back-references -- but never touched the nodes list of any LVol hosted on the surviving primary. lvol.nodes is a separate record from that storage-node bookkeeping: it's what the CSI/host initiator actually connects to for multipath failover. Leaving the old (now removed/vacated) host listed there strands every lvol hosted on that primary on a single path once the old host is gone, with nothing ever repointing it afterward -- confirmed live (2026-08-18): after a node removal relocated a peer's secondary replica elsewhere, an lvol's nodes field still named the just-removed node, and the NVMe-oF initiator showed only one live path (the primary) instead of two. The codebase already has the identical fix for the analogous case in cluster_expansion/executor.py (expansion-triggered rebalancing repoints lvol.nodes when a donor's role moves to a recipient) -- this was simply missing from the node-removal relocation path. Fix: a shared _update_lvol_nodes_for_replica_move() helper, called from both _relocate_one_replica and _relocate_replica_between right after the new replica's build is confirmed (and unconditionally, not gated on whether the build ran this pass, so a retry that resumes past an already-applied build still catches up if an earlier attempt crashed between the build and this step). --- simplyblock_core/storage_node_ops.py | 41 +++++++++ tests/unit/test_node_removal.py | 125 +++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index b640d50952..ac555690c4 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -3942,6 +3942,33 @@ def _prune_stale_lvstore_ports(node_id, lvstore, db_controller): node.write_to_db() +def _update_lvol_nodes_for_replica_move(primary_id, old_host_id, new_host_id, db_controller): + """Re-point every LVol hosted on ``primary_id`` from ``old_host_id`` to + ``new_host_id`` in its own ``nodes`` list, once that primary's + secondary/tertiary replica has been relocated between the two hosts. + + ``lvol.nodes`` is what the CSI/host initiator actually connects to for + multipath failover -- a separate record from the storage-node-level + ``secondary_node_id``/``tertiary_node_id`` bookkeeping the relocation + itself already updates. Leaving the old host listed here strands every + LVol hosted on ``primary_id`` on a single path (the primary only) once + the old host is gone/unreachable, with nothing ever repointing it -- + mirrors cluster_expansion/executor.py's identical fix for the + expansion-rebalancing case. (2026-08-18: found live during node + removal -- a splice-relocated secondary left a still-online lvol's + ``nodes`` naming the just-removed node; the CSI initiator never + reconnected to the actual new secondary.) + + Safe to call redundantly (e.g. on a retry after an earlier attempt + already applied it): each lvol is only rewritten if ``old_host_id`` is + still present in its ``nodes``.""" + for lvol in db_controller.get_lvols_by_node_id(primary_id): + nodes = list(lvol.nodes or []) + if old_host_id in nodes: + lvol.nodes = [new_host_id if n == old_host_id else n for n in nodes] + lvol.write_to_db() + + def _relocate_replicas_hosted_on(removed_node: StorageNode): """Case B: ``removed_node`` holds a secondary and/or tertiary replica for other primaries. Re-host each on a fresh, anti-affinity-valid node so the @@ -4022,6 +4049,13 @@ def _relocate_one_replica(removed_node: StorageNode, primary_id, role): f"[REMOVAL] failed to rebuild {role} replica of {primary_id} on {new_id}, will retry") return False + # Re-point every LVol hosted on primary before dropping removed_node's + # side of the relationship -- see _update_lvol_nodes_for_replica_move's + # docstring. Unconditional (not gated on "did we just build it above") + # so a retry that resumes past the build-skip branch still catches up + # if an earlier attempt crashed between the build and this step. + _update_lvol_nodes_for_replica_move(primary_id, removed_node.get_id(), new_id, db_controller) + _clear_replica_backref(removed_node, backref) return True @@ -4152,6 +4186,13 @@ def _relocate_replica_between(occupant_primary_id, old_host_id, new_host_id, rol } new_host.write_to_db() + # Re-point every LVol hosted on occupant_primary before the old_host + # teardown below -- see _update_lvol_nodes_for_replica_move's docstring. + # Unconditional (outside the "if not already built" guard above) so a + # retry that skips straight past that guard still catches up if an + # earlier attempt crashed between the build and this step. + _update_lvol_nodes_for_replica_move(occupant_primary_id, old_host_id, new_host_id, db_controller) + old_host = db_controller.get_storage_node_by_id(old_host_id) if getattr(old_host, backref) == occupant_primary_id: cluster = db_controller.get_cluster_by_id(occupant_primary.cluster_id) diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 59b16f133d..1a12e73c87 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -538,6 +538,68 @@ def test_destroy_lvstore_false_never_deletes_shared_blobstore(self): rpc.bdev_distrib_delete.assert_called_once_with("distrib_1") +# --------------------------------------------------------------------------- +# _update_lvol_nodes_for_replica_move — regression coverage for a real bug +# found live (2026-08-18): a node-removal relocation correctly repointed the +# storage-node-level secondary_node_id/tertiary_node_id bookkeeping, but left +# every LVol hosted on the surviving primary with the just-removed/vacated +# host still listed in its own `nodes` -- the field the CSI/host initiator +# actually connects to for multipath failover. That stranded a live lvol on +# a single path (the primary only) with nothing ever repointing it, since +# nothing re-syncs `nodes` later. Mirrors the identical fix already applied +# for expansion-triggered rebalancing in cluster_expansion/executor.py. +# --------------------------------------------------------------------------- + +def _lvol(node_id, nodes): + lv = MagicMock() + lv.nodes = list(nodes) + lv.write_to_db = MagicMock() + return lv + + +class TestUpdateLvolNodesForReplicaMove(unittest.TestCase): + + def test_repoints_old_host_to_new_host(self): + cl = _cluster() + db = FakeDB(cl, [], lvols={"p1": [_lvol("p1", ["p1", "old"])]}) + storage_node_ops._update_lvol_nodes_for_replica_move("p1", "old", "new", db) + lvol = db.lvols["p1"][0] + self.assertEqual(lvol.nodes, ["p1", "new"]) + lvol.write_to_db.assert_called_once() + + def test_leaves_unrelated_hosts_untouched(self): + cl = _cluster() + db = FakeDB(cl, [], lvols={"p1": [_lvol("p1", ["p1", "old", "tert"])]}) + storage_node_ops._update_lvol_nodes_for_replica_move("p1", "old", "new", db) + self.assertEqual(db.lvols["p1"][0].nodes, ["p1", "new", "tert"]) + + def test_no_op_when_old_host_not_present(self): + # e.g. a tertiary-only move must not touch an lvol with no tertiary. + cl = _cluster() + lvol = _lvol("p1", ["p1", "sec"]) + db = FakeDB(cl, [], lvols={"p1": [lvol]}) + storage_node_ops._update_lvol_nodes_for_replica_move("p1", "old", "new", db) + self.assertEqual(lvol.nodes, ["p1", "sec"]) + lvol.write_to_db.assert_not_called() + + def test_multiple_lvols_on_the_same_primary_all_updated(self): + cl = _cluster() + lvols = [_lvol("p1", ["p1", "old"]) for _ in range(3)] + db = FakeDB(cl, [], lvols={"p1": lvols}) + storage_node_ops._update_lvol_nodes_for_replica_move("p1", "old", "new", db) + for lvol in lvols: + self.assertEqual(lvol.nodes, ["p1", "new"]) + + def test_redundant_call_is_a_safe_no_op(self): + # Simulates a retry after an earlier attempt already applied it. + cl = _cluster() + lvol = _lvol("p1", ["p1", "new"]) + db = FakeDB(cl, [], lvols={"p1": [lvol]}) + storage_node_ops._update_lvol_nodes_for_replica_move("p1", "old", "new", db) + self.assertEqual(lvol.nodes, ["p1", "new"]) + lvol.write_to_db.assert_not_called() + + # --------------------------------------------------------------------------- # Case B — relocate a hosted replica # --------------------------------------------------------------------------- @@ -607,6 +669,43 @@ def test_relocate_missing_primary_just_clears(self): self.assertTrue(ret) self.assertEqual(removed.lvstore_stack_secondary, "") + def test_relocate_repoints_primarys_lvol_nodes_off_the_removed_host(self): + # Regression: the direct (non-splice) relocation path must repoint + # every LVol hosted on the primary from the removed host to the new + # one -- see TestUpdateLvolNodesForReplicaMove's docstring. + cl = _cluster() + removed = _node("n1", stack_secondary="p1") + primary = _node("p1", secondary_id="n1", lvstore="LVS_p1") + new = _node("n3") + lvol = _lvol("p1", ["p1", "n1"]) + db = FakeDB(cl, [removed, primary, new], lvols={"p1": [lvol]}) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + return_value="n3"), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + return_value=True): + ret = storage_node_ops._relocate_one_replica(removed, "p1", "secondary") + self.assertTrue(ret) + self.assertEqual(lvol.nodes, ["p1", "n3"]) + + def test_relocate_failure_does_not_repoint_lvol_nodes(self): + # A failed rebuild must leave the lvol pointed at the still-intact + # old copy -- nothing to repoint to yet. + cl = _cluster() + removed = _node("n1", stack_secondary="p1") + primary = _node("p1", secondary_id="n1", lvstore="LVS_p1") + new = _node("n3") + lvol = _lvol("p1", ["p1", "n1"]) + db = FakeDB(cl, [removed, primary, new], lvols={"p1": [lvol]}) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + return_value="n3"), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + return_value=False): + ret = storage_node_ops._relocate_one_replica(removed, "p1", "secondary") + self.assertFalse(ret) + self.assertEqual(lvol.nodes, ["p1", "n1"]) + # --------------------------------------------------------------------------- # Case B, splice fallback — _pick_replica_relocation_node returned a BUSY @@ -644,6 +743,32 @@ def test_relocate_via_splice_evicts_occupant_first(self): self.assertEqual(rec.call_count, 2) # occupant's rebuild + stranded's own rebuild self.assertEqual(removed.lvstore_stack_secondary, "") + def test_relocate_via_splice_repoints_lvol_nodes_for_both_moved_primaries(self): + # Regression: BOTH moves this splice performs -- occupant's replica + # x -> stranded, and stranded's own replica n1 -> x -- must repoint + # every lvol hosted on their respective primaries. See + # TestUpdateLvolNodesForReplicaMove's docstring. + cl = _cluster() + removed = _node("n1", stack_secondary="stranded") + stranded = _node("stranded", secondary_id="n1", lvstore="LVS_stranded") + occupant = _node("occupant", secondary_id="x", lvstore="LVS_occupant") + x = _node("x", stack_secondary="occupant") + occupant_lvol = _lvol("occupant", ["occupant", "x"]) + stranded_lvol = _lvol("stranded", ["stranded", "n1"]) + db = FakeDB(cl, [removed, stranded, occupant, x], + lvols={"occupant": [occupant_lvol], "stranded": [stranded_lvol]}) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + return_value="x"), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + return_value=True), \ + patch.object(storage_node_ops, "_delete_replica_on_peer"): + ret = storage_node_ops._relocate_one_replica(removed, "stranded", "secondary") + + self.assertTrue(ret) + self.assertEqual(occupant_lvol.nodes, ["occupant", "stranded"]) + self.assertEqual(stranded_lvol.nodes, ["stranded", "x"]) + def test_relocate_via_splice_occupant_rebuild_failure_leaves_old_copy_untouched(self): # Create-before-destroy: a failed rebuild on the stranded node must # NOT tear down or repoint the occupant's still-intact old copy on x From 82c3ab8dcfa62ea81ee34d334bf7cfbe46b52d92 Mon Sep 17 00:00:00 2001 From: wmousa Date: Tue, 18 Aug 2026 16:17:14 +0200 Subject: [PATCH 20/27] fix(node-removal): don't guess a listener port when a non-leader host's lvstore_ports entry hasn't landed yet add_lvol_thread's per-lvstore port lookup used snode.get_lvol_subsys_port(lvol.lvs_name), whose fallback to the node's own lvol_subsys_port is correct ONLY when lvs_name is that node's own primary lvstore. For any other lvs_name, a missing lvstore_ports entry means the relocation that assigned this node a new non-leader role hasn't finished committing that bookkeeping yet -- not "no per-lvstore override configured". add_lvol_thread already has one documented "callers hold stale objects" guard (the in_deletion check at the top of the function, for a stale lvol); snode carried the identical hazard uncounted. Found live (2026-08-18): a node-removal splice relocated two lvols' secondaries onto new hosts. lvol.nodes was correctly repointed (the previous fix in this area), which woke lvol_monitor's repair loop for both almost immediately -- racing _relocate_replica_between's own lvstore_ports commit on the same new host. add_lvol_thread silently fell back to the new host's OWN leader port for both, and nvmf_subsystem_add_listener published a live listener on the wrong port. Nothing ever revisits or corrects it afterward: the CSI initiator correctly detects the subsystem is degraded (active=1 expected=2) and retries the connect indefinitely, but the wrong port is never reachable, so the secondary path never comes up. Fix: when lvol.lvs_name differs from snode.lvstore (this node is a non-leader host, not the lvstore's own primary) and lvol.lvs_name is missing from snode.lvstore_ports, re-fetch snode once and check again before trusting the port lookup. If it's still missing, refuse the registration instead of guessing -- the next lvol_monitor repair cycle retries once the commit has actually landed. The node's own leader-lvstore case (lvs_name == snode.lvstore, which legitimately has no lvstore_ports entry) is untouched -- get_lvol_subsys_port's existing fallback-to-node-default remains correct and unchanged for that case, and its own dedicated integration test (test_dual_ft_secondary_fixes.py) is unaffected. --- simplyblock_core/storage_node_ops.py | 22 +++- .../unit/test_missing_namespace_path_loss.py | 105 ++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index ac555690c4..5579fb51a3 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -10081,7 +10081,27 @@ def add_lvol_thread(lvol, snode: StorageNode, lvol_ana_state="optimized"): logger.error(msg) return False, msg - # Use per-lvstore port for this lvol's lvstore + # Use per-lvstore port for this lvol's lvstore. get_lvol_subsys_port()'s + # fallback to snode.lvol_subsys_port is only correct for lvol.lvs_name == + # snode.lvstore (this node's OWN primary, which legitimately has no + # lvstore_ports entry -- it uses the plain node-level port). For any + # OTHER lvs_name, a missing entry means the relocation that assigned + # snode this non-leader role hasn't finished committing lvstore_ports + # yet -- snode here can be a stale, caller-held object (same hazard as + # the in_deletion check above). Silently falling back would register + # the listener on snode's OWN leader port instead of lvol.lvs_name's + # real one (2026-08-18: raced a node-removal relocation live, leaving + # two lvols' secondaries listening on the wrong port indefinitely, with + # nothing to ever revisit or correct it). Re-fetch once and refuse + # rather than guess; the next lvol_monitor repair cycle retries. + if lvol.lvs_name != snode.lvstore and lvol.lvs_name not in snode.lvstore_ports: + snode = db_controller.get_storage_node_by_id(snode.get_id()) + if lvol.lvs_name not in snode.lvstore_ports: + msg = (f"{snode.get_id()} has no lvstore_ports entry for " + f"{lvol.lvs_name} yet; refusing to add a listener for " + f"{lvol.nqn} on a guessed port") + logger.warning(msg) + return False, msg listener_port = snode.get_lvol_subsys_port(lvol.lvs_name) for iface in snode.data_nics: if iface.ip4_address and lvol.fabric == iface.trtype.lower(): diff --git a/tests/unit/test_missing_namespace_path_loss.py b/tests/unit/test_missing_namespace_path_loss.py index 95ac17e873..3ba2da92f3 100644 --- a/tests/unit/test_missing_namespace_path_loss.py +++ b/tests/unit/test_missing_namespace_path_loss.py @@ -57,6 +57,12 @@ def _make_node(node_id="tertiary"): node.get_lvol_subsys_port.return_value = 4440 node.data_nics = [] node.cluster_id = "cluster-1" + # Distinct from lvol.lvs_name ("LVS_25") so the node is a non-leader + # host for it, with lvstore_ports already populated -- the normal, + # settled state. Tests targeting the guard in + # TestAddLvolThreadRefusesGuessedPort override this explicitly. + node.lvstore = "" + node.lvstore_ports = {"LVS_25": {"lvol_subsys_port": 4440, "hublvol_port": 4441}} return node @@ -157,5 +163,104 @@ def test_publishes_listener_when_namespace_present(self): self.rpc.listeners_create.assert_called() +class TestAddLvolThreadRefusesGuessedPort(unittest.TestCase): + """``add_lvol_thread`` must never fall back to snode's OWN leader port + when registering a listener for a lvstore it hosts as a non-leader. + + Regression coverage for a real bug found live (2026-08-18): a + node-removal relocation correctly repointed lvol.nodes to the new + secondary, but lvol_monitor's repair loop raced the relocation's own + lvstore_ports commit -- add_lvol_thread was handed a stale snode object + (same "callers hold stale objects" hazard as the in_deletion check + above) whose lvstore_ports had no entry yet for the lvol's lvstore. + get_lvol_subsys_port()'s fallback-to-node-default (correct ONLY when + lvs_name IS the node's own primary) silently returned the node's OWN + leader port instead, and a listener was published on the wrong port + with nothing ever revisiting or correcting it.""" + + def setUp(self): + self.rpc = MagicMock(name="rpc") + self.lvol = _make_lvol() + self.snode = _make_node("secondary") + self.snode.rpc_client.return_value = self.rpc + # This node leads its OWN lvstore (LVS_9) and is only a non-leader + # host for the lvol's lvstore (LVS_25) -- the exact shape that hit + # the bug live (zqmjp leading LVS_26 while hosting LVS_14 as a + # freshly-relocated secondary). + self.snode.lvstore = "LVS_9" + + nic = MagicMock() + nic.ip4_address = "192.168.10.12" + nic.trtype = "TCP" + self.snode.data_nics = [nic] + self.lvol.fabric = "tcp" + self.lvol.lvol_type = "lvol" + + self.rpc.nvmf_subsystem_add_ns.return_value = 1 + + p = patch.object(storage_node_ops, "DBController") + self.db = p.start() + self.addCleanup(p.stop) + self.db.return_value.get_lvol_by_id.return_value = self.lvol + self.db.return_value.get_pool_by_id.return_value.has_qos.return_value = False + + def test_refuses_when_lvstore_ports_missing_even_after_refetch(self): + # Stale caller-held snode has no entry; a fresh re-fetch (simulated + # here as the SAME missing state) confirms it's genuinely not ready. + self.snode.lvstore_ports = {} + fresh = _make_node("secondary") + fresh.lvstore = "LVS_9" + fresh.lvstore_ports = {} + self.db.return_value.get_storage_node_by_id.return_value = fresh + + with patch.object(storage_node_ops, "_rpc_subsystem_has_ns", + side_effect=[False, True]): + ok, err = storage_node_ops.add_lvol_thread(self.lvol, self.snode) + + self.assertFalse(ok) + self.assertIn("lvstore_ports", err) + self.rpc.listeners_create.assert_not_called() + # Must not have guessed using snode's OWN leader port either. + self.snode.get_lvol_subsys_port.assert_not_called() + + def test_proceeds_with_fresh_port_once_refetch_finds_it(self): + # Stale caller-held snode has no entry yet, but the relocation's + # commit has already landed by the time we re-fetch -- must use + # the FRESH object's real port, not snode's own leader port. + self.snode.lvstore_ports = {} + fresh = _make_node("secondary") + fresh.lvstore = "LVS_9" + fresh.lvstore_ports = {"LVS_25": {"lvol_subsys_port": 5150, "hublvol_port": 5151}} + fresh.get_lvol_subsys_port = MagicMock(return_value=5150) + fresh.rpc_client.return_value = self.rpc + fresh.data_nics = self.snode.data_nics + self.db.return_value.get_storage_node_by_id.return_value = fresh + + with patch.object(storage_node_ops, "_rpc_subsystem_has_ns", + side_effect=[False, True]): + ok, err = storage_node_ops.add_lvol_thread(self.lvol, self.snode) + + self.assertTrue(ok, err) + self.rpc.listeners_create.assert_called_once() + _, kwargs = self.rpc.listeners_create.call_args + self.assertEqual(self.rpc.listeners_create.call_args.args[3], 5150) + + def test_own_leader_lvstore_never_triggers_the_guard(self): + # The normal case: lvol.lvs_name IS this node's own primary + # lvstore, which legitimately has no lvstore_ports entry -- must + # use the plain node-level port without re-fetching or refusing. + self.snode.lvstore = self.lvol.lvs_name + self.snode.lvstore_ports = {} + + with patch.object(storage_node_ops, "_rpc_subsystem_has_ns", + side_effect=[False, True]): + ok, err = storage_node_ops.add_lvol_thread(self.lvol, self.snode) + + self.assertTrue(ok, err) + self.db.return_value.get_storage_node_by_id.assert_not_called() + self.rpc.listeners_create.assert_called_once() + self.assertEqual(self.rpc.listeners_create.call_args.args[3], 4440) + + if __name__ == "__main__": unittest.main() From b43b2c18215a0059cca4fcad84fa643fd52c995f Mon Sep 17 00:00:00 2001 From: wmousa Date: Tue, 18 Aug 2026 16:37:02 +0200 Subject: [PATCH 21/27] fix(node-removal): tear down a vacated peer's per-lvol NVMe-oF subsystems, not just its bdev stack _delete_replica_on_peer(destroy_lvstore=False) correctly tears down a vacated peer's local raid/distrib bdev stack for the lvstore it's giving up -- which cascades to remove each hosted LVol's namespace -- but never touches the per-LVol NVMe-oF subsystem+listener, which is registered separately (via add_lvol_thread) on top of that lvstore for every individual lvol. Left behind, the listener keeps accepting connections in front of a now-empty subsystem -- exactly the "live but no path" failure test_missing_namespace_ path_loss.py guards against, just reached through a different door this time: the CSI/host initiator's existing connection to the vacated peer stays live (the endpoint genuinely still answers), and since the peer is no longer in lvol.nodes after the previous fix in this area, nothing ever tells the initiator to drop that connection either. The volume ends up with a third, live-but-empty path sitting alongside its two correct ones indefinitely. Found live (2026-08-18) immediately after verifying the lvol.nodes and wrong-port fixes: both corrected lvols still carried this stale-but-live third path to their pre-relocation host, confirmed via nvmf_get_subsystems showing the listener present with "namespaces": []. Fix: a new _teardown_lvol_subsystems_on_vacated_peer() helper, called right after _delete_replica_on_peer in the splice-eviction path (_relocate_replica_between), deletes every LVol-of-the-relocated-primary's subsystem on the vacated peer via subsystem_delete(). Best-effort, matching _delete_replica_on_peer's own pattern: RPC failures are logged, not fatal. Not needed in Case A (_teardown_replicas_of_primary): the node being removed there is guaranteed to have zero LVols (enforced at remove_storage_node's entry precondition), so there is nothing to iterate. --- simplyblock_core/storage_node_ops.py | 32 +++++++++++ tests/unit/test_node_removal.py | 85 +++++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 5579fb51a3..66d51076ba 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -3942,6 +3942,37 @@ def _prune_stale_lvstore_ports(node_id, lvstore, db_controller): node.write_to_db() +def _teardown_lvol_subsystems_on_vacated_peer(peer, primary, db_controller): + """Best-effort: delete every LVol-of-``primary``'s own NVMe-oF subsystem + on ``peer`` after ``peer`` stops hosting ``primary``'s replica. + + ``_delete_replica_on_peer(..., destroy_lvstore=False)`` tears down + ``peer``'s local raid/distrib bdev stack for the lvstore, which cascades + to remove each hosted LVol's *namespace* -- but the per-LVol NVMe-oF + *subsystem and listener* (registered separately, per lvol, via + add_lvol_thread) are never touched by that teardown. Left behind, the + listener keeps accepting connections in front of a now-empty subsystem + -- exactly the "live but no path" failure test_missing_namespace_path_ + loss.py guards against, just reached by a different door: the CSI/host + initiator's own connection to peer stays live, and since peer is no + longer in lvol.nodes nothing ever tells it to drop that connection + either. The volume ends up with an extra path that looks healthy but + carries no I/O, alongside the correct new one (2026-08-18: found live + after the lvol.nodes/wrong-port fixes above -- both corrected lvols + still carried this stale-but-live third path to their pre-relocation + host). RPC failures are logged, not fatal: peer is healthy and a + lingering empty subsystem is harmless to leave for the next restart to + clear, while blocking the relocation on it is not.""" + rpc_client = peer.rpc_client() + for lvol in db_controller.get_lvols_by_node_id(primary.get_id()): + try: + rpc_client.subsystem_delete(lvol.nqn) + except RPCException as e: + logger.warning( + f"subsystem teardown for lvol {lvol.get_id()} ({lvol.nqn}) " + f"on vacated peer {peer.get_id()} failed: {e}") + + def _update_lvol_nodes_for_replica_move(primary_id, old_host_id, new_host_id, db_controller): """Re-point every LVol hosted on ``primary_id`` from ``old_host_id`` to ``new_host_id`` in its own ``nodes`` list, once that primary's @@ -4203,6 +4234,7 @@ def _relocate_replica_between(occupant_primary_id, old_host_id, new_host_id, rol # destroy_lvstore docstring. _delete_replica_on_peer(old_host, occupant_primary, cluster, destroy_lvstore=False) + _teardown_lvol_subsystems_on_vacated_peer(old_host, occupant_primary, db_controller) _prune_stale_lvstore_ports(old_host_id, occupant_primary.lvstore, db_controller) old_host = db_controller.get_storage_node_by_id(old_host_id) setattr(old_host, backref, "") diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 1a12e73c87..9c58fca5cb 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -550,10 +550,12 @@ def test_destroy_lvstore_false_never_deletes_shared_blobstore(self): # for expansion-triggered rebalancing in cluster_expansion/executor.py. # --------------------------------------------------------------------------- -def _lvol(node_id, nodes): +def _lvol(node_id, nodes, lvol_id=None, nqn=None): lv = MagicMock() lv.nodes = list(nodes) lv.write_to_db = MagicMock() + lv.get_id = MagicMock(return_value=lvol_id or f"lvol-{node_id}") + lv.nqn = nqn or f"nqn:{node_id}" return lv @@ -600,6 +602,62 @@ def test_redundant_call_is_a_safe_no_op(self): lvol.write_to_db.assert_not_called() +# --------------------------------------------------------------------------- +# _teardown_lvol_subsystems_on_vacated_peer — regression coverage for a real +# bug found live (2026-08-18): _delete_replica_on_peer(destroy_lvstore=False) +# tears down the vacated peer's raid/distrib bdev stack (which cascades to +# remove each hosted lvol's namespace), but never touches the per-lvol NVMe- +# oF subsystem+listener registered separately via add_lvol_thread. Left +# behind, the listener keeps accepting connections in front of a now-empty +# subsystem, and since the peer is no longer in lvol.nodes nothing ever +# tells the CSI/host initiator to drop that connection either -- the volume +# carries a third, live-but-empty path indefinitely alongside its correct +# two. +# --------------------------------------------------------------------------- + +class TestTeardownLvolSubsystemsOnVacatedPeer(unittest.TestCase): + + def test_deletes_subsystem_for_every_lvol_hosted_on_the_primary(self): + cl = _cluster() + primary = _node("p1", lvstore="LVS_1") + peer = _node("peer1") + rpc = peer.rpc_client() + lvols = [ + _lvol("p1", ["p1", "peer1"], lvol_id="lv-a", nqn="nqn:a"), + _lvol("p1", ["p1", "peer1"], lvol_id="lv-b", nqn="nqn:b"), + ] + db = FakeDB(cl, [primary, peer], lvols={"p1": lvols}) + storage_node_ops._teardown_lvol_subsystems_on_vacated_peer(peer, primary, db) + rpc.subsystem_delete.assert_any_call("nqn:a") + rpc.subsystem_delete.assert_any_call("nqn:b") + self.assertEqual(rpc.subsystem_delete.call_count, 2) + + def test_no_op_when_primary_hosts_no_lvols(self): + cl = _cluster() + primary = _node("p1", lvstore="LVS_1") + peer = _node("peer1") + rpc = peer.rpc_client() + db = FakeDB(cl, [primary, peer], lvols={}) + storage_node_ops._teardown_lvol_subsystems_on_vacated_peer(peer, primary, db) + rpc.subsystem_delete.assert_not_called() + + def test_rpc_failure_on_one_lvol_does_not_block_the_others(self): + # Best-effort: an RPC failure here must not propagate and must not + # stop the remaining lvols from being cleaned up. + cl = _cluster() + primary = _node("p1", lvstore="LVS_1") + peer = _node("peer1") + rpc = peer.rpc_client() + rpc.subsystem_delete.side_effect = [RPCConnectionError("connection error"), None] + lvols = [ + _lvol("p1", ["p1", "peer1"], lvol_id="lv-a", nqn="nqn:a"), + _lvol("p1", ["p1", "peer1"], lvol_id="lv-b", nqn="nqn:b"), + ] + db = FakeDB(cl, [primary, peer], lvols={"p1": lvols}) + storage_node_ops._teardown_lvol_subsystems_on_vacated_peer(peer, primary, db) # must not raise + self.assertEqual(rpc.subsystem_delete.call_count, 2) + + # --------------------------------------------------------------------------- # Case B — relocate a hosted replica # --------------------------------------------------------------------------- @@ -769,6 +827,31 @@ def test_relocate_via_splice_repoints_lvol_nodes_for_both_moved_primaries(self): self.assertEqual(occupant_lvol.nodes, ["occupant", "stranded"]) self.assertEqual(stranded_lvol.nodes, ["stranded", "x"]) + def test_relocate_via_splice_tears_down_lvol_subsystems_on_the_vacated_peer(self): + # Regression: evicting occupant's replica off x must also delete + # occupant's own lvols' NVMe-oF subsystems on x -- see + # TestTeardownLvolSubsystemsOnVacatedPeer's docstring. Must NOT run + # for stranded's own vacate-of-n1 (n1 is the node being removed, + # already shut down, not a surviving peer to clean up on). + cl = _cluster() + removed = _node("n1", stack_secondary="stranded") + stranded = _node("stranded", secondary_id="n1", lvstore="LVS_stranded") + occupant = _node("occupant", secondary_id="x", lvstore="LVS_occupant") + x = _node("x", stack_secondary="occupant") + db = FakeDB(cl, [removed, stranded, occupant, x]) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "_pick_replica_relocation_node", + return_value="x"), \ + patch.object(storage_node_ops, "recreate_lvstore_on_non_leader", + return_value=True), \ + patch.object(storage_node_ops, "_delete_replica_on_peer"), \ + patch.object(storage_node_ops, + "_teardown_lvol_subsystems_on_vacated_peer") as teardown: + ret = storage_node_ops._relocate_one_replica(removed, "stranded", "secondary") + + self.assertTrue(ret) + teardown.assert_called_once_with(x, occupant, db) + def test_relocate_via_splice_occupant_rebuild_failure_leaves_old_copy_untouched(self): # Create-before-destroy: a failed rebuild on the stranded node must # NOT tear down or repoint the occupant's still-intact old copy on x From 59d6741b3cc2a0d7a38800367baa84187ffc7fc2 Mon Sep 17 00:00:00 2001 From: wmousa Date: Wed, 19 Aug 2026 13:09:24 +0200 Subject: [PATCH 22/27] fix(node-removal): record the resolved (override-aware) name in remote_device.jm_bdev _connect_to_remote_jm_devs() already resolves the correct name to connect under for a replacement JM device: when override_name_on_node carries an entry for this_node (set by _decommission_node_devices when a node's JM gets replaced after a peer removal, so the consumer's already-built JM raid doesn't need to change), the actual NVMe-oF connect uses that override name via controller_name/expected_bdev. But remote_device.jm_bdev was still hardcoded to org_dev.jm_bdev -- the JM owner's own natural name, regardless of any override -- so the stored record didn't match what was actually connected. Found live (2026-08-19) immediately after a node removal relocated three consumers' JM redundancy onto two replacement hosts: health_controller's diagnostic controller lookup (line ~751, f'remote_{remote_device.jm_bdev}') reads this field back to query bdev_nvme_get_controllers for logging IP/port info, and with the stale natural name it queried a controller that was never created, producing a spurious SPDK "ctrlr ... does not exist" error on every health-check pass for as long as the override is in effect. Not a false health-check failure today: the actual pass/fail gate in that same function uses remote_device.remote_bdev (the real connected name), which was already correct. The damage was log noise plus a silently-empty diagnostic IP/port line -- but a field named jm_bdev that doesn't reflect the name actually connected under is a landmine for the next piece of code that trusts it (reconnect logic, another health gate, etc). Fix: resolve the override BEFORE building remote_device, and use that same resolved name for jm_bdev, expected_bdev, and controller_name alike. Added TestConnectToRemoteJmDevsRecordsResolvedName (3 tests) to tests/unit/test_node_removal.py: no-override keeps the owner's natural name, an override for this consumer is recorded correctly, and an override keyed to a different consumer doesn't leak in. Systemic note: this is a second, independent defect in the same override_name_on_node mechanism -- separate from (and not fixed by) the open issue that _create_jm_stack_on_raid/_create_jm_stack_on_device build a fresh JMDevice with no override_name_on_node carried forward, so a replacement JM host's own restart silently drops the override. That one still needs a restart to reproduce and is left for a follow-up fix. --- simplyblock_core/storage_node_ops.py | 25 +++++-- tests/unit/test_node_removal.py | 97 ++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 7 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 66d51076ba..1d73c9c215 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -2110,20 +2110,31 @@ def _connect_to_remote_jm_devs(this_node: StorageNode, jm_ids=None, only_node_id # member of the new jm_vuid. Runtime re-attach paths (rejoin, # restart-task) carry their own reachability gating. + # Resolve the name this_node actually connects under FIRST: a JM + # device that's standing in as a replacement for a removed peer's JM + # (see _decommission_node_devices) is reachable under the OLD peer's + # name for this_node specifically, not org_dev's own. remote_device. + # jm_bdev must record that resolved name, not org_dev.jm_bdev + # unconditionally -- health_controller's diagnostic controller lookup + # (f'remote_{remote_device.jm_bdev}') reads this field back and was + # querying org_dev's natural (unconnected) name every cycle, + # producing a spurious "ctrlr does not exist" SPDK error for every + # overridden slot (found live 2026-08-19 on a replacement JM host + # serving two overridden consumers at once). + resolved_name = org_dev.jm_bdev + if org_dev.override_name_on_node and this_node.get_id() in org_dev.override_name_on_node: + resolved_name = org_dev.override_name_on_node[this_node.get_id()] + remote_device = RemoteJMDevice() remote_device.uuid = org_dev.uuid remote_device.alceml_name = org_dev.alceml_name remote_device.node_id = org_dev.node_id remote_device.size = org_dev.size - remote_device.jm_bdev = org_dev.jm_bdev + remote_device.jm_bdev = resolved_name remote_device.status = NVMeDevice.STATUS_ONLINE remote_device.nvmf_multipath = org_dev.nvmf_multipath - expected_bdev = f"remote_{org_dev.jm_bdev}n1" - controller_name = f"remote_{org_dev.jm_bdev}" - if org_dev.override_name_on_node and this_node.get_id() in org_dev.override_name_on_node: - new_name = org_dev.override_name_on_node[this_node.get_id()] - expected_bdev = f"remote_{new_name}n1" - controller_name = f"remote_{new_name}" + expected_bdev = f"remote_{resolved_name}n1" + controller_name = f"remote_{resolved_name}" connect_failed = False try: remote_device.remote_bdev = str(connect_device( diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 9c58fca5cb..084c7c11bf 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -1493,6 +1493,103 @@ def test_transient_failure_does_not_block_a_clean_connect(self): self.assertEqual(result[0].remote_bdev, "remote_jm_owner_bdevn1") +# --------------------------------------------------------------------------- +# _connect_to_remote_jm_devs — remote_device.jm_bdev must record the name +# THIS NODE actually connects under, override included +# +# override_name_on_node lets a replacement JM connect under a removed peer's +# old bdev name so the consumer's own already-built JM raid doesn't need +# touching (see _decommission_node_devices). remote_device.jm_bdev must +# record that SAME resolved name, not org_dev's own natural name, or +# health_controller's diagnostic controller lookup +# (f'remote_{remote_device.jm_bdev}') queries the wrong, never-connected name +# every cycle. Found live 2026-08-19: a replacement JM host serving two +# overridden consumers logged a spurious SPDK "ctrlr ... does not exist" +# error on every health-check pass. +# --------------------------------------------------------------------------- + +class TestConnectToRemoteJmDevsRecordsResolvedName(unittest.TestCase): + + def _owner_setup(self, this_node_id="this-node", override=None): + jm_dev = JMDevice() + jm_dev.uuid = "jm-owner" + jm_dev.jm_bdev = "jm_owner_bdev" + jm_dev.status = NVMeDevice.STATUS_ONLINE + jm_dev.override_name_on_node = override or {} + + owner_node = MagicMock(spec=StorageNode) + owner_node.get_id = MagicMock(return_value="owner-node") + owner_node.status = StorageNode.STATUS_ONLINE + owner_node.jm_device = jm_dev + + this_node = MagicMock(spec=StorageNode) + this_node.get_id = MagicMock(return_value=this_node_id) + this_node.jm_ids = [] + this_node.lvstore_stack_secondary = "" + this_node.lvstore_stack_tertiary = "" + this_node.remote_jm_devices = [] + rpc_client = MagicMock() + this_node.rpc_client = MagicMock(return_value=rpc_client) + + db = MagicMock() + db.get_jm_device_by_id.return_value = jm_dev + db.get_storage_nodes.return_value = [owner_node] + + return this_node, rpc_client, db + + def test_no_override_uses_owners_own_natural_name(self): + this_node, rpc_client, db = self._owner_setup() + rpc_client.get_bdevs.return_value = {"name": "remote_jm_owner_bdevn1"} + + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "connect_device", + return_value="remote_jm_owner_bdevn1") as connect_mock: + result = storage_node_ops._connect_to_remote_jm_devs( + this_node, jm_ids=["jm-owner"]) + + self.assertEqual(len(result), 1) + self.assertEqual(result[0].jm_bdev, "jm_owner_bdev") + self.assertEqual(connect_mock.call_args[0][0], "remote_jm_owner_bdev") + + def test_override_present_records_the_override_name_not_owners_own(self): + this_node, rpc_client, db = self._owner_setup( + this_node_id="this-node", + override={"this-node": "jm_removed_peer_bdev"}) + rpc_client.get_bdevs.return_value = {"name": "remote_jm_removed_peer_bdevn1"} + + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "connect_device", + return_value="remote_jm_removed_peer_bdevn1") as connect_mock: + result = storage_node_ops._connect_to_remote_jm_devs( + this_node, jm_ids=["jm-owner"]) + + self.assertEqual(len(result), 1) + # jm_bdev must match the name actually connected under (the + # override), not org_dev's own "jm_owner_bdev" -- this is the exact + # field health_controller reads back to build its diagnostic lookup. + self.assertEqual(result[0].jm_bdev, "jm_removed_peer_bdev") + self.assertEqual(result[0].remote_bdev, "remote_jm_removed_peer_bdevn1") + self.assertEqual(connect_mock.call_args[0][0], "remote_jm_removed_peer_bdev") + + def test_override_keyed_to_a_different_node_does_not_apply(self): + # The override only applies to the specific consumer it was recorded + # for -- a DIFFERENT this_node connecting to the same owner must + # still get the owner's own natural name. + this_node, rpc_client, db = self._owner_setup( + this_node_id="this-node", + override={"some-other-node": "jm_removed_peer_bdev"}) + rpc_client.get_bdevs.return_value = {"name": "remote_jm_owner_bdevn1"} + + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "connect_device", + return_value="remote_jm_owner_bdevn1"): + result = storage_node_ops._connect_to_remote_jm_devs( + this_node, jm_ids=["jm-owner"]) + + self.assertEqual(len(result), 1) + self.assertEqual(result[0].jm_bdev, "jm_owner_bdev") + + class TestShrinkStatusDoesNotDeadlockRemoval(unittest.TestCase): """``node_removal_orchestrate`` holds ``Cluster.STATUS_IN_SHRINK`` for the duration of an attempt, so that the restart phases its replica relocation From 8dd445fbc2220ece6b0b9225762138a1e14b95d7 Mon Sep 17 00:00:00 2001 From: wmousa Date: Wed, 19 Aug 2026 13:34:42 +0200 Subject: [PATCH 23/27] fix(node-removal): retire a JM name override once its consumer rebuilds, not when the replacement host does override_name_on_node exists to keep an ALREADY-BUILT distrib/JM-raid on a consumer node pointed at a stable bdev name across a JM replacement elsewhere (a node's JM is removed, another node's JM stands in for it, but the consumer's raid member name can't be changed live -- see _decommission_node_devices). Nothing ever explicitly cleared that entry. It only ever disappeared as an accident of _create_jm_stack_on_raid / _create_jm_stack_on_device building the replacement host a brand-new JMDevice object on ITS OWN restart -- which drops override_name_on_node along with it, regardless of whether the consumer that entry names has rebuilt anything at all. That ties the override's lifetime to the wrong node's restart: - If the replacement host restarts before the consumer does, the override vanishes and the consumer's next reconnect (driven automatically by the replacement's own _reconnect_peers_to_restarted_node) switches to the replacement's natural name -- while the consumer's own distrib/raid, unchanged, still expects the old one. Live break, not a clean handoff. - If the consumer restarts first, its own full JM-set refresh and any distrib rebuild still consult the (untouched) override and keep reconnecting under the legacy name, even though it just rebuilt from scratch and had every chance to adopt the current one. Fix: _connect_to_remote_jm_devs gains drop_stale_overrides (default False, preserving existing behavior everywhere). Every call site that runs immediately ahead of the CONSUMER rebuilding its own JM-consuming construct from scratch -- node restart (_prepare_cluster_devices_on_restart), LVS recreate (recreate_lvstore_on_non_leader, _recreate_lvstore_impl), and create_lvstore (leader and secondary) -- now passes True: the override for THIS node is ignored (the current/natural name is used instead) and the stale entry is dropped from the replacement host's JMDevice via db_controller.atomic_update, so a concurrent override for a different consumer on the same replacement device is never clobbered. Every other caller (the decommission-time reconnect that freshly establishes the override, and DELTA reconnects where the consumer's own construct is unchanged) keeps the default and continues honoring it. get_node_jm_names, which builds the jm_names list actually baked into a freshly (re)created distrib, needed no change: at every rebuild call site the corresponding _connect_to_remote_jm_devs(..., drop_stale_overrides=True) call runs first in the same flow, so by the time get_node_jm_names reads the override it has already been cleared and naturally resolves to the current name. Added TestConnectToRemoteJmDevsDropsStaleOverrideOnRebuild (4 tests): a rebuild ignores the override and uses the owner's current name; the atomic_update mutate_fn removes only this consumer's entry, leaving a different consumer's override on the same device untouched; a DELTA reconnect (drop_stale_overrides left False) leaves the override and never calls atomic_update; and no-override present never calls atomic_update either. --- simplyblock_core/storage_node_ops.py | 70 ++++++++++++---- tests/unit/test_node_removal.py | 119 +++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 17 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 1d73c9c215..e4a65355d5 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -2025,7 +2025,8 @@ def _peer_reachable_via_jm_quorum(target_node_id, this_node: StorageNode, peer_p return not probed -def _connect_to_remote_jm_devs(this_node: StorageNode, jm_ids=None, only_node_id=None): +def _connect_to_remote_jm_devs(this_node: StorageNode, jm_ids=None, only_node_id=None, + drop_stale_overrides=False): """Connect ``this_node`` to remote JM devices and return the refreshed remote-JM records. @@ -2033,6 +2034,23 @@ def _connect_to_remote_jm_devs(this_node: StorageNode, jm_ids=None, only_node_id node are (re)connected; records for JMs owned by other nodes are carried over from ``this_node.remote_jm_devices`` untouched (same rationale and measurement as _connect_to_remote_devs delta mode). + + ``drop_stale_overrides``: a JM device standing in as a replacement for a + removed peer's JM (see _decommission_node_devices) is reachable, for + this_node specifically, under the OLD peer's name via + ``JMDevice.override_name_on_node`` -- kept stable so this_node's + ALREADY-BUILT distrib/JM-raid (which has that name baked in as a member) + doesn't need touching. That crutch must not outlive its purpose: once + this_node itself is about to (re)build that construct from scratch -- + restart, LVS recreate, or a brand-new create_lvstore -- there is nothing + left for the override to protect, and continuing to honor it would just + bake the stale name into the fresh build. Callers immediately ahead of + such a rebuild pass True to ignore any override for this_node and use + the JM owner's current name, clearing the stale entry as they go. Every + other caller (decommission-time reconnect, where the override is being + freshly established, or a DELTA reconnect where this_node's own + construct is unchanged) must leave the default False so the override is + honored. """ db_controller = DBController() @@ -2110,20 +2128,38 @@ def _connect_to_remote_jm_devs(this_node: StorageNode, jm_ids=None, only_node_id # member of the new jm_vuid. Runtime re-attach paths (rejoin, # restart-task) carry their own reachability gating. - # Resolve the name this_node actually connects under FIRST: a JM - # device that's standing in as a replacement for a removed peer's JM - # (see _decommission_node_devices) is reachable under the OLD peer's - # name for this_node specifically, not org_dev's own. remote_device. - # jm_bdev must record that resolved name, not org_dev.jm_bdev + # Resolve the name this_node actually connects under. remote_device. + # jm_bdev must record this resolved name, not org_dev.jm_bdev # unconditionally -- health_controller's diagnostic controller lookup # (f'remote_{remote_device.jm_bdev}') reads this field back and was - # querying org_dev's natural (unconnected) name every cycle, - # producing a spurious "ctrlr does not exist" SPDK error for every - # overridden slot (found live 2026-08-19 on a replacement JM host - # serving two overridden consumers at once). + # querying org_dev's natural (unconnected) name every cycle whenever + # an override applied, producing a spurious "ctrlr does not exist" + # SPDK error on every health-check pass (found live 2026-08-19 on a + # replacement JM host serving two overridden consumers at once). + override_applies = bool( + org_dev.override_name_on_node + and this_node.get_id() in org_dev.override_name_on_node) resolved_name = org_dev.jm_bdev - if org_dev.override_name_on_node and this_node.get_id() in org_dev.override_name_on_node: + if override_applies and not drop_stale_overrides: resolved_name = org_dev.override_name_on_node[this_node.get_id()] + elif override_applies and drop_stale_overrides and org_dev_node is not None: + # this_node is about to (re)build the very construct the + # override was protecting -- nothing left to protect, so drop it + # and settle on org_dev's own current name (resolved_name above) + # going forward. See the docstring for why this is scoped to + # callers that pass drop_stale_overrides=True. + consumer_id = this_node.get_id() + + def _drop_stale_override(n, consumer_id=consumer_id): + if n.jm_device and consumer_id in (n.jm_device.override_name_on_node or {}): + del n.jm_device.override_name_on_node[consumer_id] + + try: + db_controller.atomic_update(org_dev_node, _drop_stale_override) + except Exception as e: + logger.warning( + f"Failed to drop stale JM name override for " + f"{consumer_id} on {org_dev_node.get_id()}: {e}") remote_device = RemoteJMDevice() remote_device.uuid = org_dev.uuid @@ -3294,7 +3330,7 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, if snode.enable_ha_jm: logger.info("Connecting to remote JMs") - snode.remote_jm_devices = _connect_to_remote_jm_devs(snode) + snode.remote_jm_devices = _connect_to_remote_jm_devs(snode, drop_stale_overrides=True) snode.write_to_db(kv_store) @@ -5261,7 +5297,7 @@ def _restart_storage_node_impl( def _jm_reconcile(): try: - jm_result["devices"] = _connect_to_remote_jm_devs(snode) + jm_result["devices"] = _connect_to_remote_jm_devs(snode, drop_stale_overrides=True) except Exception as e: jm_result["error"] = e @@ -8030,7 +8066,7 @@ def _recreate_lvstore_on_non_leader_impl(snode: StorageNode, leader_node, primar logger.warning("Soft reconnect of remote devices failed on %s: %s", snode.get_id(), e) try: - fresh_remote_jms = _connect_to_remote_jm_devs(snode) + fresh_remote_jms = _connect_to_remote_jm_devs(snode, drop_stale_overrides=True) snode = db_controller.get_storage_node_by_id(snode.get_id()) snode.remote_jm_devices = fresh_remote_jms or snode.remote_jm_devices snode.write_to_db() @@ -9059,7 +9095,7 @@ def _recreate_lvstore_impl(snode: StorageNode, force=False, lvs_primary=None, ac if not is_takeover: snode = db_controller.get_storage_node_by_id(snode.get_id()) - snode.remote_jm_devices = _connect_to_remote_jm_devs(snode) + snode.remote_jm_devices = _connect_to_remote_jm_devs(snode, drop_stale_overrides=True) snode.write_to_db() # Gather peer nodes for this LVS, EXCLUDING snode itself @@ -10743,7 +10779,7 @@ def create_lvstore(snode: StorageNode, ndcs, npcs, distr_bs, distr_chunk_bs, pag jm_vuid = utils.get_random_vuid() jm_ids = get_sorted_ha_jms(snode) logger.debug(f"online_jms: {str(jm_ids)}") - snode.remote_jm_devices = _connect_to_remote_jm_devs(snode, jm_ids) + snode.remote_jm_devices = _connect_to_remote_jm_devs(snode, jm_ids, drop_stale_overrides=True) snode.jm_ids = jm_ids snode.jm_vuid = jm_vuid snode.write_to_db() @@ -10878,7 +10914,7 @@ def create_lvstore(snode: StorageNode, ndcs, npcs, distr_bs, distr_chunk_bs, pag sec_node.lvstore_ports[lvs_name] = snode.lvstore_ports[lvs_name].copy() # creating lvstore on secondary - sec_node.remote_jm_devices = _connect_to_remote_jm_devs(sec_node) + sec_node.remote_jm_devices = _connect_to_remote_jm_devs(sec_node, drop_stale_overrides=True) sec_node.write_to_db() ret, err = _create_bdev_stack(sec_node, lvstore_stack, primary_node=snode) if err: diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 084c7c11bf..08397af0b7 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -1590,6 +1590,125 @@ def test_override_keyed_to_a_different_node_does_not_apply(self): self.assertEqual(result[0].jm_bdev, "jm_owner_bdev") +# --------------------------------------------------------------------------- +# _connect_to_remote_jm_devs — drop_stale_overrides retires the legacy-name +# crutch once this_node itself rebuilds its own JM-consuming construct +# +# override_name_on_node exists only to keep an ALREADY-BUILT distrib/JM-raid +# on this_node pointed at a stable name across a JM replacement elsewhere. +# Once this_node is about to (re)build that construct from scratch -- +# restart, LVS recreate, or a brand-new create_lvstore, all of which call +# with drop_stale_overrides=True immediately ahead of the rebuild -- there +# is nothing left for the override to protect, so it must be dropped rather +# than baked into the fresh build. A DELTA reconnect (drop_stale_overrides +# left False, e.g. a peer reconnecting to a node that just restarted) must +# keep honoring it: this_node's own construct did not change. +# --------------------------------------------------------------------------- + +class TestConnectToRemoteJmDevsDropsStaleOverrideOnRebuild(unittest.TestCase): + + def _owner_setup(self, this_node_id="this-node", override=None): + jm_dev = JMDevice() + jm_dev.uuid = "jm-owner" + jm_dev.jm_bdev = "jm_owner_bdev" + jm_dev.status = NVMeDevice.STATUS_ONLINE + jm_dev.override_name_on_node = override or {} + + owner_node = MagicMock(spec=StorageNode) + owner_node.get_id = MagicMock(return_value="owner-node") + owner_node.status = StorageNode.STATUS_ONLINE + owner_node.jm_device = jm_dev + + this_node = MagicMock(spec=StorageNode) + this_node.get_id = MagicMock(return_value=this_node_id) + this_node.jm_ids = [] + this_node.lvstore_stack_secondary = "" + this_node.lvstore_stack_tertiary = "" + this_node.remote_jm_devices = [] + rpc_client = MagicMock() + this_node.rpc_client = MagicMock(return_value=rpc_client) + + db = MagicMock() + db.get_jm_device_by_id.return_value = jm_dev + db.get_storage_nodes.return_value = [owner_node] + + return this_node, rpc_client, db, owner_node + + def test_rebuild_ignores_the_override_and_uses_the_owners_current_name(self): + this_node, rpc_client, db, owner_node = self._owner_setup( + override={"this-node": "jm_removed_peer_bdev"}) + rpc_client.get_bdevs.return_value = {"name": "remote_jm_owner_bdevn1"} + + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "connect_device", + return_value="remote_jm_owner_bdevn1") as connect_mock: + result = storage_node_ops._connect_to_remote_jm_devs( + this_node, jm_ids=["jm-owner"], drop_stale_overrides=True) + + self.assertEqual(len(result), 1) + # Uses the owner's own current name, NOT the stale override -- + # this_node is rebuilding its own construct right now, so there is + # nothing left for the legacy name to protect. + self.assertEqual(result[0].jm_bdev, "jm_owner_bdev") + self.assertEqual(connect_mock.call_args[0][0], "remote_jm_owner_bdev") + + def test_rebuild_clears_the_override_entry_via_atomic_update(self): + this_node, rpc_client, db, owner_node = self._owner_setup( + override={"this-node": "jm_removed_peer_bdev", "other-node": "jm_something_else"}) + rpc_client.get_bdevs.return_value = {"name": "remote_jm_owner_bdevn1"} + + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "connect_device", + return_value="remote_jm_owner_bdevn1"): + storage_node_ops._connect_to_remote_jm_devs( + this_node, jm_ids=["jm-owner"], drop_stale_overrides=True) + + db.atomic_update.assert_called_once() + target, mutate_fn = db.atomic_update.call_args.args[:2] + self.assertIs(target, owner_node) + + # The mutate_fn only removes THIS consumer's entry from a fresh copy + # -- it must not clear another consumer's still-live override. + fresh_jm_dev = JMDevice() + fresh_jm_dev.override_name_on_node = { + "this-node": "jm_removed_peer_bdev", "other-node": "jm_something_else"} + fresh_owner = MagicMock(spec=StorageNode) + fresh_owner.jm_device = fresh_jm_dev + mutate_fn(fresh_owner) + self.assertEqual(fresh_jm_dev.override_name_on_node, {"other-node": "jm_something_else"}) + + def test_delta_reconnect_leaves_the_override_untouched(self): + # only_node_id set (a peer reconnecting after ITS restart) with + # drop_stale_overrides left at its default False: this_node's own + # construct hasn't changed, so the override must still be honored + # and nothing should be cleared. + this_node, rpc_client, db, owner_node = self._owner_setup( + override={"this-node": "jm_removed_peer_bdev"}) + rpc_client.get_bdevs.return_value = {"name": "remote_jm_removed_peer_bdevn1"} + + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "connect_device", + return_value="remote_jm_removed_peer_bdevn1"): + result = storage_node_ops._connect_to_remote_jm_devs( + this_node, jm_ids=["jm-owner"], only_node_id="owner-node") + + self.assertEqual(len(result), 1) + self.assertEqual(result[0].jm_bdev, "jm_removed_peer_bdev") + db.atomic_update.assert_not_called() + + def test_no_override_present_never_calls_atomic_update(self): + this_node, rpc_client, db, owner_node = self._owner_setup() + rpc_client.get_bdevs.return_value = {"name": "remote_jm_owner_bdevn1"} + + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "connect_device", + return_value="remote_jm_owner_bdevn1"): + storage_node_ops._connect_to_remote_jm_devs( + this_node, jm_ids=["jm-owner"], drop_stale_overrides=True) + + db.atomic_update.assert_not_called() + + class TestShrinkStatusDoesNotDeadlockRemoval(unittest.TestCase): """``node_removal_orchestrate`` holds ``Cluster.STATUS_IN_SHRINK`` for the duration of an attempt, so that the restart phases its replica relocation From d073ba479dee9631a737452bf9b796812c206e4e Mon Sep 17 00:00:00 2001 From: wmousa Date: Wed, 19 Aug 2026 13:40:42 +0200 Subject: [PATCH 24/27] fix(node-removal): chain a consumer's inherited JM name through a second removal, not the removed node's own name _decommission_node_devices' replacement-picking branch always handed the newly-picked replacement JM device the REMOVED node's own natural jm_bdev name (removed_node.jm_device.jm_bdev) as the override for the affected consumer. That's only correct when removed_node was never itself standing in as an override for that consumer. It breaks on a two-hop chain with no restart in between: node A removed, node C picked as consumer B's replacement JM under override name "jm_A" (B's still-unrebuilt raid construct references that legacy name, never C's own). If C is then removed before B -- or C -- ever restarts, _decommission_node_devices(C) finds B still carrying C's JM id in B.jm_ids, picks a fresh replacement D, and previously handed D the override name "jm_C" -- C's own natural name, which B's construct never referenced at all (B only ever connected to C under "jm_A"). D would end up serving a name B never uses and never will, silently breaking that redundancy slot with no restart required to trigger it. Fix: check removed_node.jm_device.override_name_on_node for the consumer first, and chain that inherited name forward; only fall back to removed_node's own jm_bdev when removed_node was never itself a stand-in for this consumer (the normal, single-hop case, unchanged). Added two tests to TestDecommissionDevices: the baseline single-hop case (no prior chain -- replacement inherits removed_node's own name, as before) and the two-hop chain (replacement inherits the older "jm_A" name, not removed_node's "jm_n1"). Neither test previously existed for this branch at all -- the main replacement-picking path had no coverage of what name actually gets recorded. --- simplyblock_core/storage_node_ops.py | 14 +++++- tests/unit/test_node_removal.py | 65 ++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index e4a65355d5..d5c1252360 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4336,7 +4336,19 @@ def _decommission_node_devices(removed_node: StorageNode): if new_jm_dev: d = db_controller.get_jm_device_by_id(new_jm_dev) jm_node = db_controller.get_storage_node_by_id(d.node_id) - jm_node.jm_device.override_name_on_node[node.get_id()]=removed_node.jm_device.jm_bdev + # Chain the name node actually needs, not removed_node's + # own natural name: removed_node may itself have been + # standing in as an override for node (a second removal, + # no restart in between -- e.g. A removed, C picked as + # node's replacement JM under the name "jm_A", then C + # itself removed before anyone restarted). In that case + # node's still-unrebuilt construct references THAT + # older name, never removed_node's own -- propagate it + # forward instead of jumping to a name node never used. + inherited_name = (removed_node.jm_device.override_name_on_node or {}).get( + node.get_id()) + jm_node.jm_device.override_name_on_node[node.get_id()] = ( + inherited_name or removed_node.jm_device.jm_bdev) jm_node.write_to_db() node.jm_ids.append(new_jm_dev) node.remote_jm_devices = _connect_to_remote_jm_devs(node, node.jm_ids) diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 08397af0b7..8c98f00a80 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -1272,6 +1272,71 @@ def test_does_not_refresh_peer_with_no_dead_jm_reference_at_all(self): connect_mock.assert_not_called() peer.write_to_db.assert_not_called() + def test_picks_replacement_and_records_override_with_no_prior_chain(self): + # Baseline (no prior chain): removed node was never itself an + # override stand-in, so the new replacement inherits removed_node's + # OWN natural jm_bdev, same as always. + cl = _cluster() + removed = _node("n1", n_devices=0, with_jm=True) + removed.jm_ids = [] + consumer = _node("consumer", n_devices=0, with_jm=True) + consumer.jm_ids = [removed.jm_device.get_id()] + replacement = _node("replacement", n_devices=0, with_jm=True) + replacement.jm_ids = [] + db = FakeDB(cl, [removed, consumer, replacement]) + db.get_jm_device_by_id = MagicMock(side_effect=lambda jid: { + removed.jm_device.get_id(): removed.jm_device, + replacement.jm_device.get_id(): replacement.jm_device, + }[jid]) + dc = MagicMock() + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "device_controller", dc), \ + patch.object(storage_node_ops, "get_sorted_ha_jms", + return_value=[replacement.jm_device.get_id()]), \ + patch.object(storage_node_ops, "_connect_to_remote_jm_devs", return_value=[]): + ret = storage_node_ops._decommission_node_devices(removed) + + self.assertTrue(ret) + self.assertEqual( + replacement.jm_device.override_name_on_node.get("consumer"), + removed.jm_device.jm_bdev) + replacement.write_to_db.assert_called() + + def test_removed_node_was_itself_an_override_stand_in_chains_the_original_name(self): + # Two-hop chain, no restart in between: A removed, "removed" (here + # playing C) was picked as consumer's replacement JM under the + # override name "jm_A" -- consumer's still-unrebuilt raid construct + # references THAT name, never "removed"'s own. "removed" is now + # itself removed before consumer (or "removed") ever restarted -- + # the next replacement must inherit "jm_A", not removed's own + # natural name, which consumer's construct never referenced. + cl = _cluster() + removed = _node("n1", n_devices=0, with_jm=True) + removed.jm_ids = [] + removed.jm_device.override_name_on_node = {"consumer": "jm_A"} + consumer = _node("consumer", n_devices=0, with_jm=True) + consumer.jm_ids = [removed.jm_device.get_id()] + replacement = _node("replacement", n_devices=0, with_jm=True) + replacement.jm_ids = [] + db = FakeDB(cl, [removed, consumer, replacement]) + db.get_jm_device_by_id = MagicMock(side_effect=lambda jid: { + removed.jm_device.get_id(): removed.jm_device, + replacement.jm_device.get_id(): replacement.jm_device, + }[jid]) + dc = MagicMock() + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "device_controller", dc), \ + patch.object(storage_node_ops, "get_sorted_ha_jms", + return_value=[replacement.jm_device.get_id()]), \ + patch.object(storage_node_ops, "_connect_to_remote_jm_devs", return_value=[]): + ret = storage_node_ops._decommission_node_devices(removed) + + self.assertTrue(ret) + # Inherits "jm_A" -- NOT removed.jm_device.jm_bdev ("jm_n1"). + self.assertEqual( + replacement.jm_device.override_name_on_node.get("consumer"), "jm_A") + replacement.write_to_db.assert_called() + # --------------------------------------------------------------------------- # node_removal_orchestrate — phase-5 resume gap From 8615e8587744635abcbca28cc4cbd3512363e322 Mon Sep 17 00:00:00 2001 From: wmousa Date: Wed, 19 Aug 2026 14:31:33 +0200 Subject: [PATCH 25/27] fix(node-removal): never pick a replacement JM the consumer already reaches by another path SPDK will not attach a second, distinctly-named local controller to a target it already has a live connection to under a different name. The override_name_on_node mechanism (a consumer keeps its already-built JM raid/journal member pointed at a legacy name after that name's owner is replaced) implicitly assumed the opposite: that a fresh alias to the replacement's target could always be attached. That assumption breaks whenever the consumer already reaches the picked replacement through some OTHER, unrelated path -- most commonly: the consumer hosts some OTHER primary's secondary/tertiary lvstore copy, and that other primary's own jm_ids set already includes the same replacement node. The consumer then has a live connection to the replacement's JM subsystem under the replacement's own natural name before the override was ever set. When _connect_to_remote_jm_devs later tries to attach the SAME target under the override's legacy alias, the attach RPC returns without a bdev name; _connect_to_remote_jm_devs' own fallback then silently reuses the pre-existing connection's name for remote_bdev while jm_bdev stays at the never-connected override name, and the DB ends up recording a healthy redundancy slot that was never actually live. Found live (2026-08-19), immediately after removing a node with zero lvols (the simplest possible removal): on the affected consumer, SPDK reported "Bdev name not returned from controller attach" for the override name, "Controller ... does not exist" on a direct query, and the JC layer was in an active retry loop -- "helper_sync_setter ... failed, JM is excluded from further operation" -- every few seconds, ongoing. Fix: when picking a replacement in _decommission_node_devices, skip any candidate the consumer already reaches via ANY path -- not just its own jm_ids -- by checking node.remote_jm_devices before picking. A candidate that's already connected can never serve as an override stand-in for this consumer, so it's never a valid pick regardless of how get_sorted_ha_jms ranks it. Also fixed the test fixture that surfaced this: tests/unit/test_node_ removal.py's _node() helper built each JMDevice() bare, and BaseModel. from_dict() binds a dict-typed field's default to the CLASS-level object itself when the field is absent from the constructor's data -- every bare-built JMDevice ends up sharing ONE override_name_on_node dict until something reassigns it. Production is unaffected (every DB read round-trips through from_dict(data) with the key present, which always constructs a fresh dict via dict(data[attr])), but the bare-constructed test fixtures aren't, and an in-place mutation in one test (exactly what the decommission code does) was bleeding into unrelated later tests via the shared class default. _node() now gives each JMDevice its own dict explicitly. Added a regression test: a "colliding" candidate already in the consumer's remote_jm_devices (but never in its own jm_ids) is skipped in favor of the next, genuinely unreached candidate. --- simplyblock_core/storage_node_ops.py | 21 ++++++++++- tests/unit/test_node_removal.py | 56 ++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index d5c1252360..68b8249944 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4328,9 +4328,28 @@ def _decommission_node_devices(removed_node: StorageNode): node.jm_ids.remove(removed_node.jm_device.get_id()) jm_ids = get_sorted_ha_jms(node) logger.debug(f"online_jms: {str(jm_ids)}") + # A candidate node already reaches via some path OTHER than + # its own jm_ids -- most commonly the hosted-primary route: + # node hosts some OTHER primary's secondary/tertiary copy, + # and that primary's own jm_ids already includes this + # candidate -- can't serve as an override stand-in here. + # SPDK won't attach a second, distinctly-named local + # controller to a target it already has a live connection + # to under another name, so the override name would never + # actually connect: the attach RPC returns without a bdev + # name, _connect_to_remote_jm_devs' fallback silently reuses + # the OTHER (pre-existing) connection's name for remote_bdev, + # and the DB ends up claiming a healthy override that was + # never live (found live 2026-08-19: "Bdev name not returned + # from controller attach", the JM group "excluded from + # further operation" on an ongoing SPDK retry loop, while + # the DB metadata reported success). Skip any candidate + # node already reaches, however it reaches it, so a fresh + # pick never collides this way. + already_reachable = {rd.uuid for rd in (node.remote_jm_devices or [])} new_jm_dev = "" for jm_id in jm_ids: - if jm_id not in node.jm_ids: + if jm_id not in node.jm_ids and jm_id not in already_reachable: new_jm_dev = jm_id break if new_jm_dev: diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 8c98f00a80..91a1862588 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -86,6 +86,16 @@ def _node(node_id, status=StorageNode.STATUS_ONLINE, lvstore="", jm.uuid = f"jm-{node_id}" jm.node_id = node_id jm.status = JMDevice.STATUS_ONLINE + # BaseModel.from_dict() binds a bare JMDevice()'s dict-typed + # defaults to the CLASS-level object itself (setattr(self, attr, + # getattr(self, attr)) when the attr is absent from data) -- every + # JMDevice() built this way shares ONE override_name_on_node dict + # until something reassigns it. Production is unaffected (every DB + # read round-trips through from_dict(data) with the key present, + # which always constructs a fresh dict), but bare-constructed test + # fixtures aren't -- give each one its own so an in-place mutation + # in one test can't bleed into another via the shared class default. + jm.override_name_on_node = {} n.jm_device = jm else: n.jm_device = None @@ -1337,6 +1347,52 @@ def test_removed_node_was_itself_an_override_stand_in_chains_the_original_name(s replacement.jm_device.override_name_on_node.get("consumer"), "jm_A") replacement.write_to_db.assert_called() + def test_skips_a_candidate_the_consumer_already_reaches_via_another_path(self): + # SPDK won't attach a second, distinctly-named local controller to a + # target the consumer already has a live connection to under + # another name (found live 2026-08-19: "Bdev name not returned from + # controller attach", the JM group excluded from further operation + # on an ongoing SPDK retry loop, while the DB metadata claimed the + # override connected fine). "colliding" is already in + # consumer.remote_jm_devices (e.g. reached via hosting some OTHER + # primary's secondary copy) even though it was never in + # consumer.jm_ids -- it must be skipped in favor of "clean", the + # next candidate that isn't already reachable by any path. + cl = _cluster() + removed = _node("n1", n_devices=0, with_jm=True) + removed.jm_ids = [] + consumer = _node("consumer", n_devices=0, with_jm=True) + consumer.jm_ids = [removed.jm_device.get_id()] + colliding = _node("colliding", n_devices=0, with_jm=True) + colliding.jm_ids = [] + already_connected = RemoteJMDevice() + already_connected.uuid = colliding.jm_device.get_id() + already_connected.remote_bdev = "remote_jm_colliding-own-namen1" + consumer.remote_jm_devices = [already_connected] + clean = _node("clean", n_devices=0, with_jm=True) + clean.jm_ids = [] + db = FakeDB(cl, [removed, consumer, colliding, clean]) + db.get_jm_device_by_id = MagicMock(side_effect=lambda jid: { + removed.jm_device.get_id(): removed.jm_device, + colliding.jm_device.get_id(): colliding.jm_device, + clean.jm_device.get_id(): clean.jm_device, + }[jid]) + dc = MagicMock() + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "device_controller", dc), \ + patch.object(storage_node_ops, "get_sorted_ha_jms", + return_value=[colliding.jm_device.get_id(), clean.jm_device.get_id()]), \ + patch.object(storage_node_ops, "_connect_to_remote_jm_devs", return_value=[]): + ret = storage_node_ops._decommission_node_devices(removed) + + self.assertTrue(ret) + self.assertEqual(colliding.jm_device.override_name_on_node, {}) + self.assertEqual( + clean.jm_device.override_name_on_node.get("consumer"), + removed.jm_device.jm_bdev) + clean.write_to_db.assert_called() + colliding.write_to_db.assert_not_called() + # --------------------------------------------------------------------------- # node_removal_orchestrate — phase-5 resume gap From 8bd5ed5fce46ad927ef9f4783c87a1370d995171 Mon Sep 17 00:00:00 2001 From: wmousa Date: Wed, 19 Aug 2026 15:26:18 +0200 Subject: [PATCH 26/27] fix(node-removal): fall back to a colliding JM candidate instead of leaving the slot permanently unfilled _decommission_node_devices' replacement pick, after the collision-avoidance fix, could come up empty when every remaining candidate collides (the consumer already reaches it via some other path). The only handling for "no candidate" was logger.error(f"no jm_id found for {node.get_id()}") -- nothing else. That's worse than it looks: - node.write_to_db() only ran inside the "found a candidate" branch, so the node.jm_ids.remove(removed_node's dead id) a few lines earlier -- which DID run unconditionally -- was never persisted either. The DB kept referencing a JM device that no longer exists, forever. - Nothing anywhere else revisits or retries this later: no reconciler, no follow-up task. The slot stays silently short one redundancy member until node itself happens to restart. - It's invisible in normal tooling: a bare logger.error() never becomes a cluster event (unlike e.g. device_status errors, which go through storage_events.*() and show up in `sbctl cluster get-logs`). Fix: when no collision-free candidate exists, fall back to a colliding one rather than leaving the slot empty. This won't actually connect until node's own next restart -- that restart's own full-refresh reconnect (_connect_to_remote_jm_devs' drop_stale_overrides=True, from the prior fix in this area) drops this exact stale entry and reconnects under the name that's already live, self-healing cleanly. Until then it's a visible, ongoing SPDK JC retry/exclude loop instead of an invisible, permanent gap -- visible-and-self-healing beats invisible-and-permanent. Also moved node.write_to_db() to run unconditionally after the if/else, so the .remove() is persisted even in the genuinely-empty case (no candidate at all, not even a colliding one) instead of being silently discarded. Added two tests: falling back to a colliding candidate when no clean one exists (sets the override, appends to jm_ids, persists both writes), and persisting the .remove() even when there's no candidate whatsoever. --- simplyblock_core/storage_node_ops.py | 35 ++++++++++++++- tests/unit/test_node_removal.py | 65 ++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 68b8249944..8a4594d170 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4348,10 +4348,36 @@ def _decommission_node_devices(removed_node: StorageNode): # pick never collides this way. already_reachable = {rd.uuid for rd in (node.remote_jm_devices or [])} new_jm_dev = "" + fallback_jm_dev = "" for jm_id in jm_ids: - if jm_id not in node.jm_ids and jm_id not in already_reachable: + if jm_id in node.jm_ids: + continue + if jm_id not in already_reachable: new_jm_dev = jm_id break + elif not fallback_jm_dev: + fallback_jm_dev = jm_id + if not new_jm_dev and fallback_jm_dev: + # No collision-free candidate at all -- rather than + # leave this redundancy slot permanently and silently + # short (the alternative: log an error and walk away, + # which never gets revisited by anything), accept the + # colliding one. It won't actually connect until node + # itself next restarts -- that restart's own full + # refresh (drop_stale_overrides=True) drops this exact + # override and reconnects under the name that's already + # live, self-healing cleanly -- but until then this is a + # visible, ongoing SPDK JC retry/exclude loop rather than + # an invisible, permanent gap. Visible-and-self-healing + # beats invisible-and-permanent. + logger.warning( + f"No collision-free JM candidate for {node.get_id()}; " + f"falling back to {fallback_jm_dev}, which node " + f"already reaches via another path. This override " + f"will not actually connect until node's own next " + f"restart clears it and reconnects under the correct " + f"name.") + new_jm_dev = fallback_jm_dev if new_jm_dev: d = db_controller.get_jm_device_by_id(new_jm_dev) jm_node = db_controller.get_storage_node_by_id(d.node_id) @@ -4371,9 +4397,14 @@ def _decommission_node_devices(removed_node: StorageNode): jm_node.write_to_db() node.jm_ids.append(new_jm_dev) node.remote_jm_devices = _connect_to_remote_jm_devs(node, node.jm_ids) - node.write_to_db() else: + # Truly nothing available (not even a colliding + # candidate) -- still persist the .remove() above rather + # than silently dropping it: an explicitly short jm_ids + # is a strictly more honest state than one that still + # references a JM device that no longer exists. logger.error(f"no jm_id found for {node.get_id()}") + node.write_to_db() elif any(d.uuid == removed_node.jm_device.get_id() for d in (node.remote_jm_devices or [])): # node.jm_ids is this node's OWN redundancy set for its OWN JM # -- but _connect_to_remote_jm_devs also connects to a SECOND diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 91a1862588..07aa376a78 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -1393,6 +1393,71 @@ def test_skips_a_candidate_the_consumer_already_reaches_via_another_path(self): clean.write_to_db.assert_called() colliding.write_to_db.assert_not_called() + def test_falls_back_to_a_colliding_candidate_when_no_clean_one_exists(self): + # No collision-free candidate anywhere -- rather than leave the + # slot permanently and silently short, accept the colliding one. + # It won't actually connect until consumer's own next restart (that + # restart's drop_stale_overrides=True refresh cleans this exact + # entry up and reconnects correctly), but a visible, self-healing + # degraded state beats an invisible, permanent gap. + cl = _cluster() + removed = _node("n1", n_devices=0, with_jm=True) + removed.jm_ids = [] + consumer = _node("consumer", n_devices=0, with_jm=True) + consumer.jm_ids = [removed.jm_device.get_id()] + colliding = _node("colliding", n_devices=0, with_jm=True) + colliding.jm_ids = [] + already_connected = RemoteJMDevice() + already_connected.uuid = colliding.jm_device.get_id() + already_connected.remote_bdev = "remote_jm_colliding-own-namen1" + consumer.remote_jm_devices = [already_connected] + db = FakeDB(cl, [removed, consumer, colliding]) + db.get_jm_device_by_id = MagicMock(side_effect=lambda jid: { + removed.jm_device.get_id(): removed.jm_device, + colliding.jm_device.get_id(): colliding.jm_device, + }[jid]) + dc = MagicMock() + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "device_controller", dc), \ + patch.object(storage_node_ops, "get_sorted_ha_jms", + return_value=[colliding.jm_device.get_id()]), \ + patch.object(storage_node_ops, "_connect_to_remote_jm_devs", return_value=[]): + ret = storage_node_ops._decommission_node_devices(removed) + + self.assertTrue(ret) + self.assertEqual( + colliding.jm_device.override_name_on_node.get("consumer"), + removed.jm_device.jm_bdev) + self.assertIn(colliding.jm_device.get_id(), consumer.jm_ids) + colliding.write_to_db.assert_called() + consumer.write_to_db.assert_called() + + def test_no_candidate_at_all_still_persists_the_removed_jm_id(self): + # Regression guard for a real bug found while explaining this + # branch: the ONLY prior action, "no jm_id found" -> logger.error, + # never called node.write_to_db(). The node.jm_ids.remove() earlier + # in this same code path was therefore silently discarded -- the DB + # kept referencing a JM device that no longer exists, forever, with + # nothing left to show even that removal was attempted. An + # explicitly short jm_ids is a strictly more honest persisted state. + cl = _cluster() + removed = _node("n1", n_devices=0, with_jm=True) + removed.jm_ids = [] + consumer = _node("consumer", n_devices=0, with_jm=True) + consumer.jm_ids = [removed.jm_device.get_id()] + db = FakeDB(cl, [removed, consumer]) + dc = MagicMock() + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "device_controller", dc), \ + patch.object(storage_node_ops, "get_sorted_ha_jms", return_value=[]), \ + patch.object(storage_node_ops, "_connect_to_remote_jm_devs") as connect_mock: + ret = storage_node_ops._decommission_node_devices(removed) + + self.assertTrue(ret) + self.assertNotIn(removed.jm_device.get_id(), consumer.jm_ids) + consumer.write_to_db.assert_called() + connect_mock.assert_not_called() + # --------------------------------------------------------------------------- # node_removal_orchestrate — phase-5 resume gap From 04541427a090bd5d896414cd38dcb0a0fb0e17a4 Mon Sep 17 00:00:00 2001 From: wmousa Date: Fri, 21 Aug 2026 15:21:53 +0200 Subject: [PATCH 27/27] feat(node-removal): replace the JM name-override hack with SPDK's jc_replace_jm Retire override_name_on_node entirely. It existed only because there was no way to swap a JM device backing a live JC (journal consistency) member without rebuilding the consumer's already-built distrib/JM-raid construct -- so a replacement JM was instead forced to answer under the removed peer's OLD bdev name, faking continuity. That naming trick was the root cause of five separate bugs fixed on this branch: the resolved name not being recorded on remote_device.jm_bdev, the override never being retired when its consumer rebuilt, the override not chaining across a second removal, a colliding candidate silently producing a phantom "connected" override that was never actually live, and a missing candidate silently dropping the node.jm_ids.remove() that should have been persisted. SPDK now ships jc_replace_jm(name_old, name_new): it swaps a live JC member in place and re-syncs the new JM's journal in the background. This removes the entire reason the naming trick existed: - _connect_to_remote_jm_devs always connects under the JM owner's own natural name now. Its drop_stale_overrides parameter (and the 6 call sites that passed it) is gone -- there's nothing left to drop. - _decommission_node_devices connects a picked replacement under its own name, then calls jc_replace_jm(name_old, name_new) on the consumer's own SPDK, where name_old is read back from the consumer's own remote_jm_devices record (already the resolved name, whatever it is -- no more chaining logic needed to guess it). - The collision-avoidance pick (skip a candidate the consumer already reaches via another path) stays, since jc_replace_jm rejects a name_new already in use by JC (-14) for the same underlying reason SPDK could never attach a second connection to it. But the old "fall back to a colliding candidate anyway, self-heal on the consumer's next restart" path is gone: jc_replace_jm would just reject it outright, so there's no point attempting it. No collision-free candidate now leaves the redundancy slot honestly short instead. - A jc_replace_jm failure (any code) leaves the slot short and best-effort detaches the now-unused connection it just made -- except for -14, where the bdev is legitimately already claimed by JC and detaching it would tear down something in active use. - get_node_jm_names() and JMDevice.override_name_on_node are gone; a node's jm_ids now always reflects the live JC membership directly. No mixed-version rollout path is needed: this assumes jc_replace_jm is available on every node's SPDK by the time this code runs. --- simplyblock_core/models/nvme_device.py | 3 - simplyblock_core/rpc_client.py | 25 ++ simplyblock_core/storage_node_ops.py | 231 +++++++------- tests/unit/test_node_removal.py | 414 +++++++++++-------------- 4 files changed, 309 insertions(+), 364 deletions(-) diff --git a/simplyblock_core/models/nvme_device.py b/simplyblock_core/models/nvme_device.py index e36a80c700..4c82223f13 100644 --- a/simplyblock_core/models/nvme_device.py +++ b/simplyblock_core/models/nvme_device.py @@ -128,9 +128,6 @@ class JMDevice(NVMeDevice): # the per-leg member partitions. Empty for single-device (no-raid) JMs. jm_leg_bdevs: List[str] = [] jm_leg_members: List = [] - # When attaching this JM to a node, override the device name on that node. - # This is needed when a JM device is removed and needed to be replaced, but the name must be the same. - override_name_on_node: dict[str, str] = {} # node_id: new_name class RemoteDevice(BaseModel): diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 1bba9177e3..04fc1a2dae 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1418,6 +1418,31 @@ def jc_explicit_synchronization(self, jm_vuid): } return self._request("jc_explicit_synchronization", params) + def jc_replace_jm(self, name_old: str, name_new: str): + """Swap the JM bdev backing a live JC member from ``name_old`` to + ``name_new`` in place -- JC re-syncs the new JM's journal in the + background and, from then on, treats it as the member for this slot. + Replaces the old override_name_on_node naming trick (which faked the + replacement under the removed peer's old name so JC wouldn't need + touching): this RPC updates JC's live state directly, so the caller + can connect the replacement under its own natural name. + + ``name_new`` must already exist as a bdev (the caller connects it + first) and must not already be in use by JC. Raises RPCRemoteError + with one of the documented codes on rejection/failure: + -10 JC is closing + -11 invalid JM names (empty, or name_old == name_new) + -12 another JM replacement is already in progress + -13 name_old is not currently used by JC + -14 name_new is already used by JC + -15 the JM of name_old is being removed + -3 JC started closing during the operation + -4 the JM context disappeared during the operation + -5 failed to re-key the JM lists (OOM) -- affected vuids stopped + -6 timed out connecting to the new JM bdev + """ + return self._request3("jc_replace_jm", name_old=name_old, name_new=name_new) + def listeners_del(self, nqn, trtype, traddr, trsvcid): """" nqn: Subsystem NQN. diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 8a4594d170..39a3c9c030 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -2025,8 +2025,7 @@ def _peer_reachable_via_jm_quorum(target_node_id, this_node: StorageNode, peer_p return not probed -def _connect_to_remote_jm_devs(this_node: StorageNode, jm_ids=None, only_node_id=None, - drop_stale_overrides=False): +def _connect_to_remote_jm_devs(this_node: StorageNode, jm_ids=None, only_node_id=None): """Connect ``this_node`` to remote JM devices and return the refreshed remote-JM records. @@ -2035,22 +2034,17 @@ def _connect_to_remote_jm_devs(this_node: StorageNode, jm_ids=None, only_node_id over from ``this_node.remote_jm_devices`` untouched (same rationale and measurement as _connect_to_remote_devs delta mode). - ``drop_stale_overrides``: a JM device standing in as a replacement for a - removed peer's JM (see _decommission_node_devices) is reachable, for - this_node specifically, under the OLD peer's name via - ``JMDevice.override_name_on_node`` -- kept stable so this_node's - ALREADY-BUILT distrib/JM-raid (which has that name baked in as a member) - doesn't need touching. That crutch must not outlive its purpose: once - this_node itself is about to (re)build that construct from scratch -- - restart, LVS recreate, or a brand-new create_lvstore -- there is nothing - left for the override to protect, and continuing to honor it would just - bake the stale name into the fresh build. Callers immediately ahead of - such a rebuild pass True to ignore any override for this_node and use - the JM owner's current name, clearing the stale entry as they go. Every - other caller (decommission-time reconnect, where the override is being - freshly established, or a DELTA reconnect where this_node's own - construct is unchanged) must leave the default False so the override is - honored. + Always connects under the JM owner's own natural name. A replacement JM + picked for a removed peer (see _decommission_node_devices) used to be + forced to answer under the removed peer's OLD name here, via + JMDevice.override_name_on_node, so this_node's already-built distrib/ + JM-raid construct (which has that name baked in as a member) wouldn't + need touching. That naming trick is retired now that SPDK's + jc_replace_jm RPC can swap a live JC member by name directly: + _decommission_node_devices connects the replacement under its own name + and calls jc_replace_jm to update the construct in place, so nothing + downstream needs to keep pretending the new device is named after the + old one. """ db_controller = DBController() @@ -2128,38 +2122,9 @@ def _connect_to_remote_jm_devs(this_node: StorageNode, jm_ids=None, only_node_id # member of the new jm_vuid. Runtime re-attach paths (rejoin, # restart-task) carry their own reachability gating. - # Resolve the name this_node actually connects under. remote_device. - # jm_bdev must record this resolved name, not org_dev.jm_bdev - # unconditionally -- health_controller's diagnostic controller lookup - # (f'remote_{remote_device.jm_bdev}') reads this field back and was - # querying org_dev's natural (unconnected) name every cycle whenever - # an override applied, producing a spurious "ctrlr does not exist" - # SPDK error on every health-check pass (found live 2026-08-19 on a - # replacement JM host serving two overridden consumers at once). - override_applies = bool( - org_dev.override_name_on_node - and this_node.get_id() in org_dev.override_name_on_node) + # Always connect under org_dev's own current name -- see the + # docstring for why no override resolution happens here anymore. resolved_name = org_dev.jm_bdev - if override_applies and not drop_stale_overrides: - resolved_name = org_dev.override_name_on_node[this_node.get_id()] - elif override_applies and drop_stale_overrides and org_dev_node is not None: - # this_node is about to (re)build the very construct the - # override was protecting -- nothing left to protect, so drop it - # and settle on org_dev's own current name (resolved_name above) - # going forward. See the docstring for why this is scoped to - # callers that pass drop_stale_overrides=True. - consumer_id = this_node.get_id() - - def _drop_stale_override(n, consumer_id=consumer_id): - if n.jm_device and consumer_id in (n.jm_device.override_name_on_node or {}): - del n.jm_device.override_name_on_node[consumer_id] - - try: - db_controller.atomic_update(org_dev_node, _drop_stale_override) - except Exception as e: - logger.warning( - f"Failed to drop stale JM name override for " - f"{consumer_id} on {org_dev_node.get_id()}: {e}") remote_device = RemoteJMDevice() remote_device.uuid = org_dev.uuid @@ -3330,7 +3295,7 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, if snode.enable_ha_jm: logger.info("Connecting to remote JMs") - snode.remote_jm_devices = _connect_to_remote_jm_devs(snode, drop_stale_overrides=True) + snode.remote_jm_devices = _connect_to_remote_jm_devs(snode) snode.write_to_db(kv_store) @@ -4325,85 +4290,100 @@ def _decommission_node_devices(removed_node: StorageNode): if node.status == StorageNode.STATUS_REMOVED: continue if node.jm_ids and removed_node.jm_device.get_id() in node.jm_ids: - node.jm_ids.remove(removed_node.jm_device.get_id()) + removed_jm_id = removed_node.jm_device.get_id() + # Capture the exact bdev name node's JC currently has live + # for this slot BEFORE any bookkeeping changes below -- this + # is jc_replace_jm's ``name_old``. It's already the resolved + # name whatever it is (this node's own natural connect, or a + # leftover override name from before this RPC existed), so + # there's no need to separately consult the removed device's + # own naming the way the old override-chaining logic did. + old_remote_dev = next( + (rd for rd in (node.remote_jm_devices or []) if rd.uuid == removed_jm_id), None) + name_old = old_remote_dev.remote_bdev if old_remote_dev else None + + node.jm_ids.remove(removed_jm_id) jm_ids = get_sorted_ha_jms(node) logger.debug(f"online_jms: {str(jm_ids)}") # A candidate node already reaches via some path OTHER than # its own jm_ids -- most commonly the hosted-primary route: # node hosts some OTHER primary's secondary/tertiary copy, # and that primary's own jm_ids already includes this - # candidate -- can't serve as an override stand-in here. - # SPDK won't attach a second, distinctly-named local - # controller to a target it already has a live connection - # to under another name, so the override name would never - # actually connect: the attach RPC returns without a bdev - # name, _connect_to_remote_jm_devs' fallback silently reuses - # the OTHER (pre-existing) connection's name for remote_bdev, - # and the DB ends up claiming a healthy override that was - # never live (found live 2026-08-19: "Bdev name not returned - # from controller attach", the JM group "excluded from - # further operation" on an ongoing SPDK retry loop, while - # the DB metadata reported success). Skip any candidate - # node already reaches, however it reaches it, so a fresh - # pick never collides this way. + # candidate. SPDK won't attach a second, distinctly-named + # local controller to a target it already has a live + # connection to under another name (the attach RPC returns + # without a bdev name), and jc_replace_jm itself rejects a + # name_new already in use by JC (-14) -- so a candidate node + # already reaches, however it reaches it, can never actually + # serve as the replacement. Skip it so a fresh pick never + # collides this way (found live 2026-08-19: "Bdev name not + # returned from controller attach", the JM group "excluded + # from further operation" on an ongoing SPDK retry loop, + # while the DB metadata reported success -- back when a + # collision here was silently accepted anyway). already_reachable = {rd.uuid for rd in (node.remote_jm_devices or [])} new_jm_dev = "" - fallback_jm_dev = "" for jm_id in jm_ids: - if jm_id in node.jm_ids: + if jm_id in node.jm_ids or jm_id in already_reachable: continue - if jm_id not in already_reachable: - new_jm_dev = jm_id - break - elif not fallback_jm_dev: - fallback_jm_dev = jm_id - if not new_jm_dev and fallback_jm_dev: - # No collision-free candidate at all -- rather than - # leave this redundancy slot permanently and silently - # short (the alternative: log an error and walk away, - # which never gets revisited by anything), accept the - # colliding one. It won't actually connect until node - # itself next restarts -- that restart's own full - # refresh (drop_stale_overrides=True) drops this exact - # override and reconnects under the name that's already - # live, self-healing cleanly -- but until then this is a - # visible, ongoing SPDK JC retry/exclude loop rather than - # an invisible, permanent gap. Visible-and-self-healing - # beats invisible-and-permanent. - logger.warning( - f"No collision-free JM candidate for {node.get_id()}; " - f"falling back to {fallback_jm_dev}, which node " - f"already reaches via another path. This override " - f"will not actually connect until node's own next " - f"restart clears it and reconnects under the correct " - f"name.") - new_jm_dev = fallback_jm_dev - if new_jm_dev: + new_jm_dev = jm_id + break + + replaced = False + if new_jm_dev and name_old: d = db_controller.get_jm_device_by_id(new_jm_dev) - jm_node = db_controller.get_storage_node_by_id(d.node_id) - # Chain the name node actually needs, not removed_node's - # own natural name: removed_node may itself have been - # standing in as an override for node (a second removal, - # no restart in between -- e.g. A removed, C picked as - # node's replacement JM under the name "jm_A", then C - # itself removed before anyone restarted). In that case - # node's still-unrebuilt construct references THAT - # older name, never removed_node's own -- propagate it - # forward instead of jumping to a name node never used. - inherited_name = (removed_node.jm_device.override_name_on_node or {}).get( - node.get_id()) - jm_node.jm_device.override_name_on_node[node.get_id()] = ( - inherited_name or removed_node.jm_device.jm_bdev) - jm_node.write_to_db() - node.jm_ids.append(new_jm_dev) - node.remote_jm_devices = _connect_to_remote_jm_devs(node, node.jm_ids) - else: - # Truly nothing available (not even a colliding - # candidate) -- still persist the .remove() above rather - # than silently dropping it: an explicitly short jm_ids - # is a strictly more honest state than one that still - # references a JM device that no longer exists. - logger.error(f"no jm_id found for {node.get_id()}") + controller_name = f"remote_{d.jm_bdev}" + try: + # Connect the replacement under its own name first -- + # jc_replace_jm hands off to an already-live bdev, it + # doesn't create the connection itself. + connected = _connect_to_remote_jm_devs( + node, jm_ids=[new_jm_dev], only_node_id=d.node_id) + new_remote_dev = next( + (rd for rd in connected if rd.uuid == new_jm_dev), None) + if not new_remote_dev or not new_remote_dev.remote_bdev: + raise RPCException( + f"failed to connect replacement JM device {new_jm_dev}") + name_new = new_remote_dev.remote_bdev + node.rpc_client(timeout=30, retry=2).jc_replace_jm( + name_old=name_old, name_new=name_new) + logger.info( + f"[REMOVAL] {node.get_id()}: jc_replace_jm {name_old} -> " + f"{name_new} ({new_jm_dev}) replacing removed JM {removed_jm_id}") + node.jm_ids.append(new_jm_dev) + node.remote_jm_devices = connected + replaced = True + except Exception as e: + logger.error( + f"[REMOVAL] {node.get_id()}: jc_replace_jm failed replacing " + f"{name_old} with candidate {new_jm_dev} ({controller_name}): {e}") + # Best-effort: don't leave an attached-but-unused + # connection sitting around. -14 (name_new already + # used by JC) is the one failure where the bdev is + # legitimately claimed by JC already -- detaching it + # would tear down something in active use, so skip + # the cleanup in that specific case. + if getattr(e, "code", None) != -14: + try: + node.rpc_client().bdev_nvme_detach_controller(controller_name) + except Exception as de: + logger.warning( + f"Failed to detach unused controller " + f"{controller_name} on {node.get_id()}: {de}") + if not replaced: + # No collision-free candidate, or the replace RPC itself + # failed -- leave the redundancy slot honestly short + # (jm_ids already has the dead id removed above) rather + # than claim a replacement that isn't actually live in + # JC. Nothing currently revisits this automatically; it + # stays a visible gap until a future removal/reconnect + # cycle retries it. + if not new_jm_dev: + logger.error(f"no jm_id found for {node.get_id()}") + elif not name_old: + logger.error( + f"[REMOVAL] {node.get_id()}: no recorded bdev name for removed " + f"JM {removed_jm_id}; cannot call jc_replace_jm") node.write_to_db() elif any(d.uuid == removed_node.jm_device.get_id() for d in (node.remote_jm_devices or [])): # node.jm_ids is this node's OWN redundancy set for its OWN JM @@ -5359,7 +5339,7 @@ def _restart_storage_node_impl( def _jm_reconcile(): try: - jm_result["devices"] = _connect_to_remote_jm_devs(snode, drop_stale_overrides=True) + jm_result["devices"] = _connect_to_remote_jm_devs(snode) except Exception as e: jm_result["error"] = e @@ -8128,7 +8108,7 @@ def _recreate_lvstore_on_non_leader_impl(snode: StorageNode, leader_node, primar logger.warning("Soft reconnect of remote devices failed on %s: %s", snode.get_id(), e) try: - fresh_remote_jms = _connect_to_remote_jm_devs(snode, drop_stale_overrides=True) + fresh_remote_jms = _connect_to_remote_jm_devs(snode) snode = db_controller.get_storage_node_by_id(snode.get_id()) snode.remote_jm_devices = fresh_remote_jms or snode.remote_jm_devices snode.write_to_db() @@ -9157,7 +9137,7 @@ def _recreate_lvstore_impl(snode: StorageNode, force=False, lvs_primary=None, ac if not is_takeover: snode = db_controller.get_storage_node_by_id(snode.get_id()) - snode.remote_jm_devices = _connect_to_remote_jm_devs(snode, drop_stale_overrides=True) + snode.remote_jm_devices = _connect_to_remote_jm_devs(snode) snode.write_to_db() # Gather peer nodes for this LVS, EXCLUDING snode itself @@ -10487,10 +10467,7 @@ def get_node_jm_names(current_node: StorageNode, remote_node=None): continue jm_dev = DBController().get_jm_device_by_id(jm_id) - if jm_dev.override_name_on_node and current_node.get_id() in jm_dev.override_name_on_node: - jm_list.append(f"remote_{jm_dev.override_name_on_node[current_node.get_id()]}n1") - else: - jm_list.append(f"remote_{jm_dev.jm_bdev}n1") + jm_list.append(f"remote_{jm_dev.jm_bdev}n1") return jm_list[:current_node.ha_jm_count] @@ -10841,7 +10818,7 @@ def create_lvstore(snode: StorageNode, ndcs, npcs, distr_bs, distr_chunk_bs, pag jm_vuid = utils.get_random_vuid() jm_ids = get_sorted_ha_jms(snode) logger.debug(f"online_jms: {str(jm_ids)}") - snode.remote_jm_devices = _connect_to_remote_jm_devs(snode, jm_ids, drop_stale_overrides=True) + snode.remote_jm_devices = _connect_to_remote_jm_devs(snode, jm_ids) snode.jm_ids = jm_ids snode.jm_vuid = jm_vuid snode.write_to_db() @@ -10976,7 +10953,7 @@ def create_lvstore(snode: StorageNode, ndcs, npcs, distr_bs, distr_chunk_bs, pag sec_node.lvstore_ports[lvs_name] = snode.lvstore_ports[lvs_name].copy() # creating lvstore on secondary - sec_node.remote_jm_devices = _connect_to_remote_jm_devs(sec_node, drop_stale_overrides=True) + sec_node.remote_jm_devices = _connect_to_remote_jm_devs(sec_node) sec_node.write_to_db() ret, err = _create_bdev_stack(sec_node, lvstore_stack, primary_node=snode) if err: diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index 07aa376a78..58df97389d 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -23,7 +23,7 @@ from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice, RemoteJMDevice from simplyblock_core.models.cluster import Cluster -from simplyblock_core.rpc_client import RPCConnectionError, RPCException +from simplyblock_core.rpc_client import RPCConnectionError, RPCException, RPCRemoteError # --------------------------------------------------------------------------- @@ -86,16 +86,6 @@ def _node(node_id, status=StorageNode.STATUS_ONLINE, lvstore="", jm.uuid = f"jm-{node_id}" jm.node_id = node_id jm.status = JMDevice.STATUS_ONLINE - # BaseModel.from_dict() binds a bare JMDevice()'s dict-typed - # defaults to the CLASS-level object itself (setattr(self, attr, - # getattr(self, attr)) when the attr is absent from data) -- every - # JMDevice() built this way shares ONE override_name_on_node dict - # until something reassigns it. Production is unaffected (every DB - # read round-trips through from_dict(data) with the key present, - # which always constructs a fresh dict), but bare-constructed test - # fixtures aren't -- give each one its own so an in-place mutation - # in one test can't bleed into another via the shared class default. - jm.override_name_on_node = {} n.jm_device = jm else: n.jm_device = None @@ -1282,50 +1272,68 @@ def test_does_not_refresh_peer_with_no_dead_jm_reference_at_all(self): connect_mock.assert_not_called() peer.write_to_db.assert_not_called() - def test_picks_replacement_and_records_override_with_no_prior_chain(self): - # Baseline (no prior chain): removed node was never itself an - # override stand-in, so the new replacement inherits removed_node's - # OWN natural jm_bdev, same as always. + def test_picks_replacement_connects_it_and_calls_jc_replace_jm(self): + # Baseline: the replacement is connected under its OWN natural name + # (no override), and jc_replace_jm is told to swap consumer's live + # JC member from whatever it currently is (name_old, taken from + # consumer's own remote_jm_devices record) to that new name. cl = _cluster() removed = _node("n1", n_devices=0, with_jm=True) removed.jm_ids = [] consumer = _node("consumer", n_devices=0, with_jm=True) consumer.jm_ids = [removed.jm_device.get_id()] + live_old = RemoteJMDevice() + live_old.uuid = removed.jm_device.get_id() + live_old.remote_bdev = "remote_jm_n1n1" + consumer.remote_jm_devices = [live_old] replacement = _node("replacement", n_devices=0, with_jm=True) replacement.jm_ids = [] + replacement.jm_device.jm_bdev = "jm_replacement" db = FakeDB(cl, [removed, consumer, replacement]) db.get_jm_device_by_id = MagicMock(side_effect=lambda jid: { removed.jm_device.get_id(): removed.jm_device, replacement.jm_device.get_id(): replacement.jm_device, }[jid]) dc = MagicMock() + connected_new = RemoteJMDevice() + connected_new.uuid = replacement.jm_device.get_id() + connected_new.remote_bdev = "remote_jm_replacementn1" with patch.object(storage_node_ops, "DBController", return_value=db), \ patch.object(storage_node_ops, "device_controller", dc), \ patch.object(storage_node_ops, "get_sorted_ha_jms", return_value=[replacement.jm_device.get_id()]), \ - patch.object(storage_node_ops, "_connect_to_remote_jm_devs", return_value=[]): + patch.object(storage_node_ops, "_connect_to_remote_jm_devs", + return_value=[connected_new]) as connect_mock: ret = storage_node_ops._decommission_node_devices(removed) self.assertTrue(ret) - self.assertEqual( - replacement.jm_device.override_name_on_node.get("consumer"), - removed.jm_device.jm_bdev) - replacement.write_to_db.assert_called() - - def test_removed_node_was_itself_an_override_stand_in_chains_the_original_name(self): - # Two-hop chain, no restart in between: A removed, "removed" (here - # playing C) was picked as consumer's replacement JM under the - # override name "jm_A" -- consumer's still-unrebuilt raid construct - # references THAT name, never "removed"'s own. "removed" is now - # itself removed before consumer (or "removed") ever restarted -- - # the next replacement must inherit "jm_A", not removed's own - # natural name, which consumer's construct never referenced. + connect_mock.assert_called_once_with( + consumer, jm_ids=[replacement.jm_device.get_id()], + only_node_id=replacement.get_id()) + consumer.rpc_client().jc_replace_jm.assert_called_once_with( + name_old="remote_jm_n1n1", name_new="remote_jm_replacementn1") + self.assertIn(replacement.jm_device.get_id(), consumer.jm_ids) + self.assertNotIn(removed.jm_device.get_id(), consumer.jm_ids) + self.assertEqual(consumer.remote_jm_devices, [connected_new]) + consumer.write_to_db.assert_called() + + def test_name_old_is_whatever_the_consumer_currently_has_live_not_removeds_own_name(self): + # A prior replacement (back when this used the retired + # override_name_on_node trick, or simply a longer chain of + # replacements) can leave consumer's live JC member named something + # that has nothing to do with removed_node's own jm_bdev. name_old + # must reflect reality (consumer's own remote_jm_devices record), + # never removed_node.jm_device.jm_bdev blindly. cl = _cluster() removed = _node("n1", n_devices=0, with_jm=True) removed.jm_ids = [] - removed.jm_device.override_name_on_node = {"consumer": "jm_A"} + removed.jm_device.jm_bdev = "jm_n1" # consumer's JC never actually used this name consumer = _node("consumer", n_devices=0, with_jm=True) consumer.jm_ids = [removed.jm_device.get_id()] + live_old = RemoteJMDevice() + live_old.uuid = removed.jm_device.get_id() + live_old.remote_bdev = "jm_A" # the name actually live in consumer's JC + consumer.remote_jm_devices = [live_old] replacement = _node("replacement", n_devices=0, with_jm=True) replacement.jm_ids = [] db = FakeDB(cl, [removed, consumer, replacement]) @@ -1334,30 +1342,30 @@ def test_removed_node_was_itself_an_override_stand_in_chains_the_original_name(s replacement.jm_device.get_id(): replacement.jm_device, }[jid]) dc = MagicMock() + connected_new = RemoteJMDevice() + connected_new.uuid = replacement.jm_device.get_id() + connected_new.remote_bdev = "remote_jm_replacementn1" with patch.object(storage_node_ops, "DBController", return_value=db), \ patch.object(storage_node_ops, "device_controller", dc), \ patch.object(storage_node_ops, "get_sorted_ha_jms", return_value=[replacement.jm_device.get_id()]), \ - patch.object(storage_node_ops, "_connect_to_remote_jm_devs", return_value=[]): + patch.object(storage_node_ops, "_connect_to_remote_jm_devs", + return_value=[connected_new]): ret = storage_node_ops._decommission_node_devices(removed) self.assertTrue(ret) - # Inherits "jm_A" -- NOT removed.jm_device.jm_bdev ("jm_n1"). - self.assertEqual( - replacement.jm_device.override_name_on_node.get("consumer"), "jm_A") - replacement.write_to_db.assert_called() + consumer.rpc_client().jc_replace_jm.assert_called_once_with( + name_old="jm_A", name_new="remote_jm_replacementn1") def test_skips_a_candidate_the_consumer_already_reaches_via_another_path(self): # SPDK won't attach a second, distinctly-named local controller to a # target the consumer already has a live connection to under - # another name (found live 2026-08-19: "Bdev name not returned from - # controller attach", the JM group excluded from further operation - # on an ongoing SPDK retry loop, while the DB metadata claimed the - # override connected fine). "colliding" is already in - # consumer.remote_jm_devices (e.g. reached via hosting some OTHER - # primary's secondary copy) even though it was never in - # consumer.jm_ids -- it must be skipped in favor of "clean", the - # next candidate that isn't already reachable by any path. + # another name, and jc_replace_jm itself rejects a name_new already + # used by JC (-14) -- so "colliding" (already in + # consumer.remote_jm_devices, e.g. reached via hosting some OTHER + # primary's secondary copy, even though it was never in + # consumer.jm_ids) must be skipped in favor of "clean", the next + # candidate that isn't already reachable by any path. cl = _cluster() removed = _node("n1", n_devices=0, with_jm=True) removed.jm_ids = [] @@ -1368,7 +1376,10 @@ def test_skips_a_candidate_the_consumer_already_reaches_via_another_path(self): already_connected = RemoteJMDevice() already_connected.uuid = colliding.jm_device.get_id() already_connected.remote_bdev = "remote_jm_colliding-own-namen1" - consumer.remote_jm_devices = [already_connected] + live_old = RemoteJMDevice() + live_old.uuid = removed.jm_device.get_id() + live_old.remote_bdev = "remote_jm_n1n1" + consumer.remote_jm_devices = [already_connected, live_old] clean = _node("clean", n_devices=0, with_jm=True) clean.jm_ids = [] db = FakeDB(cl, [removed, consumer, colliding, clean]) @@ -1378,28 +1389,32 @@ def test_skips_a_candidate_the_consumer_already_reaches_via_another_path(self): clean.jm_device.get_id(): clean.jm_device, }[jid]) dc = MagicMock() + connected_new = RemoteJMDevice() + connected_new.uuid = clean.jm_device.get_id() + connected_new.remote_bdev = "remote_jm_cleann1" with patch.object(storage_node_ops, "DBController", return_value=db), \ patch.object(storage_node_ops, "device_controller", dc), \ patch.object(storage_node_ops, "get_sorted_ha_jms", return_value=[colliding.jm_device.get_id(), clean.jm_device.get_id()]), \ - patch.object(storage_node_ops, "_connect_to_remote_jm_devs", return_value=[]): + patch.object(storage_node_ops, "_connect_to_remote_jm_devs", + return_value=[connected_new]) as connect_mock: ret = storage_node_ops._decommission_node_devices(removed) self.assertTrue(ret) - self.assertEqual(colliding.jm_device.override_name_on_node, {}) - self.assertEqual( - clean.jm_device.override_name_on_node.get("consumer"), - removed.jm_device.jm_bdev) - clean.write_to_db.assert_called() - colliding.write_to_db.assert_not_called() - - def test_falls_back_to_a_colliding_candidate_when_no_clean_one_exists(self): - # No collision-free candidate anywhere -- rather than leave the - # slot permanently and silently short, accept the colliding one. - # It won't actually connect until consumer's own next restart (that - # restart's drop_stale_overrides=True refresh cleans this exact - # entry up and reconnects correctly), but a visible, self-healing - # degraded state beats an invisible, permanent gap. + connect_mock.assert_called_once_with( + consumer, jm_ids=[clean.jm_device.get_id()], only_node_id=clean.get_id()) + consumer.rpc_client().jc_replace_jm.assert_called_once_with( + name_old="remote_jm_n1n1", name_new="remote_jm_cleann1") + self.assertIn(clean.jm_device.get_id(), consumer.jm_ids) + self.assertNotIn(colliding.jm_device.get_id(), consumer.jm_ids) + + def test_no_collision_free_candidate_leaves_slot_honestly_short(self): + # No collision-free candidate anywhere -- unlike the retired + # override mechanism (which used to fake-accept a colliding + # candidate and rely on a later restart to self-heal it), + # jc_replace_jm would just reject a colliding name_new outright + # (-14), so there's no point even attempting it. Leave the + # redundancy slot honestly short instead. cl = _cluster() removed = _node("n1", n_devices=0, with_jm=True) removed.jm_ids = [] @@ -1421,17 +1436,99 @@ def test_falls_back_to_a_colliding_candidate_when_no_clean_one_exists(self): patch.object(storage_node_ops, "device_controller", dc), \ patch.object(storage_node_ops, "get_sorted_ha_jms", return_value=[colliding.jm_device.get_id()]), \ - patch.object(storage_node_ops, "_connect_to_remote_jm_devs", return_value=[]): + patch.object(storage_node_ops, "_connect_to_remote_jm_devs") as connect_mock: + ret = storage_node_ops._decommission_node_devices(removed) + + self.assertTrue(ret) + connect_mock.assert_not_called() + self.assertNotIn(colliding.jm_device.get_id(), consumer.jm_ids) + self.assertNotIn(removed.jm_device.get_id(), consumer.jm_ids) + colliding.write_to_db.assert_not_called() + consumer.write_to_db.assert_called() + + def test_jc_replace_jm_failure_leaves_slot_short_and_detaches_unused_connection(self): + # The candidate connects fine but the swap itself is rejected (e.g. + # a timeout connecting to the new JM bdev, code -6) -- don't claim + # the replacement, and don't leave the now-unused connection + # dangling: best-effort detach it. + cl = _cluster() + removed = _node("n1", n_devices=0, with_jm=True) + removed.jm_ids = [] + consumer = _node("consumer", n_devices=0, with_jm=True) + consumer.jm_ids = [removed.jm_device.get_id()] + live_old = RemoteJMDevice() + live_old.uuid = removed.jm_device.get_id() + live_old.remote_bdev = "remote_jm_n1n1" + consumer.remote_jm_devices = [live_old] + replacement = _node("replacement", n_devices=0, with_jm=True) + replacement.jm_ids = [] + replacement.jm_device.jm_bdev = "jm_replacement" + db = FakeDB(cl, [removed, consumer, replacement]) + db.get_jm_device_by_id = MagicMock(side_effect=lambda jid: { + removed.jm_device.get_id(): removed.jm_device, + replacement.jm_device.get_id(): replacement.jm_device, + }[jid]) + dc = MagicMock() + connected_new = RemoteJMDevice() + connected_new.uuid = replacement.jm_device.get_id() + connected_new.remote_bdev = "remote_jm_replacementn1" + consumer.rpc_client.return_value.jc_replace_jm.side_effect = RPCRemoteError( + "timed out connecting to the new JM bdev", code=-6) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "device_controller", dc), \ + patch.object(storage_node_ops, "get_sorted_ha_jms", + return_value=[replacement.jm_device.get_id()]), \ + patch.object(storage_node_ops, "_connect_to_remote_jm_devs", + return_value=[connected_new]): ret = storage_node_ops._decommission_node_devices(removed) self.assertTrue(ret) - self.assertEqual( - colliding.jm_device.override_name_on_node.get("consumer"), - removed.jm_device.jm_bdev) - self.assertIn(colliding.jm_device.get_id(), consumer.jm_ids) - colliding.write_to_db.assert_called() + self.assertNotIn(replacement.jm_device.get_id(), consumer.jm_ids) + self.assertNotIn(removed.jm_device.get_id(), consumer.jm_ids) + self.assertEqual(consumer.remote_jm_devices, [live_old]) + consumer.rpc_client.return_value.bdev_nvme_detach_controller.assert_called_once_with( + "remote_jm_replacement") consumer.write_to_db.assert_called() + def test_jc_replace_jm_dash14_skips_the_detach_cleanup(self): + # -14 (name_new already used by JC) means the bdev is legitimately + # claimed by JC already -- detaching it would tear down something + # in active use, so the cleanup must be skipped for this one code. + cl = _cluster() + removed = _node("n1", n_devices=0, with_jm=True) + removed.jm_ids = [] + consumer = _node("consumer", n_devices=0, with_jm=True) + consumer.jm_ids = [removed.jm_device.get_id()] + live_old = RemoteJMDevice() + live_old.uuid = removed.jm_device.get_id() + live_old.remote_bdev = "remote_jm_n1n1" + consumer.remote_jm_devices = [live_old] + replacement = _node("replacement", n_devices=0, with_jm=True) + replacement.jm_ids = [] + replacement.jm_device.jm_bdev = "jm_replacement" + db = FakeDB(cl, [removed, consumer, replacement]) + db.get_jm_device_by_id = MagicMock(side_effect=lambda jid: { + removed.jm_device.get_id(): removed.jm_device, + replacement.jm_device.get_id(): replacement.jm_device, + }[jid]) + dc = MagicMock() + connected_new = RemoteJMDevice() + connected_new.uuid = replacement.jm_device.get_id() + connected_new.remote_bdev = "remote_jm_replacementn1" + consumer.rpc_client.return_value.jc_replace_jm.side_effect = RPCRemoteError( + "name_new is already used by JC", code=-14) + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops, "device_controller", dc), \ + patch.object(storage_node_ops, "get_sorted_ha_jms", + return_value=[replacement.jm_device.get_id()]), \ + patch.object(storage_node_ops, "_connect_to_remote_jm_devs", + return_value=[connected_new]): + ret = storage_node_ops._decommission_node_devices(removed) + + self.assertTrue(ret) + self.assertNotIn(replacement.jm_device.get_id(), consumer.jm_ids) + consumer.rpc_client.return_value.bdev_nvme_detach_controller.assert_not_called() + def test_no_candidate_at_all_still_persists_the_removed_jm_id(self): # Regression guard for a real bug found while explaining this # branch: the ONLY prior action, "no jm_id found" -> logger.error, @@ -1681,27 +1778,32 @@ def test_transient_failure_does_not_block_a_clean_connect(self): # --------------------------------------------------------------------------- # _connect_to_remote_jm_devs — remote_device.jm_bdev must record the name -# THIS NODE actually connects under, override included +# THIS NODE actually connects under # -# override_name_on_node lets a replacement JM connect under a removed peer's -# old bdev name so the consumer's own already-built JM raid doesn't need -# touching (see _decommission_node_devices). remote_device.jm_bdev must -# record that SAME resolved name, not org_dev's own natural name, or +# Used to matter most when an override applied (a replacement JM connecting +# under a removed peer's old bdev name, per the now-retired +# override_name_on_node -- see _decommission_node_devices for why +# jc_replace_jm replaced that trick): remote_device.jm_bdev had to record +# the resolved (override) name, not org_dev's own natural name, or # health_controller's diagnostic controller lookup -# (f'remote_{remote_device.jm_bdev}') queries the wrong, never-connected name -# every cycle. Found live 2026-08-19: a replacement JM host serving two -# overridden consumers logged a spurious SPDK "ctrlr ... does not exist" -# error on every health-check pass. +# (f'remote_{remote_device.jm_bdev}') queried the wrong, never-connected +# name every cycle. Now there's only ever one name to record -- the owner's +# own -- but the field still has to be right for the same diagnostic lookup. # --------------------------------------------------------------------------- class TestConnectToRemoteJmDevsRecordsResolvedName(unittest.TestCase): + # override_name_on_node (and the drop_stale_overrides parameter that + # existed only to retire a stale entry) is gone now that SPDK's + # jc_replace_jm RPC swaps a live JC member by name directly -- + # _connect_to_remote_jm_devs always connects under the owner's own + # current name. This class is now just a baseline regression guard for + # that natural-name path. - def _owner_setup(self, this_node_id="this-node", override=None): + def _owner_setup(self, this_node_id="this-node"): jm_dev = JMDevice() jm_dev.uuid = "jm-owner" jm_dev.jm_bdev = "jm_owner_bdev" jm_dev.status = NVMeDevice.STATUS_ONLINE - jm_dev.override_name_on_node = override or {} owner_node = MagicMock(spec=StorageNode) owner_node.get_id = MagicMock(return_value="owner-node") @@ -1723,7 +1825,7 @@ def _owner_setup(self, this_node_id="this-node", override=None): return this_node, rpc_client, db - def test_no_override_uses_owners_own_natural_name(self): + def test_connects_under_owners_own_natural_name(self): this_node, rpc_client, db = self._owner_setup() rpc_client.get_bdevs.return_value = {"name": "remote_jm_owner_bdevn1"} @@ -1735,165 +1837,9 @@ def test_no_override_uses_owners_own_natural_name(self): self.assertEqual(len(result), 1) self.assertEqual(result[0].jm_bdev, "jm_owner_bdev") + self.assertEqual(result[0].remote_bdev, "remote_jm_owner_bdevn1") self.assertEqual(connect_mock.call_args[0][0], "remote_jm_owner_bdev") - def test_override_present_records_the_override_name_not_owners_own(self): - this_node, rpc_client, db = self._owner_setup( - this_node_id="this-node", - override={"this-node": "jm_removed_peer_bdev"}) - rpc_client.get_bdevs.return_value = {"name": "remote_jm_removed_peer_bdevn1"} - - with patch.object(storage_node_ops, "DBController", return_value=db), \ - patch.object(storage_node_ops, "connect_device", - return_value="remote_jm_removed_peer_bdevn1") as connect_mock: - result = storage_node_ops._connect_to_remote_jm_devs( - this_node, jm_ids=["jm-owner"]) - - self.assertEqual(len(result), 1) - # jm_bdev must match the name actually connected under (the - # override), not org_dev's own "jm_owner_bdev" -- this is the exact - # field health_controller reads back to build its diagnostic lookup. - self.assertEqual(result[0].jm_bdev, "jm_removed_peer_bdev") - self.assertEqual(result[0].remote_bdev, "remote_jm_removed_peer_bdevn1") - self.assertEqual(connect_mock.call_args[0][0], "remote_jm_removed_peer_bdev") - - def test_override_keyed_to_a_different_node_does_not_apply(self): - # The override only applies to the specific consumer it was recorded - # for -- a DIFFERENT this_node connecting to the same owner must - # still get the owner's own natural name. - this_node, rpc_client, db = self._owner_setup( - this_node_id="this-node", - override={"some-other-node": "jm_removed_peer_bdev"}) - rpc_client.get_bdevs.return_value = {"name": "remote_jm_owner_bdevn1"} - - with patch.object(storage_node_ops, "DBController", return_value=db), \ - patch.object(storage_node_ops, "connect_device", - return_value="remote_jm_owner_bdevn1"): - result = storage_node_ops._connect_to_remote_jm_devs( - this_node, jm_ids=["jm-owner"]) - - self.assertEqual(len(result), 1) - self.assertEqual(result[0].jm_bdev, "jm_owner_bdev") - - -# --------------------------------------------------------------------------- -# _connect_to_remote_jm_devs — drop_stale_overrides retires the legacy-name -# crutch once this_node itself rebuilds its own JM-consuming construct -# -# override_name_on_node exists only to keep an ALREADY-BUILT distrib/JM-raid -# on this_node pointed at a stable name across a JM replacement elsewhere. -# Once this_node is about to (re)build that construct from scratch -- -# restart, LVS recreate, or a brand-new create_lvstore, all of which call -# with drop_stale_overrides=True immediately ahead of the rebuild -- there -# is nothing left for the override to protect, so it must be dropped rather -# than baked into the fresh build. A DELTA reconnect (drop_stale_overrides -# left False, e.g. a peer reconnecting to a node that just restarted) must -# keep honoring it: this_node's own construct did not change. -# --------------------------------------------------------------------------- - -class TestConnectToRemoteJmDevsDropsStaleOverrideOnRebuild(unittest.TestCase): - - def _owner_setup(self, this_node_id="this-node", override=None): - jm_dev = JMDevice() - jm_dev.uuid = "jm-owner" - jm_dev.jm_bdev = "jm_owner_bdev" - jm_dev.status = NVMeDevice.STATUS_ONLINE - jm_dev.override_name_on_node = override or {} - - owner_node = MagicMock(spec=StorageNode) - owner_node.get_id = MagicMock(return_value="owner-node") - owner_node.status = StorageNode.STATUS_ONLINE - owner_node.jm_device = jm_dev - - this_node = MagicMock(spec=StorageNode) - this_node.get_id = MagicMock(return_value=this_node_id) - this_node.jm_ids = [] - this_node.lvstore_stack_secondary = "" - this_node.lvstore_stack_tertiary = "" - this_node.remote_jm_devices = [] - rpc_client = MagicMock() - this_node.rpc_client = MagicMock(return_value=rpc_client) - - db = MagicMock() - db.get_jm_device_by_id.return_value = jm_dev - db.get_storage_nodes.return_value = [owner_node] - - return this_node, rpc_client, db, owner_node - - def test_rebuild_ignores_the_override_and_uses_the_owners_current_name(self): - this_node, rpc_client, db, owner_node = self._owner_setup( - override={"this-node": "jm_removed_peer_bdev"}) - rpc_client.get_bdevs.return_value = {"name": "remote_jm_owner_bdevn1"} - - with patch.object(storage_node_ops, "DBController", return_value=db), \ - patch.object(storage_node_ops, "connect_device", - return_value="remote_jm_owner_bdevn1") as connect_mock: - result = storage_node_ops._connect_to_remote_jm_devs( - this_node, jm_ids=["jm-owner"], drop_stale_overrides=True) - - self.assertEqual(len(result), 1) - # Uses the owner's own current name, NOT the stale override -- - # this_node is rebuilding its own construct right now, so there is - # nothing left for the legacy name to protect. - self.assertEqual(result[0].jm_bdev, "jm_owner_bdev") - self.assertEqual(connect_mock.call_args[0][0], "remote_jm_owner_bdev") - - def test_rebuild_clears_the_override_entry_via_atomic_update(self): - this_node, rpc_client, db, owner_node = self._owner_setup( - override={"this-node": "jm_removed_peer_bdev", "other-node": "jm_something_else"}) - rpc_client.get_bdevs.return_value = {"name": "remote_jm_owner_bdevn1"} - - with patch.object(storage_node_ops, "DBController", return_value=db), \ - patch.object(storage_node_ops, "connect_device", - return_value="remote_jm_owner_bdevn1"): - storage_node_ops._connect_to_remote_jm_devs( - this_node, jm_ids=["jm-owner"], drop_stale_overrides=True) - - db.atomic_update.assert_called_once() - target, mutate_fn = db.atomic_update.call_args.args[:2] - self.assertIs(target, owner_node) - - # The mutate_fn only removes THIS consumer's entry from a fresh copy - # -- it must not clear another consumer's still-live override. - fresh_jm_dev = JMDevice() - fresh_jm_dev.override_name_on_node = { - "this-node": "jm_removed_peer_bdev", "other-node": "jm_something_else"} - fresh_owner = MagicMock(spec=StorageNode) - fresh_owner.jm_device = fresh_jm_dev - mutate_fn(fresh_owner) - self.assertEqual(fresh_jm_dev.override_name_on_node, {"other-node": "jm_something_else"}) - - def test_delta_reconnect_leaves_the_override_untouched(self): - # only_node_id set (a peer reconnecting after ITS restart) with - # drop_stale_overrides left at its default False: this_node's own - # construct hasn't changed, so the override must still be honored - # and nothing should be cleared. - this_node, rpc_client, db, owner_node = self._owner_setup( - override={"this-node": "jm_removed_peer_bdev"}) - rpc_client.get_bdevs.return_value = {"name": "remote_jm_removed_peer_bdevn1"} - - with patch.object(storage_node_ops, "DBController", return_value=db), \ - patch.object(storage_node_ops, "connect_device", - return_value="remote_jm_removed_peer_bdevn1"): - result = storage_node_ops._connect_to_remote_jm_devs( - this_node, jm_ids=["jm-owner"], only_node_id="owner-node") - - self.assertEqual(len(result), 1) - self.assertEqual(result[0].jm_bdev, "jm_removed_peer_bdev") - db.atomic_update.assert_not_called() - - def test_no_override_present_never_calls_atomic_update(self): - this_node, rpc_client, db, owner_node = self._owner_setup() - rpc_client.get_bdevs.return_value = {"name": "remote_jm_owner_bdevn1"} - - with patch.object(storage_node_ops, "DBController", return_value=db), \ - patch.object(storage_node_ops, "connect_device", - return_value="remote_jm_owner_bdevn1"): - storage_node_ops._connect_to_remote_jm_devs( - this_node, jm_ids=["jm-owner"], drop_stale_overrides=True) - - db.atomic_update.assert_not_called() - class TestShrinkStatusDoesNotDeadlockRemoval(unittest.TestCase): """``node_removal_orchestrate`` holds ``Cluster.STATUS_IN_SHRINK`` for the