From 470a77e6f1aff6a36924ec1619ded4e932fc2de9 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 29 Sep 2026 20:59:43 +0300 Subject: [PATCH 01/27] IGNITE-29088 Add reproducer --- ...CachePartitionLossIgnorePolicyMapTest.java | 205 ++++++++++++++++++ .../testsuites/IgniteCacheTestSuite15.java | 2 + 2 files changed, 207 insertions(+) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java new file mode 100644 index 0000000000000..ed00db5c23834 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl; +import org.apache.ignite.internal.processors.resource.DependencyResolver; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.testframework.TestDependencyResolver; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static org.apache.ignite.cache.PartitionLossPolicy.IGNORE; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** + * Checks that the coordinator's partition map agrees with the nodes after a node leaves with the only copy of its + * partitions under the {@link org.apache.ignite.cache.PartitionLossPolicy#IGNORE IGNORE} loss policy. + *

+ * Each lost partition is recreated empty on its new primary as MOVING and owned there when the node detects the loss. + * If the new primary sends its partition map between these two steps, the map carries MOVING and overwrites what the + * coordinator already knows, so the node must send its map again once it owns the partition. + */ +public class CachePartitionLossIgnorePolicyMapTest extends GridCommonAbstractTest { + /** */ + private static final String CACHE = "cache"; + + /** */ + private static final int PARTS = 64; + + /** Index of the node that becomes the new primary of the lost partitions. */ + private static final int NEW_PRIMARY = 1; + + /** Index of the node that leaves. */ + private static final int LEAVING = 2; + + /** When set, the new primary sends its partition map right before it owns a MOVING partition of the cache. */ + private final AtomicBoolean sendBeforeOwn = new AtomicBoolean(); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setCommunicationSpi(new TestRecordingCommunicationSpi()) + .setCacheConfiguration(new CacheConfiguration(CACHE) + .setBackups(0) + .setPartitionLossPolicy(IGNORE) + .setAffinity(new RendezvousAffinityFunction(false, PARTS))); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + super.afterTest(); + } + + /** + * A node joins and takes partitions over, the old owners evict their copies, and the node leaves. Its partitions + * are lost, and the new primary sends a stale map, with the lost partition still MOVING, before it owns them. + * + * @throws Exception If failed. + */ + @Test + public void testStaleMapBeforeLostPartitionOwned() throws Exception { + IgniteEx crd = startGrid(0); + + startGrid(NEW_PRIMARY, sendBeforeOwnResolver()); + + IgniteCache cache = crd.cache(CACHE); + + for (int i = 0; i < PARTS * 4; i++) + cache.put(i, i); + + startGrid(LEAVING); + + awaitPartitionMapExchange(true, true, null); + + UUID crdId = crd.localNode().id(); + + TestRecordingCommunicationSpi newPrimarySpi = TestRecordingCommunicationSpi.spi(grid(NEW_PRIMARY)); + + // Hold the new primary's maps sent outside an exchange until the coordinator finishes the exchange. + newPrimarySpi.blockMessages((node, msg) -> node.id().equals(crdId) + && msg instanceof GridDhtPartitionsSingleMessage + && ((GridDhtPartitionsSingleMessage)msg).exchangeId() == null); + + sendBeforeOwn.set(true); + + long leaveVer = crd.cluster().topologyVersion() + 1; + + stopGrid(LEAVING); + + newPrimarySpi.waitForBlocked(); + + assertFalse("The new primary didn't own a lost partition", sendBeforeOwn.get()); + + assertTrue(waitForCondition(() -> { + GridDhtPartitionsExchangeFuture fut = crd.context().cache().context().exchange().lastFinishedFuture(); + + return fut != null && fut.topologyVersion().topologyVersion() >= leaveVer; + }, getTestTimeout())); + + newPrimarySpi.stopBlock(); + + if (!waitForCondition(() -> mismatches(crd).isEmpty(), 10_000)) + fail("The coordinator's partition map differs from the nodes: " + mismatches(crd)); + + awaitPartitionMapExchange(); + } + + /** + * @return Resolver that makes the new primary send its partition map right before it owns a MOVING partition of + * the cache, once {@link #sendBeforeOwn} is set. + */ + private TestDependencyResolver sendBeforeOwnResolver() { + return new TestDependencyResolver(new DependencyResolver() { + @Override public T resolve(T instance) { + if (instance instanceof GridDhtPartitionTopologyImpl) { + GridDhtPartitionTopologyImpl top = (GridDhtPartitionTopologyImpl)instance; + + top.partitionFactory((ctx, grp, id, recovery) -> new GridDhtLocalPartition(ctx, grp, id, recovery) { + @Override public boolean own() { + if (CACHE.equals(grp.cacheOrGroupName()) && state() == MOVING + && sendBeforeOwn.compareAndSet(true, false)) + ctx.exchange().refreshPartitions(Collections.singleton(grp)); + + return super.own(); + } + }); + } + + return instance; + } + }); + } + + /** + * @param crd Coordinator. + * @return Partitions whose state in the coordinator's map differs from the state on the node itself. + */ + private List mismatches(IgniteEx crd) { + GridDhtPartitionTopology crdTop = crd.cachex(CACHE).context().topology(); + + List res = new ArrayList<>(); + + for (Ignite g : G.allGrids()) { + UUID id = g.cluster().localNode().id(); + + GridDhtPartitionMap crdView = crdTop.partitionMap(false).get(id); + + GridDhtPartitionTopology top = ((IgniteEx)g).cachex(CACHE).context().topology(); + + for (int p = 0; p < PARTS; p++) { + GridDhtLocalPartition locPart = top.localPartition(p, AffinityTopologyVersion.NONE, false, true); + + GridDhtPartitionState loc = locPart == null ? null : locPart.state(); + GridDhtPartitionState seen = crdView == null ? null : crdView.get(p); + + // The coordinator keeps evicted partitions in its map, the node doesn't. + if (seen == EVICTED) + seen = null; + + if (loc != seen) + res.add(g.name() + " p=" + p + " local=" + loc + " crd=" + seen); + } + } + + return res; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java index cef8063e41e98..6938d8f2fb5c0 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java @@ -37,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheStoreCollectionTest; import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentNodeJoinValidationTest; import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentTest; +import org.apache.ignite.internal.processors.cache.distributed.CachePartitionLossIgnorePolicyMapTest; import org.apache.ignite.internal.processors.cache.distributed.GridExchangeFreeCellularSwitchComplexOperationsTest; import org.apache.ignite.internal.processors.cache.distributed.GridExchangeFreeCellularSwitchIsolationTest; import org.apache.ignite.internal.processors.cache.distributed.GridExchangeFreeCellularSwitchTxContinuationTest; @@ -98,6 +99,7 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, ClusterActivationFailureTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheCreateDestroyClusterReadOnlyModeTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteCachePartitionLossPolicySelfTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CachePartitionLossIgnorePolicyMapTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteCacheGroupsPartitionLossPolicySelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheRebalancingSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheManualRebalancingTest.class, ignoredTests); From a781d9a65506c6e3231746b61d4ed7ec72581c11 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Tue, 29 Sep 2026 22:28:19 +0300 Subject: [PATCH 02/27] IGNITE-29088 Fix stale lost partition states in the coordinator's partition map --- .../GridDhtPartitionsExchangeFuture.java | 21 ++++++++++++++++++- .../GridDhtPartitionTopologyImpl.java | 6 ++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index ef1dd7419e7cd..6c4b93134fcd1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3502,6 +3502,8 @@ private void findCounterForReservation( * @param resTopVer Result topology version. */ private void detectLostPartitions(AffinityTopologyVersion resTopVer) { + Collection changedGrps = ConcurrentHashMap.newKeySet(); + try { // Reserve at least 2 threads for system operations. doInParallelUninterruptibly( @@ -3509,7 +3511,12 @@ private void detectLostPartitions(AffinityTopologyVersion resTopVer) { cctx.kernalContext().pools().getSystemExecutorService(), cctx.affinity().cacheGroups().values(), desc -> { - partitionTopology(desc.groupId()).detectLostPartitions(resTopVer, this); + if (partitionTopology(desc.groupId()).detectLostPartitions(resTopVer, this)) { + CacheGroupContext grp = cctx.cache().cacheGroup(desc.groupId()); + + if (grp != null) + changedGrps.add(grp); + } return null; }); @@ -3518,6 +3525,18 @@ private void detectLostPartitions(AffinityTopologyVersion resTopVer) { throw new IgniteException(e); } + // Local partitions changed their states after this node had sent its partition map: a lost partition was + // owned under the IGNORE policy or marked LOST. Send the maps again, otherwise the coordinator can keep + // a state the node reported earlier. + if (!changedGrps.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("Refresh partitions due to lost partitions detected [grps=" + + changedGrps.stream().map(CacheGroupContext::cacheOrGroupName).collect(Collectors.toList()) + ']'); + } + + cctx.exchange().refreshPartitions(changedGrps); + } + timeBag.finishGlobalStage("Detect lost partitions"); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index afbb0c191f682..194b366debc9e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -2260,9 +2260,11 @@ private void rebuildDiff(AffinityAssignment affAssignment) { final GridDhtPartitionState prevState = locPart.state(); - changed = isLossIgnored ? locPart.own() : locPart.markLost(); + boolean locChanged = isLossIgnored ? locPart.own() : locPart.markLost(); + + if (locChanged) { + changed = true; - if (changed) { long updSeq = updateSeq.incrementAndGet(); updateLocal(locPart.id(), locPart.state(), updSeq, resTopVer); From 2d2ee1aebdf16a3402d4b778649c642a1d9598de Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 30 Sep 2026 12:47:07 +0300 Subject: [PATCH 03/27] IGNITE-29088 Check owners on a third node --- ...CachePartitionLossIgnorePolicyMapTest.java | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java index ed00db5c23834..ef84d6edb96b8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java @@ -25,6 +25,7 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; @@ -38,6 +39,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl; import org.apache.ignite.internal.processors.resource.DependencyResolver; +import org.apache.ignite.internal.util.lang.ClusterNodeFunc; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.testframework.TestDependencyResolver; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -54,7 +56,8 @@ *

* Each lost partition is recreated empty on its new primary as MOVING and owned there when the node detects the loss. * If the new primary sends its partition map between these two steps, the map carries MOVING and overwrites what the - * coordinator already knows, so the node must send its map again once it owns the partition. + * coordinator already knows, so the node must send its map again once it owns the partition. Otherwise the other nodes + * get the stale state from the coordinator and see no owner of the partition. */ public class CachePartitionLossIgnorePolicyMapTest extends GridCommonAbstractTest { /** */ @@ -69,6 +72,9 @@ public class CachePartitionLossIgnorePolicyMapTest extends GridCommonAbstractTes /** Index of the node that leaves. */ private static final int LEAVING = 2; + /** Index of the node that checks who owns the lost partitions: neither the coordinator nor their new primary. */ + private static final int OBSERVER = 3; + /** When set, the new primary sends its partition map right before it owns a MOVING partition of the cache. */ private final AtomicBoolean sendBeforeOwn = new AtomicBoolean(); @@ -101,6 +107,8 @@ public void testStaleMapBeforeLostPartitionOwned() throws Exception { startGrid(NEW_PRIMARY, sendBeforeOwnResolver()); + IgniteEx observer = startGrid(OBSERVER); + IgniteCache cache = crd.cache(CACHE); for (int i = 0; i < PARTS * 4; i++) @@ -110,6 +118,8 @@ public void testStaleMapBeforeLostPartitionOwned() throws Exception { awaitPartitionMapExchange(true, true, null); + int[] lostParts = crd.affinity(CACHE).primaryPartitions(grid(LEAVING).localNode()); + UUID crdId = crd.localNode().id(); TestRecordingCommunicationSpi newPrimarySpi = TestRecordingCommunicationSpi.spi(grid(NEW_PRIMARY)); @@ -137,12 +147,42 @@ public void testStaleMapBeforeLostPartitionOwned() throws Exception { newPrimarySpi.stopBlock(); - if (!waitForCondition(() -> mismatches(crd).isEmpty(), 10_000)) - fail("The coordinator's partition map differs from the nodes: " + mismatches(crd)); + boolean agreed = waitForCondition( + () -> mismatches(crd).isEmpty() && ownerMismatches(observer, lostParts).isEmpty(), 10_000); + + if (!agreed) { + fail("The coordinator's partition map differs from the nodes: " + mismatches(crd) + + ", the observer sees wrong owners of the lost partitions: " + ownerMismatches(observer, lostParts)); + } awaitPartitionMapExchange(); } + /** + * @param observer Node that is neither the coordinator nor the new primary of the lost partitions it checks. + * @param lostParts Lost partitions. + * @return Lost partitions whose owners, as the observer sees them, are not just their primary. + */ + private List ownerMismatches(IgniteEx observer, int[] lostParts) { + GridDhtPartitionTopology top = observer.cachex(CACHE).context().topology(); + + List res = new ArrayList<>(); + + for (int p : lostParts) { + ClusterNode primary = observer.affinity(CACHE).mapPartitionToNode(p); + + if (primary.isLocal()) + continue; + + List owners = top.owners(p); + + if (!owners.equals(Collections.singletonList(primary))) + res.add("p=" + p + " owners=" + ClusterNodeFunc.nodeIds(owners) + " primary=" + primary.id()); + } + + return res; + } + /** * @return Resolver that makes the new primary send its partition map right before it owns a MOVING partition of * the cache, once {@link #sendBeforeOwn} is set. From 4e4aeb71a17ceae876a31794a7f47e5dca54db6c Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 30 Sep 2026 12:58:45 +0300 Subject: [PATCH 04/27] IGNITE-29088 Revert the third-node check --- ...CachePartitionLossIgnorePolicyMapTest.java | 46 ++----------------- 1 file changed, 3 insertions(+), 43 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java index ef84d6edb96b8..ed00db5c23834 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java @@ -25,7 +25,6 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; -import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; @@ -39,7 +38,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl; import org.apache.ignite.internal.processors.resource.DependencyResolver; -import org.apache.ignite.internal.util.lang.ClusterNodeFunc; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.testframework.TestDependencyResolver; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -56,8 +54,7 @@ *

* Each lost partition is recreated empty on its new primary as MOVING and owned there when the node detects the loss. * If the new primary sends its partition map between these two steps, the map carries MOVING and overwrites what the - * coordinator already knows, so the node must send its map again once it owns the partition. Otherwise the other nodes - * get the stale state from the coordinator and see no owner of the partition. + * coordinator already knows, so the node must send its map again once it owns the partition. */ public class CachePartitionLossIgnorePolicyMapTest extends GridCommonAbstractTest { /** */ @@ -72,9 +69,6 @@ public class CachePartitionLossIgnorePolicyMapTest extends GridCommonAbstractTes /** Index of the node that leaves. */ private static final int LEAVING = 2; - /** Index of the node that checks who owns the lost partitions: neither the coordinator nor their new primary. */ - private static final int OBSERVER = 3; - /** When set, the new primary sends its partition map right before it owns a MOVING partition of the cache. */ private final AtomicBoolean sendBeforeOwn = new AtomicBoolean(); @@ -107,8 +101,6 @@ public void testStaleMapBeforeLostPartitionOwned() throws Exception { startGrid(NEW_PRIMARY, sendBeforeOwnResolver()); - IgniteEx observer = startGrid(OBSERVER); - IgniteCache cache = crd.cache(CACHE); for (int i = 0; i < PARTS * 4; i++) @@ -118,8 +110,6 @@ public void testStaleMapBeforeLostPartitionOwned() throws Exception { awaitPartitionMapExchange(true, true, null); - int[] lostParts = crd.affinity(CACHE).primaryPartitions(grid(LEAVING).localNode()); - UUID crdId = crd.localNode().id(); TestRecordingCommunicationSpi newPrimarySpi = TestRecordingCommunicationSpi.spi(grid(NEW_PRIMARY)); @@ -147,42 +137,12 @@ public void testStaleMapBeforeLostPartitionOwned() throws Exception { newPrimarySpi.stopBlock(); - boolean agreed = waitForCondition( - () -> mismatches(crd).isEmpty() && ownerMismatches(observer, lostParts).isEmpty(), 10_000); - - if (!agreed) { - fail("The coordinator's partition map differs from the nodes: " + mismatches(crd) + - ", the observer sees wrong owners of the lost partitions: " + ownerMismatches(observer, lostParts)); - } + if (!waitForCondition(() -> mismatches(crd).isEmpty(), 10_000)) + fail("The coordinator's partition map differs from the nodes: " + mismatches(crd)); awaitPartitionMapExchange(); } - /** - * @param observer Node that is neither the coordinator nor the new primary of the lost partitions it checks. - * @param lostParts Lost partitions. - * @return Lost partitions whose owners, as the observer sees them, are not just their primary. - */ - private List ownerMismatches(IgniteEx observer, int[] lostParts) { - GridDhtPartitionTopology top = observer.cachex(CACHE).context().topology(); - - List res = new ArrayList<>(); - - for (int p : lostParts) { - ClusterNode primary = observer.affinity(CACHE).mapPartitionToNode(p); - - if (primary.isLocal()) - continue; - - List owners = top.owners(p); - - if (!owners.equals(Collections.singletonList(primary))) - res.add("p=" + p + " owners=" + ClusterNodeFunc.nodeIds(owners) + " primary=" + primary.id()); - } - - return res; - } - /** * @return Resolver that makes the new primary send its partition map right before it owns a MOVING partition of * the cache, once {@link #sendBeforeOwn} is set. From 92d046e1bbbf533fe409b9d887dbf3b3e18b8051 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 30 Sep 2026 13:11:44 +0300 Subject: [PATCH 05/27] IGNITE-29088 Fix comment --- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 6c4b93134fcd1..3ea00fe8b8da5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3525,9 +3525,8 @@ private void detectLostPartitions(AffinityTopologyVersion resTopVer) { throw new IgniteException(e); } - // Local partitions changed their states after this node had sent its partition map: a lost partition was - // owned under the IGNORE policy or marked LOST. Send the maps again, otherwise the coordinator can keep - // a state the node reported earlier. + // Detecting lost partitions changed local partition states: a lost partition was owned under the IGNORE policy + // or marked LOST. Send the maps again, otherwise the coordinator can keep a state this node reported before. if (!changedGrps.isEmpty()) { if (log.isDebugEnabled()) { log.debug("Refresh partitions due to lost partitions detected [grps=" + From 44a9132b7018ded15a8d66f7ca7acfedd18ee28a Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 30 Sep 2026 13:43:56 +0300 Subject: [PATCH 06/27] IGNITE-29088 Don't resend from the coordinator --- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 3ea00fe8b8da5..d3e6247384978 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3527,7 +3527,11 @@ private void detectLostPartitions(AffinityTopologyVersion resTopVer) { // Detecting lost partitions changed local partition states: a lost partition was owned under the IGNORE policy // or marked LOST. Send the maps again, otherwise the coordinator can keep a state this node reported before. - if (!changedGrps.isEmpty()) { + // The coordinator itself doesn't need to: it keeps its own states in its map, and every node sets the same + // states for the other nodes when it detects the lost partitions. + boolean crdNode = crd != null && crd.isLocal(); + + if (!crdNode && !changedGrps.isEmpty()) { if (log.isDebugEnabled()) { log.debug("Refresh partitions due to lost partitions detected [grps=" + changedGrps.stream().map(CacheGroupContext::cacheOrGroupName).collect(Collectors.toList()) + ']'); From 9877eed0d4f1d96a615e8787d2723cf7c550bf4b Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 30 Sep 2026 13:58:32 +0300 Subject: [PATCH 07/27] IGNITE-29088 Update test javadoc --- .../distributed/CachePartitionLossIgnorePolicyMapTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java index ed00db5c23834..367b0e8e02d7b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java @@ -55,6 +55,12 @@ * Each lost partition is recreated empty on its new primary as MOVING and owned there when the node detects the loss. * If the new primary sends its partition map between these two steps, the map carries MOVING and overwrites what the * coordinator already knows, so the node must send its map again once it owns the partition. + *

+ * The window is short, and in a real cluster a map gets into it by chance: a resend scheduled by + * {@code scheduleResendPartitions()} (after an eviction, a partition moving to RENTING or a partition map change) fires + * there. In {@code IgniteTopologyValidatorGridSplitCacheTest} (32 nodes, 50 caches without backups) this happened in + * about half of the runs. The test makes the same send happen in the window every time: the new primary refreshes its + * partition map right before it owns a lost partition. */ public class CachePartitionLossIgnorePolicyMapTest extends GridCommonAbstractTest { /** */ From 08219cbc7ad3048002f45db054e3602c3836e1df Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 7 Oct 2026 15:07:15 +0300 Subject: [PATCH 08/27] IGNITE-29088 Fix after review --- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 6 ++++-- .../CachePartitionLossIgnorePolicyMapTest.java | 9 ++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index d3e6247384978..6211177fb8ec5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3527,8 +3527,10 @@ private void detectLostPartitions(AffinityTopologyVersion resTopVer) { // Detecting lost partitions changed local partition states: a lost partition was owned under the IGNORE policy // or marked LOST. Send the maps again, otherwise the coordinator can keep a state this node reported before. - // The coordinator itself doesn't need to: it keeps its own states in its map, and every node sets the same - // states for the other nodes when it detects the lost partitions. + // The coordinator doesn't resend: its own states are already in its map, and every node sets the same states + // for the other nodes when it detects the lost partitions. A full map the coordinator sends before that can + // still reach a node that has finished the exchange, but only if the coordinator stalls between sending the + // full message and detecting lost partitions. boolean crdNode = crd != null && crd.isLocal(); if (!crdNode && !changedGrps.isEmpty()) { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java index 367b0e8e02d7b..60d859bdd8ced 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java @@ -56,11 +56,10 @@ * If the new primary sends its partition map between these two steps, the map carries MOVING and overwrites what the * coordinator already knows, so the node must send its map again once it owns the partition. *

- * The window is short, and in a real cluster a map gets into it by chance: a resend scheduled by - * {@code scheduleResendPartitions()} (after an eviction, a partition moving to RENTING or a partition map change) fires - * there. In {@code IgniteTopologyValidatorGridSplitCacheTest} (32 nodes, 50 caches without backups) this happened in - * about half of the runs. The test makes the same send happen in the window every time: the new primary refreshes its - * partition map right before it owns a lost partition. + * The window spans most of the exchange on the new primary: it opens when the node applies the coordinator's full + * message and creates the partition, and closes when the node detects lost partitions at the end of the exchange. In a + * real cluster a map gets into it when a resend scheduled by {@code scheduleResendPartitions()} (after an eviction, a + * partition moving to RENTING or a partition map change) fires there. */ public class CachePartitionLossIgnorePolicyMapTest extends GridCommonAbstractTest { /** */ From b4faad17a1666a1360ec8eec68b3c62a094d5cd9 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 7 Oct 2026 15:11:01 +0300 Subject: [PATCH 09/27] IGNITE-29088 Clarify coordinator comment --- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 6211177fb8ec5..638dedd199f0b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3528,9 +3528,10 @@ private void detectLostPartitions(AffinityTopologyVersion resTopVer) { // Detecting lost partitions changed local partition states: a lost partition was owned under the IGNORE policy // or marked LOST. Send the maps again, otherwise the coordinator can keep a state this node reported before. // The coordinator doesn't resend: its own states are already in its map, and every node sets the same states - // for the other nodes when it detects the lost partitions. A full map the coordinator sends before that can - // still reach a node that has finished the exchange, but only if the coordinator stalls between sending the - // full message and detecting lost partitions. + // for the other nodes when it detects the lost partitions. A scheduled full-map resend from the coordinator + // that falls between its full message and its own detection can still leave a node that has finished the + // exchange with the coordinator's partition MOVING, and nothing resends after it. That needs the coordinator + // to stall between sending the full message and detecting lost partitions. boolean crdNode = crd != null && crd.isLocal(); if (!crdNode && !changedGrps.isEmpty()) { From 21a22624402366e8d8c48b1f2a35f364072233b7 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 8 Oct 2026 14:00:12 +0300 Subject: [PATCH 10/27] IGNITE-29088 Add READ_WRITE_SAFE case --- ...va => CachePartitionLossStaleMapTest.java} | 81 ++++++++++++++----- .../testsuites/IgniteCacheTestSuite15.java | 4 +- 2 files changed, 65 insertions(+), 20 deletions(-) rename modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/{CachePartitionLossIgnorePolicyMapTest.java => CachePartitionLossStaleMapTest.java} (76%) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java similarity index 76% rename from modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java rename to modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java index 60d859bdd8ced..e22e2610d650d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossIgnorePolicyMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java @@ -24,6 +24,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.PartitionLossPolicy; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -44,24 +45,26 @@ import org.junit.Test; import static org.apache.ignite.cache.PartitionLossPolicy.IGNORE; +import static org.apache.ignite.cache.PartitionLossPolicy.READ_WRITE_SAFE; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING; import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; /** * Checks that the coordinator's partition map agrees with the nodes after a node leaves with the only copy of its - * partitions under the {@link org.apache.ignite.cache.PartitionLossPolicy#IGNORE IGNORE} loss policy. + * partitions. *

- * Each lost partition is recreated empty on its new primary as MOVING and owned there when the node detects the loss. - * If the new primary sends its partition map between these two steps, the map carries MOVING and overwrites what the - * coordinator already knows, so the node must send its map again once it owns the partition. + * Each lost partition is recreated empty on its new primary as MOVING. When the node detects the loss, it owns the + * partition under the {@link PartitionLossPolicy#IGNORE IGNORE} policy or marks it LOST under the safe policies. If the + * new primary sends its partition map between these two steps, the map carries MOVING and overwrites what the + * coordinator already knows, so the node must send its map again after the change. *

* The window spans most of the exchange on the new primary: it opens when the node applies the coordinator's full * message and creates the partition, and closes when the node detects lost partitions at the end of the exchange. In a * real cluster a map gets into it when a resend scheduled by {@code scheduleResendPartitions()} (after an eviction, a * partition moving to RENTING or a partition map change) fires there. */ -public class CachePartitionLossIgnorePolicyMapTest extends GridCommonAbstractTest { +public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { /** */ private static final String CACHE = "cache"; @@ -74,8 +77,14 @@ public class CachePartitionLossIgnorePolicyMapTest extends GridCommonAbstractTes /** Index of the node that leaves. */ private static final int LEAVING = 2; - /** When set, the new primary sends its partition map right before it owns a MOVING partition of the cache. */ - private final AtomicBoolean sendBeforeOwn = new AtomicBoolean(); + /** + * When set, the new primary sends its partition map right before it owns a MOVING partition of the cache or marks + * it LOST. + */ + private final AtomicBoolean sendBeforeChange = new AtomicBoolean(); + + /** Partition loss policy of the cache. */ + private PartitionLossPolicy plc; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { @@ -83,7 +92,7 @@ public class CachePartitionLossIgnorePolicyMapTest extends GridCommonAbstractTes .setCommunicationSpi(new TestRecordingCommunicationSpi()) .setCacheConfiguration(new CacheConfiguration(CACHE) .setBackups(0) - .setPartitionLossPolicy(IGNORE) + .setPartitionLossPolicy(plc) .setAffinity(new RendezvousAffinityFunction(false, PARTS))); } @@ -95,16 +104,38 @@ public class CachePartitionLossIgnorePolicyMapTest extends GridCommonAbstractTes } /** - * A node joins and takes partitions over, the old owners evict their copies, and the node leaves. Its partitions - * are lost, and the new primary sends a stale map, with the lost partition still MOVING, before it owns them. + * The new primary sends a stale map before it owns a lost partition under the IGNORE policy. * * @throws Exception If failed. */ @Test public void testStaleMapBeforeLostPartitionOwned() throws Exception { + checkStaleMapBeforeLostPartitionChanged(IGNORE); + } + + /** + * The new primary sends a stale map before it marks a lost partition LOST under the READ_WRITE_SAFE policy. + * + * @throws Exception If failed. + */ + @Test + public void testStaleMapBeforeLostPartitionMarkedLost() throws Exception { + checkStaleMapBeforeLostPartitionChanged(READ_WRITE_SAFE); + } + + /** + * A node joins and takes partitions over, the old owners evict their copies, and the node leaves. Its partitions + * are lost, and the new primary sends a stale map, with a lost partition still MOVING, before it changes them. + * + * @param plc Partition loss policy. + * @throws Exception If failed. + */ + private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) throws Exception { + this.plc = plc; + IgniteEx crd = startGrid(0); - startGrid(NEW_PRIMARY, sendBeforeOwnResolver()); + startGrid(NEW_PRIMARY, sendBeforeChangeResolver()); IgniteCache cache = crd.cache(CACHE); @@ -124,7 +155,7 @@ public void testStaleMapBeforeLostPartitionOwned() throws Exception { && msg instanceof GridDhtPartitionsSingleMessage && ((GridDhtPartitionsSingleMessage)msg).exchangeId() == null); - sendBeforeOwn.set(true); + sendBeforeChange.set(true); long leaveVer = crd.cluster().topologyVersion() + 1; @@ -132,7 +163,7 @@ public void testStaleMapBeforeLostPartitionOwned() throws Exception { newPrimarySpi.waitForBlocked(); - assertFalse("The new primary didn't own a lost partition", sendBeforeOwn.get()); + assertFalse("The new primary didn't change a lost partition", sendBeforeChange.get()); assertTrue(waitForCondition(() -> { GridDhtPartitionsExchangeFuture fut = crd.context().cache().context().exchange().lastFinishedFuture(); @@ -145,14 +176,17 @@ public void testStaleMapBeforeLostPartitionOwned() throws Exception { if (!waitForCondition(() -> mismatches(crd).isEmpty(), 10_000)) fail("The coordinator's partition map differs from the nodes: " + mismatches(crd)); + if (plc != IGNORE) + crd.resetLostPartitions(Collections.singleton(CACHE)); + awaitPartitionMapExchange(); } /** * @return Resolver that makes the new primary send its partition map right before it owns a MOVING partition of - * the cache, once {@link #sendBeforeOwn} is set. + * the cache or marks it LOST, once {@link #sendBeforeChange} is set. */ - private TestDependencyResolver sendBeforeOwnResolver() { + private TestDependencyResolver sendBeforeChangeResolver() { return new TestDependencyResolver(new DependencyResolver() { @Override public T resolve(T instance) { if (instance instanceof GridDhtPartitionTopologyImpl) { @@ -160,12 +194,23 @@ private TestDependencyResolver sendBeforeOwnResolver() { top.partitionFactory((ctx, grp, id, recovery) -> new GridDhtLocalPartition(ctx, grp, id, recovery) { @Override public boolean own() { - if (CACHE.equals(grp.cacheOrGroupName()) && state() == MOVING - && sendBeforeOwn.compareAndSet(true, false)) - ctx.exchange().refreshPartitions(Collections.singleton(grp)); + sendIfArmed(); return super.own(); } + + @Override public boolean markLost() { + sendIfArmed(); + + return super.markLost(); + } + + /** Sends the partition map once, if this is a MOVING partition of the cache. */ + private void sendIfArmed() { + if (CACHE.equals(grp.cacheOrGroupName()) && state() == MOVING + && sendBeforeChange.compareAndSet(true, false)) + ctx.exchange().refreshPartitions(Collections.singleton(grp)); + } }); } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java index 6938d8f2fb5c0..d38d9154e4887 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java @@ -37,7 +37,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheStoreCollectionTest; import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentNodeJoinValidationTest; import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentTest; -import org.apache.ignite.internal.processors.cache.distributed.CachePartitionLossIgnorePolicyMapTest; +import org.apache.ignite.internal.processors.cache.distributed.CachePartitionLossStaleMapTest; import org.apache.ignite.internal.processors.cache.distributed.GridExchangeFreeCellularSwitchComplexOperationsTest; import org.apache.ignite.internal.processors.cache.distributed.GridExchangeFreeCellularSwitchIsolationTest; import org.apache.ignite.internal.processors.cache.distributed.GridExchangeFreeCellularSwitchTxContinuationTest; @@ -99,7 +99,7 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, ClusterActivationFailureTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheCreateDestroyClusterReadOnlyModeTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteCachePartitionLossPolicySelfTest.class, ignoredTests); - GridTestUtils.addTestIfNeeded(suite, CachePartitionLossIgnorePolicyMapTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CachePartitionLossStaleMapTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteCacheGroupsPartitionLossPolicySelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheRebalancingSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheManualRebalancingTest.class, ignoredTests); From 58a3349b9947c815b04f79d4dd80befa961a0df8 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 8 Oct 2026 14:12:44 +0300 Subject: [PATCH 11/27] IGNITE-29088 Add coordinator case --- .../CachePartitionLossStaleMapTest.java | 238 ++++++++++++++++-- 1 file changed, 218 insertions(+), 20 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java index e22e2610d650d..a3fbb2bdb7409 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java @@ -19,35 +19,51 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.UUID; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteException; import org.apache.ignite.cache.PartitionLossPolicy; +import org.apache.ignite.cache.affinity.AffinityFunction; +import org.apache.ignite.cache.affinity.AffinityFunctionContext; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.NodeStoppingException; import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.EvictionContext; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl; import org.apache.ignite.internal.processors.resource.DependencyResolver; import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.testframework.TestDependencyResolver; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.ignite.cache.PartitionLossPolicy.IGNORE; import static org.apache.ignite.cache.PartitionLossPolicy.READ_WRITE_SAFE; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING; import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; /** @@ -83,21 +99,49 @@ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { */ private final AtomicBoolean sendBeforeChange = new AtomicBoolean(); + /** When set, the next eviction of a partition of the cache waits for {@link #evictionResume}. */ + private final AtomicBoolean holdEviction = new AtomicBoolean(); + + /** Counted down when an eviction starts waiting. */ + private final CountDownLatch evictionHeld = new CountDownLatch(1); + + /** Lets the held eviction go on. */ + private final CountDownLatch evictionResume = new CountDownLatch(1); + + /** When set, the coordinator waits for {@link #crdResume} right after it sends the next exchange full message. */ + private final AtomicBoolean stallAfterFullMessage = new AtomicBoolean(); + + /** Counted down when the coordinator starts waiting. */ + private final CountDownLatch crdStalled = new CountDownLatch(1); + + /** Lets the coordinator go on. */ + private final CountDownLatch crdResume = new CountDownLatch(1); + /** Partition loss policy of the cache. */ - private PartitionLossPolicy plc; + private PartitionLossPolicy plc = IGNORE; + + /** Whether the cache uses {@link JoinOrderAffinityFunction}. */ + private boolean joinOrderAff; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { return super.getConfiguration(igniteInstanceName) - .setCommunicationSpi(new TestRecordingCommunicationSpi()) + .setCommunicationSpi(new StallingCommunicationSpi()) + // Other evictions don't queue behind a held one. + .setRebalanceThreadPoolSize(4) .setCacheConfiguration(new CacheConfiguration(CACHE) .setBackups(0) .setPartitionLossPolicy(plc) - .setAffinity(new RendezvousAffinityFunction(false, PARTS))); + .setAffinity(joinOrderAff + ? new JoinOrderAffinityFunction() + : new RendezvousAffinityFunction(false, PARTS))); } /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { + evictionResume.countDown(); + crdResume.countDown(); + stopAllGrids(); super.afterTest(); @@ -135,7 +179,7 @@ private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) th IgniteEx crd = startGrid(0); - startGrid(NEW_PRIMARY, sendBeforeChangeResolver()); + startGrid(NEW_PRIMARY, partitionHooksResolver()); IgniteCache cache = crd.cache(CACHE); @@ -165,11 +209,7 @@ private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) th assertFalse("The new primary didn't change a lost partition", sendBeforeChange.get()); - assertTrue(waitForCondition(() -> { - GridDhtPartitionsExchangeFuture fut = crd.context().cache().context().exchange().lastFinishedFuture(); - - return fut != null && fut.topologyVersion().topologyVersion() >= leaveVer; - }, getTestTimeout())); + assertTrue(waitForCondition(() -> exchangeFinished(crd, leaveVer), getTestTimeout())); newPrimarySpi.stopBlock(); @@ -183,10 +223,86 @@ private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) th } /** - * @return Resolver that makes the new primary send its partition map right before it owns a MOVING partition of - * the cache or marks it LOST, once {@link #sendBeforeChange} is set. + * The coordinator itself is the new primary of all lost partitions. It creates them before it sends the full + * message, so a plain resend carries the same map version and the other node ignores it. Here the coordinator + * stalls after it sends the full message, as when sending to the next node is slow, and an eviction finishes there + * meanwhile. The resend scheduled after the eviction carries a newer map with the lost partitions still MOVING, + * and the other node, which has already finished the exchange, takes it. The coordinator must send its map again + * once it owns the partitions. + *

+ * The other node gets no lost partitions: otherwise its own resend after detecting them makes the coordinator + * resend its map later, which hides the problem. + * + * @throws Exception If failed. */ - private TestDependencyResolver sendBeforeChangeResolver() { + @Test + public void testStaleCoordinatorMapAfterEviction() throws Exception { + joinOrderAff = true; + + IgniteEx crd = startGrid(0, partitionHooksResolver()); + + IgniteCache cache = crd.cache(CACHE); + + for (int i = 0; i < PARTS * 4; i++) + cache.put(i, i); + + // The next eviction on the coordinator waits until the exchange, as the eviction of a large partition would. + holdEviction.set(true); + + IgniteEx other = startGrid(1); + + awaitPartitionMapExchange(); + + assertTrue("The coordinator didn't start an eviction", evictionHeld.await(getTestTimeout(), MILLISECONDS)); + + startGrid(LEAVING); + + awaitPartitionMapExchange(); + + GridDhtPartitionTopology crdTop = crd.cachex(CACHE).context().topology(); + + // Only the held eviction is left, so the resend is scheduled when it finishes. + assertTrue(waitForCondition(() -> crdTop.localPartitions().stream() + .filter(p -> p.state() == RENTING).count() == 1, getTestTimeout())); + + UUID crdId = crd.localNode().id(); + + stallAfterFullMessage.set(true); + + long leaveVer = crd.cluster().topologyVersion() + 1; + + stopGrid(LEAVING); + + assertTrue("The coordinator didn't send the full message", crdStalled.await(getTestTimeout(), MILLISECONDS)); + + assertTrue(waitForCondition(() -> exchangeFinished(other, leaveVer), getTestTimeout())); + + evictionResume.countDown(); + + GridDhtPartitionTopology otherTop = other.cachex(CACHE).context().topology(); + + assertTrue("The other node didn't take a stale map of the coordinator", waitForCondition(() -> { + GridDhtPartitionMap crdMap = otherTop.partitionMap(false).get(crdId); + + return crdMap != null && crdMap.hasMovingPartitions(); + }, 10_000)); + + crdResume.countDown(); + + assertTrue(waitForCondition(() -> exchangeFinished(crd, leaveVer), getTestTimeout())); + + if (!waitForCondition(() -> mismatches(other).isEmpty(), 10_000)) + fail("The other node's partition map differs from the nodes: " + mismatches(other)); + + awaitPartitionMapExchange(); + } + + /** + * @return Resolver whose partitions send the partition map right before a MOVING partition of the cache is owned or + * marked LOST, once {@link #sendBeforeChange} is set, and hold the next eviction of a partition of the cache, once + * {@link #holdEviction} is set. + */ + private TestDependencyResolver partitionHooksResolver() { return new TestDependencyResolver(new DependencyResolver() { @Override public T resolve(T instance) { if (instance instanceof GridDhtPartitionTopologyImpl) { @@ -205,6 +321,18 @@ private TestDependencyResolver sendBeforeChangeResolver() { return super.markLost(); } + /** {@inheritDoc} */ + @Override protected long clearAll(EvictionContext evictionCtx) throws NodeStoppingException { + if (CACHE.equals(grp.cacheOrGroupName()) && state() == RENTING + && holdEviction.compareAndSet(true, false)) { + evictionHeld.countDown(); + + U.awaitQuiet(evictionResume); + } + + return super.clearAll(evictionCtx); + } + /** Sends the partition map once, if this is a MOVING partition of the cache. */ private void sendIfArmed() { if (CACHE.equals(grp.cacheOrGroupName()) && state() == MOVING @@ -220,18 +348,29 @@ private void sendIfArmed() { } /** - * @param crd Coordinator. - * @return Partitions whose state in the coordinator's map differs from the state on the node itself. + * @param node Node. + * @param topVer Topology version. + * @return Whether the node has finished the exchange for the topology version. */ - private List mismatches(IgniteEx crd) { - GridDhtPartitionTopology crdTop = crd.cachex(CACHE).context().topology(); + private boolean exchangeFinished(IgniteEx node, long topVer) { + GridDhtPartitionsExchangeFuture fut = node.context().cache().context().exchange().lastFinishedFuture(); + + return fut != null && fut.topologyVersion().topologyVersion() >= topVer; + } + + /** + * @param viewer Node whose partition map is checked. + * @return Partitions whose state in the viewer's map differs from the state on the node itself. + */ + private List mismatches(IgniteEx viewer) { + GridDhtPartitionTopology viewerTop = viewer.cachex(CACHE).context().topology(); List res = new ArrayList<>(); for (Ignite g : G.allGrids()) { UUID id = g.cluster().localNode().id(); - GridDhtPartitionMap crdView = crdTop.partitionMap(false).get(id); + GridDhtPartitionMap view = viewerTop.partitionMap(false).get(id); GridDhtPartitionTopology top = ((IgniteEx)g).cachex(CACHE).context().topology(); @@ -239,17 +378,76 @@ private List mismatches(IgniteEx crd) { GridDhtLocalPartition locPart = top.localPartition(p, AffinityTopologyVersion.NONE, false, true); GridDhtPartitionState loc = locPart == null ? null : locPart.state(); - GridDhtPartitionState seen = crdView == null ? null : crdView.get(p); + GridDhtPartitionState seen = view == null ? null : view.get(p); - // The coordinator keeps evicted partitions in its map, the node doesn't. + // The map keeps evicted partitions, the node doesn't. if (seen == EVICTED) seen = null; if (loc != seen) - res.add(g.name() + " p=" + p + " local=" + loc + " crd=" + seen); + res.add(g.name() + " p=" + p + " local=" + loc + " seen=" + seen); } } return res; } + + /** Holds the coordinator right after it sends the full message of an exchange, once armed. */ + private class StallingCommunicationSpi extends TestRecordingCommunicationSpi { + /** {@inheritDoc} */ + @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure ackC) + throws IgniteSpiException { + super.sendMessage(node, msg, ackC); + + if (msg instanceof GridIoMessage + && ((GridIoMessage)msg).message() instanceof GridDhtPartitionsFullMessage + && ((GridDhtPartitionsFullMessage)((GridIoMessage)msg).message()).exchangeId() != null + && stallAfterFullMessage.compareAndSet(true, false)) { + crdStalled.countDown(); + + U.awaitQuiet(crdResume); + } + } + } + + /** + * Assigns partition {@code p} to the server node with index {@code p % 3} in join order, or to the oldest node if + * there is no such node. The third node takes the partitions with {@code p % 3 == 2} from the oldest node and gives + * them back when it leaves. + */ + private static class JoinOrderAffinityFunction implements AffinityFunction { + /** {@inheritDoc} */ + @Override public void reset() { + // No-op. + } + + /** {@inheritDoc} */ + @Override public int partitions() { + return PARTS; + } + + /** {@inheritDoc} */ + @Override public int partition(Object key) { + return U.safeAbs(key.hashCode()) % PARTS; + } + + /** {@inheritDoc} */ + @Override public List> assignPartitions(AffinityFunctionContext ctx) { + List nodes = new ArrayList<>(ctx.currentTopologySnapshot()); + + nodes.sort(Comparator.comparingLong(ClusterNode::order)); + + List> res = new ArrayList<>(PARTS); + + for (int p = 0; p < PARTS; p++) + res.add(Collections.singletonList(nodes.get(p % 3 < nodes.size() ? p % 3 : 0))); + + return res; + } + + /** {@inheritDoc} */ + @Override public void removeNode(UUID nodeId) { + // No-op. + } + } } From 1f47f918ceffa979a5da4147e126711ab0eec361 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 8 Oct 2026 14:26:20 +0300 Subject: [PATCH 12/27] IGNITE-29088 Resend from the coordinator too --- .../preloader/GridDhtPartitionsExchangeFuture.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 638dedd199f0b..9e4708249ae94 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3526,15 +3526,9 @@ private void detectLostPartitions(AffinityTopologyVersion resTopVer) { } // Detecting lost partitions changed local partition states: a lost partition was owned under the IGNORE policy - // or marked LOST. Send the maps again, otherwise the coordinator can keep a state this node reported before. - // The coordinator doesn't resend: its own states are already in its map, and every node sets the same states - // for the other nodes when it detects the lost partitions. A scheduled full-map resend from the coordinator - // that falls between its full message and its own detection can still leave a node that has finished the - // exchange with the coordinator's partition MOVING, and nothing resends after it. That needs the coordinator - // to stall between sending the full message and detecting lost partitions. - boolean crdNode = crd != null && crd.isLocal(); - - if (!crdNode && !changedGrps.isEmpty()) { + // or marked LOST. Send the maps again: a map this node sent before the change can override the new states on + // other nodes. + if (!changedGrps.isEmpty()) { if (log.isDebugEnabled()) { log.debug("Refresh partitions due to lost partitions detected [grps=" + changedGrps.stream().map(CacheGroupContext::cacheOrGroupName).collect(Collectors.toList()) + ']'); From 2205279dccc4cb3622420f6df9b7503a4b599ad5 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 8 Oct 2026 14:27:11 +0300 Subject: [PATCH 13/27] IGNITE-29088 Fix javadocs --- .../dht/topology/GridDhtPartitionTopology.java | 2 +- .../distributed/CachePartitionLossStaleMapTest.java | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java index 0c2f415a7a1c1..903967f246a2c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java @@ -347,7 +347,7 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId, * @param resTopVer Exchange result version. * @param fut Exchange futute for topology events to detect. * - * @return {@code True} if partitions state got updated. + * @return {@code True} if the state of at least one local partition changed. */ public boolean detectLostPartitions(AffinityTopologyVersion resTopVer, GridDhtPartitionsExchangeFuture fut); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java index a3fbb2bdb7409..2e0f027f33352 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java @@ -75,10 +75,11 @@ * new primary sends its partition map between these two steps, the map carries MOVING and overwrites what the * coordinator already knows, so the node must send its map again after the change. *

- * The window spans most of the exchange on the new primary: it opens when the node applies the coordinator's full - * message and creates the partition, and closes when the node detects lost partitions at the end of the exchange. In a - * real cluster a map gets into it when a resend scheduled by {@code scheduleResendPartitions()} (after an eviction, a - * partition moving to RENTING or a partition map change) fires there. + * The window is the end of the exchange on the new primary: it opens when the node applies the coordinator's full + * message and creates the partition, and closes when the node detects lost partitions. In a real cluster a map gets + * into it when a resend scheduled by {@code scheduleResendPartitions()} (after an eviction, a partition moving to + * RENTING or a partition map change) fires there. The coordinator as the new primary is checked by + * {@link #testStaleCoordinatorMapAfterEviction()}. */ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { /** */ From 7f2771cee12e03be7231125ddb12614406ce073d Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 8 Oct 2026 14:46:43 +0300 Subject: [PATCH 14/27] IGNITE-29088 Fix test after review --- .../GridDhtPartitionsExchangeFuture.java | 2 +- .../CachePartitionLossStaleMapTest.java | 72 ++++++++++++------- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 9e4708249ae94..9ebb19a6c515c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3497,7 +3497,7 @@ private void findCounterForReservation( /** * Detect lost partitions in case of node left or failed. For topology coordinator is called when all {@link * GridDhtPartitionsSingleMessage} were received. For other nodes is called when exchange future is completed by - * {@link GridDhtPartitionsFullMessage}. + * {@link GridDhtPartitionsFullMessage}. If a local partition changes, sends the partition maps of its group again. * * @param resTopVer Result topology version. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java index 2e0f027f33352..93ecf361bc069 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java @@ -67,8 +67,8 @@ import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; /** - * Checks that the coordinator's partition map agrees with the nodes after a node leaves with the only copy of its - * partitions. + * Checks that partition maps agree with the local partition states of the nodes after a node leaves with the only copy + * of its partitions. *

* Each lost partition is recreated empty on its new primary as MOVING. When the node detects the loss, it owns the * partition under the {@link PartitionLossPolicy#IGNORE IGNORE} policy or marks it LOST under the safe policies. If the @@ -94,6 +94,9 @@ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { /** Index of the node that leaves. */ private static final int LEAVING = 2; + /** Time to wait for a step of a test, in milliseconds. */ + private static final long WAIT = 30_000; + /** * When set, the new primary sends its partition map right before it owns a MOVING partition of the cache or marks * it LOST. @@ -121,21 +124,34 @@ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { /** Partition loss policy of the cache. */ private PartitionLossPolicy plc = IGNORE; - /** Whether the cache uses {@link JoinOrderAffinityFunction}. */ - private boolean joinOrderAff; + /** + * Whether the coordinator becomes the new primary of all lost partitions: the cache uses + * {@link JoinOrderAffinityFunction}, and the coordinator can stall after it sends a full message. + */ + private boolean crdNewPrimary; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - return super.getConfiguration(igniteInstanceName) - .setCommunicationSpi(new StallingCommunicationSpi()) - // Other evictions don't queue behind a held one. - .setRebalanceThreadPoolSize(4) - .setCacheConfiguration(new CacheConfiguration(CACHE) - .setBackups(0) - .setPartitionLossPolicy(plc) - .setAffinity(joinOrderAff - ? new JoinOrderAffinityFunction() - : new RendezvousAffinityFunction(false, PARTS))); + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setCacheConfiguration(new CacheConfiguration(CACHE) + .setBackups(0) + .setPartitionLossPolicy(plc) + .setAffinity(crdNewPrimary + ? new JoinOrderAffinityFunction() + : new RendezvousAffinityFunction(false, PARTS))); + + if (crdNewPrimary) { + cfg.setCommunicationSpi(new StallingCommunicationSpi()); + + // By default the pool has one thread on machines with fewer than 8 cores, and the other evictions would + // wait behind the held one. + cfg.setRebalanceThreadPoolSize(4); + } + else + cfg.setCommunicationSpi(new TestRecordingCommunicationSpi()); + + return cfg; } /** {@inheritDoc} */ @@ -206,11 +222,11 @@ private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) th stopGrid(LEAVING); - newPrimarySpi.waitForBlocked(); + assertTrue("The new primary didn't change a lost partition", newPrimarySpi.waitForBlocked(1, WAIT)); assertFalse("The new primary didn't change a lost partition", sendBeforeChange.get()); - assertTrue(waitForCondition(() -> exchangeFinished(crd, leaveVer), getTestTimeout())); + assertTrue(waitForCondition(() -> exchangeFinished(crd, leaveVer), WAIT)); newPrimarySpi.stopBlock(); @@ -226,10 +242,9 @@ private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) th /** * The coordinator itself is the new primary of all lost partitions. It creates them before it sends the full * message, so a plain resend carries the same map version and the other node ignores it. Here the coordinator - * stalls after it sends the full message, as when sending to the next node is slow, and an eviction finishes there - * meanwhile. The resend scheduled after the eviction carries a newer map with the lost partitions still MOVING, - * and the other node, which has already finished the exchange, takes it. The coordinator must send its map again - * once it owns the partitions. + * stalls after it sends the full message, as when sending to the next node is slow, and an eviction finishes on the + * coordinator meanwhile. The resend scheduled after the eviction carries a newer map with the lost partitions still + * MOVING, and the other node takes it. The coordinator must send its map again once it owns the partitions. *

* The other node gets no lost partitions: otherwise its own resend after detecting them makes the coordinator * resend its map later, which hides the problem. @@ -238,7 +253,7 @@ private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) th */ @Test public void testStaleCoordinatorMapAfterEviction() throws Exception { - joinOrderAff = true; + crdNewPrimary = true; IgniteEx crd = startGrid(0, partitionHooksResolver()); @@ -254,7 +269,7 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { awaitPartitionMapExchange(); - assertTrue("The coordinator didn't start an eviction", evictionHeld.await(getTestTimeout(), MILLISECONDS)); + assertTrue("The coordinator didn't start an eviction", evictionHeld.await(WAIT, MILLISECONDS)); startGrid(LEAVING); @@ -264,7 +279,7 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { // Only the held eviction is left, so the resend is scheduled when it finishes. assertTrue(waitForCondition(() -> crdTop.localPartitions().stream() - .filter(p -> p.state() == RENTING).count() == 1, getTestTimeout())); + .filter(p -> p.state() == RENTING).count() == 1, WAIT)); UUID crdId = crd.localNode().id(); @@ -274,9 +289,9 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { stopGrid(LEAVING); - assertTrue("The coordinator didn't send the full message", crdStalled.await(getTestTimeout(), MILLISECONDS)); + assertTrue("The coordinator didn't send the full message", crdStalled.await(WAIT, MILLISECONDS)); - assertTrue(waitForCondition(() -> exchangeFinished(other, leaveVer), getTestTimeout())); + assertTrue(waitForCondition(() -> exchangeFinished(other, leaveVer), WAIT)); evictionResume.countDown(); @@ -290,7 +305,7 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { crdResume.countDown(); - assertTrue(waitForCondition(() -> exchangeFinished(crd, leaveVer), getTestTimeout())); + assertTrue(waitForCondition(() -> exchangeFinished(crd, leaveVer), WAIT)); if (!waitForCondition(() -> mismatches(other).isEmpty(), 10_000)) fail("The other node's partition map differs from the nodes: " + mismatches(other)); @@ -310,12 +325,14 @@ private TestDependencyResolver partitionHooksResolver() { GridDhtPartitionTopologyImpl top = (GridDhtPartitionTopologyImpl)instance; top.partitionFactory((ctx, grp, id, recovery) -> new GridDhtLocalPartition(ctx, grp, id, recovery) { + /** {@inheritDoc} */ @Override public boolean own() { sendIfArmed(); return super.own(); } + /** {@inheritDoc} */ @Override public boolean markLost() { sendIfArmed(); @@ -417,6 +434,9 @@ private class StallingCommunicationSpi extends TestRecordingCommunicationSpi { * them back when it leaves. */ private static class JoinOrderAffinityFunction implements AffinityFunction { + /** */ + private static final long serialVersionUID = 0L; + /** {@inheritDoc} */ @Override public void reset() { // No-op. From 9331f4411b286efc19dc8e3ca717ec73d56a0e3a Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 8 Oct 2026 19:24:39 +0300 Subject: [PATCH 15/27] IGNITE-29088 Fix comments after review --- .../GridDhtPartitionsExchangeFuture.java | 3 - .../CachePartitionLossStaleMapTest.java | 80 ++++++++++--------- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 9ebb19a6c515c..28bc1b280ada3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3525,9 +3525,6 @@ private void detectLostPartitions(AffinityTopologyVersion resTopVer) { throw new IgniteException(e); } - // Detecting lost partitions changed local partition states: a lost partition was owned under the IGNORE policy - // or marked LOST. Send the maps again: a map this node sent before the change can override the new states on - // other nodes. if (!changedGrps.isEmpty()) { if (log.isDebugEnabled()) { log.debug("Refresh partitions due to lost partitions detected [grps=" + diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java index 93ecf361bc069..92980c69f06ef 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java @@ -67,19 +67,18 @@ import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; /** - * Checks that partition maps agree with the local partition states of the nodes after a node leaves with the only copy - * of its partitions. + * Checks that partition maps match the local partition states of the nodes after a node leaves with the only copy of + * some partitions. *

- * Each lost partition is recreated empty on its new primary as MOVING. When the node detects the loss, it owns the - * partition under the {@link PartitionLossPolicy#IGNORE IGNORE} policy or marks it LOST under the safe policies. If the - * new primary sends its partition map between these two steps, the map carries MOVING and overwrites what the - * coordinator already knows, so the node must send its map again after the change. + * The new primary recreates each lost partition empty as MOVING when it applies the coordinator's full message. At + * the end of the exchange it owns the partition under the {@link PartitionLossPolicy#IGNORE IGNORE} policy or marks + * it LOST under the safe policies. If the node sends its partition map between these two steps, the map shows the + * partition MOVING. The coordinator accepts this map and replaces the state it has already set with MOVING. So the + * node must send its map again after the change. *

- * The window is the end of the exchange on the new primary: it opens when the node applies the coordinator's full - * message and creates the partition, and closes when the node detects lost partitions. In a real cluster a map gets - * into it when a resend scheduled by {@code scheduleResendPartitions()} (after an eviction, a partition moving to - * RENTING or a partition map change) fires there. The coordinator as the new primary is checked by - * {@link #testStaleCoordinatorMapAfterEviction()}. + * In a real cluster such a map is a resend scheduled by {@code scheduleResendPartitions()}, for example after an + * eviction, that fires between the two steps. {@link #testStaleCoordinatorMapAfterEviction()} checks the case where + * the coordinator is the new primary. */ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { /** */ @@ -125,8 +124,8 @@ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { private PartitionLossPolicy plc = IGNORE; /** - * Whether the coordinator becomes the new primary of all lost partitions: the cache uses - * {@link JoinOrderAffinityFunction}, and the coordinator can stall after it sends a full message. + * Whether the coordinator is the new primary of all lost partitions. Sets {@link JoinOrderAffinityFunction}, a + * {@link StallingCommunicationSpi} and a rebalance pool of 4 threads. */ private boolean crdNewPrimary; @@ -144,8 +143,7 @@ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { if (crdNewPrimary) { cfg.setCommunicationSpi(new StallingCommunicationSpi()); - // By default the pool has one thread on machines with fewer than 8 cores, and the other evictions would - // wait behind the held one. + // Evictions use the rebalance pool, one thread by default below 8 cores: the held one would block the rest. cfg.setRebalanceThreadPoolSize(4); } else @@ -185,8 +183,9 @@ public void testStaleMapBeforeLostPartitionMarkedLost() throws Exception { } /** - * A node joins and takes partitions over, the old owners evict their copies, and the node leaves. Its partitions - * are lost, and the new primary sends a stale map, with a lost partition still MOVING, before it changes them. + * Checks that the coordinator's partition map matches the local states of the nodes after node 1 sends a stale + * map. Node 2 joins, takes over partitions from nodes 0 and 1, and leaves with the only copies. Right before node 1 + * owns a lost partition or marks it LOST, it sends its map, which still shows the partition MOVING. * * @param plc Partition loss policy. * @throws Exception If failed. @@ -232,22 +231,23 @@ private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) th if (!waitForCondition(() -> mismatches(crd).isEmpty(), 10_000)) fail("The coordinator's partition map differs from the nodes: " + mismatches(crd)); - - if (plc != IGNORE) - crd.resetLostPartitions(Collections.singleton(CACHE)); - - awaitPartitionMapExchange(); } /** - * The coordinator itself is the new primary of all lost partitions. It creates them before it sends the full - * message, so a plain resend carries the same map version and the other node ignores it. Here the coordinator - * stalls after it sends the full message, as when sending to the next node is slow, and an eviction finishes on the - * coordinator meanwhile. The resend scheduled after the eviction carries a newer map with the lost partitions still - * MOVING, and the other node takes it. The coordinator must send its map again once it owns the partitions. + * Checks that the coordinator sends its partition map again after it owns the lost partitions assigned to it. *

- * The other node gets no lost partitions: otherwise its own resend after detecting them makes the coordinator - * resend its map later, which hides the problem. + * The coordinator creates these partitions as MOVING, sends the full message, and only then owns them. When an + * eviction finishes, the map changes and the coordinator schedules a resend. If the resend fires before the + * coordinator owns the partitions, it carries them as MOVING. Its version is newer than the full message's, so + * node 1 takes it. + *

+ * The test: + *

    + *
  1. holds the eviction of one partition that the coordinator gave to node 1;
    2. + *
  2. stops node 2 and stalls the coordinator right after it sends the full message;
    3. + *
  3. lets the eviction finish once node 1 has finished the exchange;
    4. + *
  4. checks that node 1 took the stale map, then lets the coordinator go on.
    5. + *
* * @throws Exception If failed. */ @@ -262,7 +262,7 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { for (int i = 0; i < PARTS * 4; i++) cache.put(i, i); - // The next eviction on the coordinator waits until the exchange, as the eviction of a large partition would. + // Hold the eviction of one partition that node 1 takes over, as if the partition were large. holdEviction.set(true); IgniteEx other = startGrid(1); @@ -277,7 +277,7 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { GridDhtPartitionTopology crdTop = crd.cachex(CACHE).context().topology(); - // Only the held eviction is left, so the resend is scheduled when it finishes. + // No copy of node 2's partitions may stay on the coordinator: wait until only the held eviction is left. assertTrue(waitForCondition(() -> crdTop.localPartitions().stream() .filter(p -> p.state() == RENTING).count() == 1, WAIT)); @@ -309,14 +309,14 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { if (!waitForCondition(() -> mismatches(other).isEmpty(), 10_000)) fail("The other node's partition map differs from the nodes: " + mismatches(other)); - - awaitPartitionMapExchange(); } /** - * @return Resolver whose partitions send the partition map right before a MOVING partition of the cache is owned or - * marked LOST, once {@link #sendBeforeChange} is set, and hold the next eviction of a partition of the cache, once - * {@link #holdEviction} is set. + * Creates a resolver that adds two hooks to the partitions of the cache. Once {@link #sendBeforeChange} is set, a + * MOVING partition sends the partition map right before it is owned or marked LOST. Once {@link #holdEviction} is + * set, the next eviction waits for {@link #evictionResume}. + * + * @return Dependency resolver with the partition hooks. */ private TestDependencyResolver partitionHooksResolver() { return new TestDependencyResolver(new DependencyResolver() { @@ -410,7 +410,10 @@ private List mismatches(IgniteEx viewer) { return res; } - /** Holds the coordinator right after it sends the full message of an exchange, once armed. */ + /** + * When armed, stops the coordinator's thread after it sends an exchange full message, before it owns its lost + * partitions. + */ private class StallingCommunicationSpi extends TestRecordingCommunicationSpi { /** {@inheritDoc} */ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure ackC) @@ -430,8 +433,7 @@ private class StallingCommunicationSpi extends TestRecordingCommunicationSpi { /** * Assigns partition {@code p} to the server node with index {@code p % 3} in join order, or to the oldest node if - * there is no such node. The third node takes the partitions with {@code p % 3 == 2} from the oldest node and gives - * them back when it leaves. + * there is no such node. */ private static class JoinOrderAffinityFunction implements AffinityFunction { /** */ From e4afd519810a2ad9556179be8db21b8a8e452224 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 8 Oct 2026 19:45:54 +0300 Subject: [PATCH 16/27] IGNITE-29088 Fix test wording --- .../CachePartitionLossStaleMapTest.java | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java index 92980c69f06ef..1055b2153594d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java @@ -87,8 +87,8 @@ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { /** */ private static final int PARTS = 64; - /** Index of the node that becomes the new primary of the lost partitions. */ - private static final int NEW_PRIMARY = 1; + /** Index of the node that sends a stale partition map before it owns a lost partition or marks it LOST. */ + private static final int STALE_MAP_SENDER = 1; /** Index of the node that leaves. */ private static final int LEAVING = 2; @@ -163,7 +163,7 @@ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { } /** - * The new primary sends a stale map before it owns a lost partition under the IGNORE policy. + * Node 1 sends a stale map before it owns a lost partition under the IGNORE policy. * * @throws Exception If failed. */ @@ -173,7 +173,7 @@ public void testStaleMapBeforeLostPartitionOwned() throws Exception { } /** - * The new primary sends a stale map before it marks a lost partition LOST under the READ_WRITE_SAFE policy. + * Node 1 sends a stale map before it marks a lost partition LOST under the READ_WRITE_SAFE policy. * * @throws Exception If failed. */ @@ -195,7 +195,7 @@ private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) th IgniteEx crd = startGrid(0); - startGrid(NEW_PRIMARY, partitionHooksResolver()); + startGrid(STALE_MAP_SENDER, partitionHooksResolver()); IgniteCache cache = crd.cache(CACHE); @@ -208,10 +208,10 @@ private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) th UUID crdId = crd.localNode().id(); - TestRecordingCommunicationSpi newPrimarySpi = TestRecordingCommunicationSpi.spi(grid(NEW_PRIMARY)); + TestRecordingCommunicationSpi sndSpi = TestRecordingCommunicationSpi.spi(grid(STALE_MAP_SENDER)); - // Hold the new primary's maps sent outside an exchange until the coordinator finishes the exchange. - newPrimarySpi.blockMessages((node, msg) -> node.id().equals(crdId) + // Hold node 1's maps sent outside an exchange until the coordinator finishes the exchange. + sndSpi.blockMessages((node, msg) -> node.id().equals(crdId) && msg instanceof GridDhtPartitionsSingleMessage && ((GridDhtPartitionsSingleMessage)msg).exchangeId() == null); @@ -221,13 +221,13 @@ private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) th stopGrid(LEAVING); - assertTrue("The new primary didn't change a lost partition", newPrimarySpi.waitForBlocked(1, WAIT)); + assertTrue("Node 1 didn't change a lost partition", sndSpi.waitForBlocked(1, WAIT)); - assertFalse("The new primary didn't change a lost partition", sendBeforeChange.get()); + assertFalse("Node 1 didn't change a lost partition", sendBeforeChange.get()); assertTrue(waitForCondition(() -> exchangeFinished(crd, leaveVer), WAIT)); - newPrimarySpi.stopBlock(); + sndSpi.stopBlock(); if (!waitForCondition(() -> mismatches(crd).isEmpty(), 10_000)) fail("The coordinator's partition map differs from the nodes: " + mismatches(crd)); @@ -236,10 +236,10 @@ private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) th /** * Checks that the coordinator sends its partition map again after it owns the lost partitions assigned to it. *

- * The coordinator creates these partitions as MOVING, sends the full message, and only then owns them. When an - * eviction finishes, the map changes and the coordinator schedules a resend. If the resend fires before the - * coordinator owns the partitions, it carries them as MOVING. Its version is newer than the full message's, so - * node 1 takes it. + * The coordinator creates these partitions as MOVING, sends the full message, and only then owns them. When its + * last RENTING partition is evicted, the map changes and the coordinator schedules a resend. If the resend fires + * before the coordinator owns the partitions, it carries them as MOVING. Its version is newer than the full + * message's, so node 1 takes it. *

* The test: *

    @@ -277,7 +277,8 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { GridDhtPartitionTopology crdTop = crd.cachex(CACHE).context().topology(); - // No copy of node 2's partitions may stay on the coordinator: wait until only the held eviction is left. + // Wait until only the held eviction is left: the last eviction schedules the resend, + // and no copy of node 2's partitions may stay on the coordinator. assertTrue(waitForCondition(() -> crdTop.localPartitions().stream() .filter(p -> p.state() == RENTING).count() == 1, WAIT)); @@ -297,7 +298,7 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { GridDhtPartitionTopology otherTop = other.cachex(CACHE).context().topology(); - assertTrue("The other node didn't take a stale map of the coordinator", waitForCondition(() -> { + assertTrue("Node 1 didn't take a stale map of the coordinator", waitForCondition(() -> { GridDhtPartitionMap crdMap = otherTop.partitionMap(false).get(crdId); return crdMap != null && crdMap.hasMovingPartitions(); @@ -308,13 +309,13 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { assertTrue(waitForCondition(() -> exchangeFinished(crd, leaveVer), WAIT)); if (!waitForCondition(() -> mismatches(other).isEmpty(), 10_000)) - fail("The other node's partition map differs from the nodes: " + mismatches(other)); + fail("Node 1's partition map differs from the nodes: " + mismatches(other)); } /** - * Creates a resolver that adds two hooks to the partitions of the cache. Once {@link #sendBeforeChange} is set, a - * MOVING partition sends the partition map right before it is owned or marked LOST. Once {@link #holdEviction} is - * set, the next eviction waits for {@link #evictionResume}. + * Creates a resolver that adds two hooks to the partitions of the cache. Once {@link #sendBeforeChange} is set, the + * node sends its partition map right before the next MOVING partition is owned or marked LOST. Once + * {@link #holdEviction} is set, the next eviction waits for {@link #evictionResume}. * * @return Dependency resolver with the partition hooks. */ @@ -411,8 +412,8 @@ private List mismatches(IgniteEx viewer) { } /** - * When armed, stops the coordinator's thread after it sends an exchange full message, before it owns its lost - * partitions. + * Once {@link #stallAfterFullMessage} is set, holds the coordinator's thread right after it sends an exchange full + * message, before it owns its lost partitions. The thread waits for {@link #crdResume}. */ private class StallingCommunicationSpi extends TestRecordingCommunicationSpi { /** {@inheritDoc} */ From 9dfc3070fef898a61d5c16060596f6ecae900282 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 28 Oct 2026 14:50:49 +0300 Subject: [PATCH 17/27] IGNITE-29088 Add observer, client, early map and orphan tests --- .../CacheNoSupplierPartitionMapTest.java | 147 ++++++++++++++ .../CachePartitionLossStaleMapTest.java | 179 ++++++++++++++---- .../testsuites/IgniteCacheTestSuite15.java | 2 + ...ndexingCachePartitionLossStaleMapTest.java | 47 +++++ .../IgniteBinaryCacheQueryTestSuite3.java | 2 + 5 files changed, 344 insertions(+), 33 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheNoSupplierPartitionMapTest.java create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossStaleMapTest.java diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheNoSupplierPartitionMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheNoSupplierPartitionMapTest.java new file mode 100644 index 0000000000000..459a35fe3b68d --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheNoSupplierPartitionMapTest.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.util.AttributeNodeFilter; +import org.junit.Test; + +import static org.apache.ignite.internal.TestRecordingCommunicationSpi.blockDemandMessageForGroup; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** + * Checks that the other nodes see the partitions that a data node owns at the end of an exchange because no node has + * them. Only the data node writes OWNING into its partition map, so it must send the map. + *

    + * The cache keeps its data only on nodes with the {@link #DATA_ATTR} attribute. The coordinator, another server and a + * client have no data and watch the data node's partitions in their partition maps. The data node joins outside the + * baseline and gets its partitions when the baseline changes. + *

    + * A node that joins the baseline also rebalances the system cache, and the affinity change exchange after that + * rebalance carries the node's map anyway. The test holds the system cache's rebalance on the data node, as if the + * system cache were large. + */ +public class CacheNoSupplierPartitionMapTest extends GridCommonAbstractTest { + /** */ + private static final String CACHE = "cache"; + + /** */ + private static final int PARTS = 32; + + /** Attribute of the nodes that keep the cache's data. */ + private static final String DATA_ATTR = "data"; + + /** Index of the client node. */ + private static final int CLIENT = 2; + + /** Index of the data node. */ + private static final int DATA = 3; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + boolean data = igniteInstanceName.equals(getTestIgniteInstanceName(DATA)); + + return super.getConfiguration(igniteInstanceName) + .setUserAttributes(Collections.singletonMap(DATA_ATTR, data)) + .setCommunicationSpi(new TestRecordingCommunicationSpi()) + .setCacheConfiguration(new CacheConfiguration(CACHE) + .setBackups(0) + .setAffinity(new RendezvousAffinityFunction(false, PARTS)) + .setNodeFilter(new AttributeNodeFilter(DATA_ATTR, true))); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + super.afterTest(); + } + + /** + * The first data node gets all partitions when it joins the baseline and owns them, since no node has them. + * + * @throws Exception If failed. + */ + @Test + public void testOrphanPartitionsOwned() throws Exception { + IgniteEx crd = startGrid(0); + + crd.cluster().baselineAutoAdjustEnabled(false); + + startGrid(1); + startClientGrid(CLIENT); + + IgniteEx data = startGrid(DATA); + + TestRecordingCommunicationSpi.spi(data) + .blockMessages(blockDemandMessageForGroup(CU.cacheId(CU.UTILITY_CACHE_NAME))); + + crd.cluster().setBaselineTopology(crd.cluster().topologyVersion()); + + assertTrue("The data node didn't own its partitions", waitForCondition(() -> { + List parts = data.cachex(CACHE).context().topology().localPartitions(); + + return parts.size() == PARTS && parts.stream().allMatch(p -> p.state() == OWNING); + }, 10_000)); + + UUID dataId = data.localNode().id(); + + for (int idx = 0; idx <= CLIENT; idx++) { + IgniteEx watcher = grid(idx); + + if (!waitForCondition(() -> notOwning(watcher, dataId).isEmpty(), 10_000)) { + fail("The partition map of " + watcher.name() + " differs from the data node: " + + notOwning(watcher, dataId)); + } + } + } + + /** + * @param watcher Node whose partition map is checked. + * @param dataId ID of the data node. + * @return Partitions that the watcher's copy of the data node's map doesn't show OWNING. + */ + private List notOwning(IgniteEx watcher, UUID dataId) { + GridDhtPartitionMap view = watcher.cachex(CACHE).context().topology().partitionMap(false).get(dataId); + + List res = new ArrayList<>(); + + for (int p = 0; p < PARTS; p++) { + GridDhtPartitionState seen = view == null ? null : view.get(p); + + if (seen != OWNING) + res.add("p=" + p + " seen=" + seen); + } + + return res; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java index 1055b2153594d..0711efbae1237 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java @@ -31,6 +31,7 @@ import org.apache.ignite.cache.affinity.AffinityFunction; import org.apache.ignite.cache.affinity.AffinityFunctionContext; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cache.query.ScanQuery; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -72,17 +73,17 @@ *

    * The new primary recreates each lost partition empty as MOVING when it applies the coordinator's full message. At * the end of the exchange it owns the partition under the {@link PartitionLossPolicy#IGNORE IGNORE} policy or marks - * it LOST under the safe policies. If the node sends its partition map between these two steps, the map shows the - * partition MOVING. The coordinator accepts this map and replaces the state it has already set with MOVING. So the - * node must send its map again after the change. + * it LOST under the safe policies. Only the new primary writes this state into its partition map and sends the map. + * A map the node sent earlier shows the partition MOVING or doesn't have it at all. The other nodes must take the + * new primary's state in the end, whatever order its maps arrive in. *

    - * In a real cluster such a map is a resend scheduled by {@code scheduleResendPartitions()}, for example after an - * eviction, that fires between the two steps. {@link #testStaleCoordinatorMapAfterEviction()} checks the case where - * the coordinator is the new primary. + * In a real cluster such an earlier map is a resend scheduled by {@code scheduleResendPartitions()}, for example after + * an eviction. {@link #testStaleCoordinatorMapAfterEviction()} checks the case where the coordinator is the new + * primary. */ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { /** */ - private static final String CACHE = "cache"; + protected static final String CACHE = "cache"; /** */ private static final int PARTS = 64; @@ -93,6 +94,12 @@ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { /** Index of the node that leaves. */ private static final int LEAVING = 2; + /** Index of a server node that watches node 1's partitions in its partition map. */ + private static final int OBSERVER = 3; + + /** Index of the client node. */ + private static final int CLIENT = 4; + /** Time to wait for a step of a test, in milliseconds. */ private static final long WAIT = 30_000; @@ -124,8 +131,8 @@ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { private PartitionLossPolicy plc = IGNORE; /** - * Whether the coordinator is the new primary of all lost partitions. Sets {@link JoinOrderAffinityFunction}, a - * {@link StallingCommunicationSpi} and a rebalance pool of 4 threads. + * Whether the coordinator is the new primary of all lost partitions. Sets {@link JoinOrderAffinityFunction} and a + * {@link StallingCommunicationSpi}. */ private boolean crdNewPrimary; @@ -140,14 +147,10 @@ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { ? new JoinOrderAffinityFunction() : new RendezvousAffinityFunction(false, PARTS))); - if (crdNewPrimary) { - cfg.setCommunicationSpi(new StallingCommunicationSpi()); + cfg.setCommunicationSpi(crdNewPrimary ? new StallingCommunicationSpi() : new TestRecordingCommunicationSpi()); - // Evictions use the rebalance pool, one thread by default below 8 cores: the held one would block the rest. - cfg.setRebalanceThreadPoolSize(4); - } - else - cfg.setCommunicationSpi(new TestRecordingCommunicationSpi()); + // Evictions run in the rebalance pool, one thread by default below 8 cores: a held one would block the rest. + cfg.setRebalanceThreadPoolSize(4); return cfg; } @@ -183,9 +186,10 @@ public void testStaleMapBeforeLostPartitionMarkedLost() throws Exception { } /** - * Checks that the coordinator's partition map matches the local states of the nodes after node 1 sends a stale - * map. Node 2 joins, takes over partitions from nodes 0 and 1, and leaves with the only copies. Right before node 1 - * owns a lost partition or marks it LOST, it sends its map, which still shows the partition MOVING. + * Checks that the partition maps of the coordinator, the observer and the client match the local states of the + * nodes after node 1 sends a stale map. Node 2 joins, takes over partitions from the other servers, and leaves with + * the only copies. Right before node 1 owns a lost partition or marks it LOST, it sends its map, which still shows + * the partition MOVING. The coordinator gets this map once its own exchange is finished. * * @param plc Partition loss policy. * @throws Exception If failed. @@ -193,16 +197,7 @@ public void testStaleMapBeforeLostPartitionMarkedLost() throws Exception { private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) throws Exception { this.plc = plc; - IgniteEx crd = startGrid(0); - - startGrid(STALE_MAP_SENDER, partitionHooksResolver()); - - IgniteCache cache = crd.cache(CACHE); - - for (int i = 0; i < PARTS * 4; i++) - cache.put(i, i); - - startGrid(LEAVING); + IgniteEx crd = startNodes(); awaitPartitionMapExchange(true, true, null); @@ -229,8 +224,71 @@ private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) th sndSpi.stopBlock(); - if (!waitForCondition(() -> mismatches(crd).isEmpty(), 10_000)) - fail("The coordinator's partition map differs from the nodes: " + mismatches(crd)); + checkNodes(crd, grid(OBSERVER), grid(CLIENT)); + } + + /** + * Checks that the partition maps of the coordinator, the observer and the client match the local states of the + * nodes after node 1 sends its map before it gets the full message. + *

    + * When node 1's last RENTING partition is evicted, the map changes and node 1 schedules a resend. Node 1 doesn't + * know its lost partitions until it gets the full message, so the resent map doesn't have them. Its version is + * newer than the one the coordinator has, so the coordinator takes it. The test: + *

      + *
    1. holds the eviction of one partition that node 1 gives to the observer;
      2. + *
    2. stops node 2 and holds the coordinator's full message to node 1;
      3. + *
    3. lets the eviction finish once the coordinator has finished the exchange;
      4. + *
    4. lets the full message go once the coordinator has taken node 1's resent map.
      5. + *
    + * + * @throws Exception If failed. + */ + @Test + public void testStaleMapBeforeFullMessage() throws Exception { + holdEviction.set(true); + + IgniteEx crd = startNodes(); + + awaitPartitionMapExchange(); + + assertTrue("Node 1 didn't start an eviction", evictionHeld.await(WAIT, MILLISECONDS)); + + IgniteEx snd = grid(STALE_MAP_SENDER); + + GridDhtPartitionTopology sndTop = snd.cachex(CACHE).context().topology(); + + // Wait until only the held eviction is left: the last eviction schedules the resend. + assertTrue(waitForCondition(() -> sndTop.localPartitions().stream() + .filter(p -> p.state() == RENTING).count() == 1, WAIT)); + + UUID sndId = snd.localNode().id(); + + TestRecordingCommunicationSpi crdSpi = TestRecordingCommunicationSpi.spi(crd); + + crdSpi.blockMessages((node, msg) -> node.id().equals(sndId) + && msg instanceof GridDhtPartitionsFullMessage + && ((GridDhtPartitionsFullMessage)msg).exchangeId() != null); + + long leaveVer = crd.cluster().topologyVersion() + 1; + + stopGrid(LEAVING); + + assertTrue("The coordinator didn't send the full message", crdSpi.waitForBlocked(1, WAIT)); + + assertTrue(waitForCondition(() -> exchangeFinished(crd, leaveVer), WAIT)); + + GridDhtPartitionTopology crdTop = crd.cachex(CACHE).context().topology(); + + long seq = crdTop.partitionMap(false).get(sndId).updateSequence(); + + evictionResume.countDown(); + + assertTrue("The coordinator didn't take node 1's resent map", waitForCondition( + () -> crdTop.partitionMap(false).get(sndId).updateSequence() > seq, WAIT)); + + crdSpi.stopBlock(); + + checkNodes(crd, grid(OBSERVER), grid(CLIENT)); } /** @@ -267,6 +325,8 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { IgniteEx other = startGrid(1); + IgniteEx client = startClientGrid(CLIENT); + awaitPartitionMapExchange(); assertTrue("The coordinator didn't start an eviction", evictionHeld.await(WAIT, MILLISECONDS)); @@ -308,8 +368,61 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { assertTrue(waitForCondition(() -> exchangeFinished(crd, leaveVer), WAIT)); - if (!waitForCondition(() -> mismatches(other).isEmpty(), 10_000)) - fail("Node 1's partition map differs from the nodes: " + mismatches(other)); + checkNodes(other, client); + } + + /** + * Runs a query over the cache on a node. Called while the node may still see a lost partition without an owner. + * + * @param node Node. + */ + protected void runQuery(Ignite node) { + node.cache(CACHE).query(new ScanQuery<>()).getAll(); + } + + /** + * Starts the coordinator, node 1 with the partition hooks and the client, loads the cache, then starts the + * observer and node 2. + * + * @return Coordinator. + * @throws Exception If failed. + */ + private IgniteEx startNodes() throws Exception { + IgniteEx crd = startGrid(0); + + startGrid(STALE_MAP_SENDER, partitionHooksResolver()); + + startClientGrid(CLIENT); + + IgniteCache cache = crd.cache(CACHE); + + for (int i = 0; i < PARTS * 4; i++) + cache.put(i, i); + + startGrid(OBSERVER); + + startGrid(LEAVING); + + return crd; + } + + /** + * Under the IGNORE policy runs a query on each viewer. Then checks that the partition map of each viewer matches the + * local states of the nodes. + * + * @param viewers Nodes whose partition maps are checked. + * @throws Exception If failed. + */ + private void checkNodes(IgniteEx... viewers) throws Exception { + if (plc == IGNORE) { + for (IgniteEx viewer : viewers) + runQuery(viewer); + } + + for (IgniteEx viewer : viewers) { + if (!waitForCondition(() -> mismatches(viewer).isEmpty(), 10_000)) + fail("The partition map of " + viewer.name() + " differs from the nodes: " + mismatches(viewer)); + } } /** diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java index d38d9154e4887..5bfb5ad021d12 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java @@ -37,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheStoreCollectionTest; import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentNodeJoinValidationTest; import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheNoSupplierPartitionMapTest; import org.apache.ignite.internal.processors.cache.distributed.CachePartitionLossStaleMapTest; import org.apache.ignite.internal.processors.cache.distributed.GridExchangeFreeCellularSwitchComplexOperationsTest; import org.apache.ignite.internal.processors.cache.distributed.GridExchangeFreeCellularSwitchIsolationTest; @@ -100,6 +101,7 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, CacheCreateDestroyClusterReadOnlyModeTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteCachePartitionLossPolicySelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CachePartitionLossStaleMapTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheNoSupplierPartitionMapTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteCacheGroupsPartitionLossPolicySelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheRebalancingSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheManualRebalancingTest.class, ignoredTests); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossStaleMapTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossStaleMapTest.java new file mode 100644 index 0000000000000..50a2ce4437fe0 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossStaleMapTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import org.apache.ignite.Ignite; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.cache.distributed.CachePartitionLossStaleMapTest; + +import static java.util.concurrent.TimeUnit.SECONDS; + +/** + * Runs {@link CachePartitionLossStaleMapTest} with a SQL query. A query fails if the node that runs it sees a partition + * without an owner until the query's timeout. + */ +public class IndexingCachePartitionLossStaleMapTest extends CachePartitionLossStaleMapTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + for (CacheConfiguration ccfg : cfg.getCacheConfiguration()) + ccfg.setIndexedTypes(Integer.class, Integer.class); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void runQuery(Ignite node) { + node.cache(CACHE).query(new SqlFieldsQuery("select count(*) from Integer").setTimeout(10, SECONDS)).getAll(); + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite3.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite3.java index d1d2e840db5b8..43e15c7c31f74 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite3.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite3.java @@ -62,6 +62,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheP2pUnmarshallingQueryErrorTest; import org.apache.ignite.internal.processors.cache.IgniteCheckClusterStateBeforeExecuteQueryTest; import org.apache.ignite.internal.processors.cache.IndexingCachePartitionLossPolicySelfTest; +import org.apache.ignite.internal.processors.cache.IndexingCachePartitionLossStaleMapTest; import org.apache.ignite.internal.processors.cache.QueryEntityCaseMismatchTest; import org.apache.ignite.internal.processors.cache.ReservationsOnDoneAfterTopologyUnlockFailTest; import org.apache.ignite.internal.processors.cache.SqlFieldsQuerySelfTest; @@ -293,6 +294,7 @@ // Partition loss. IndexingCachePartitionLossPolicySelfTest.class, + IndexingCachePartitionLossStaleMapTest.class, // Partitions eviction SqlPartitionEvictionTest.class, From e139034c99dcf61dbf0f0252313a04c431f6752b Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 28 Oct 2026 14:54:17 +0300 Subject: [PATCH 18/27] IGNITE-29088 Report lost and orphan partition states from their owner --- .../GridDhtPartitionsExchangeFuture.java | 36 +++++++----- .../topology/GridClientPartitionTopology.java | 10 +++- .../topology/GridDhtPartitionTopology.java | 4 +- .../GridDhtPartitionTopologyImpl.java | 56 +++++++++++++------ .../feature/CoreFeatureRegistry.java | 6 ++ 5 files changed, 78 insertions(+), 34 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 28bc1b280ada3..b23b115d5296b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -2295,6 +2295,8 @@ private String exchangeTimingsLogMessage(String header, List timings) { boolean cleanIdxRebuildFutures = true; + Collection locChangedGrps = new HashSet<>(); + try { waitUntilNewCachesAreRegistered(); @@ -2349,7 +2351,7 @@ private String exchangeTimingsLogMessage(String header, List timings) { } if (exchCtx.events().hasServerLeft() || activateCluster()) - detectLostPartitions(res); + locChangedGrps.addAll(detectLostPartitions(res)); Map m = U.newHashMap(cctx.cache().cacheGroups().size()); @@ -2391,8 +2393,20 @@ private String exchangeTimingsLogMessage(String header, List timings) { cleanIdxRebuildFutures = false; - for (CacheGroupContext grp : cctx.cache().cacheGroups()) - grp.topology().onExchangeDone(this, grp.affinity().readyAffinity(res), false); + for (CacheGroupContext grp : cctx.cache().cacheGroups()) { + if (grp.topology().onExchangeDone(this, grp.affinity().readyAffinity(res), false)) + locChangedGrps.add(grp); + } + + if (!locChangedGrps.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("Refresh partitions due to local partition changes at the end of exchange [grps=" + + locChangedGrps.stream().map(CacheGroupContext::cacheOrGroupName) + .collect(Collectors.toList()) + ']'); + } + + cctx.exchange().refreshPartitions(locChangedGrps); + } if (changedAffinity()) cctx.walState().disableGroupDurabilityForPreloading(this); @@ -3497,11 +3511,12 @@ private void findCounterForReservation( /** * Detect lost partitions in case of node left or failed. For topology coordinator is called when all {@link * GridDhtPartitionsSingleMessage} were received. For other nodes is called when exchange future is completed by - * {@link GridDhtPartitionsFullMessage}. If a local partition changes, sends the partition maps of its group again. + * {@link GridDhtPartitionsFullMessage}. * * @param resTopVer Result topology version. + * @return Cache groups where the state of a local partition changed. */ - private void detectLostPartitions(AffinityTopologyVersion resTopVer) { + private Collection detectLostPartitions(AffinityTopologyVersion resTopVer) { Collection changedGrps = ConcurrentHashMap.newKeySet(); try { @@ -3525,16 +3540,9 @@ private void detectLostPartitions(AffinityTopologyVersion resTopVer) { throw new IgniteException(e); } - if (!changedGrps.isEmpty()) { - if (log.isDebugEnabled()) { - log.debug("Refresh partitions due to lost partitions detected [grps=" + - changedGrps.stream().map(CacheGroupContext::cacheOrGroupName).collect(Collectors.toList()) + ']'); - } - - cctx.exchange().refreshPartitions(changedGrps); - } - timeBag.finishGlobalStage("Detect lost partitions"); + + return changedGrps; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java index 750f99f3767c4..e64ce2ab24ada 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java @@ -67,6 +67,7 @@ import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; +import static org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry.PART_STATE_FROM_OWNER_FEATURE; import static org.apache.ignite.internal.util.lang.ClusterNodeFunc.nodeIds; /** @@ -1014,9 +1015,9 @@ else if (isStaleUpdate(cur, parts)) { } /** {@inheritDoc} */ - @Override public void onExchangeDone(GridDhtPartitionsExchangeFuture fut, AffinityAssignment assignment, + @Override public boolean onExchangeDone(GridDhtPartitionsExchangeFuture fut, AffinityAssignment assignment, boolean updateRebalanceVer) { - // No-op. + return false; } /** {@inheritDoc} */ @@ -1043,6 +1044,9 @@ else if (isStaleUpdate(cur, parts)) { boolean changed = false; + boolean updRmtMaps = + !cctx.kernalContext().rollingUpgrade().features().isActive(PART_STATE_FROM_OWNER_FEATURE); + for (int part = 0; part < parts; part++) { boolean lost = F.contains(lostParts, part); @@ -1071,7 +1075,7 @@ else if (isStaleUpdate(cur, parts)) { } } - if (lost) { + if (lost && updRmtMaps) { // Update remote maps according to policy. for (Map.Entry entry : node2part.entrySet()) { if (entry.getValue().get(part) != null) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java index 903967f246a2c..88a45125c34a9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java @@ -454,8 +454,10 @@ public Map> resetOwners( * * @param assignment New affinity assignment. * @param updateRebalanceVer {@code True} if need check rebalance state. + * @return {@code True} if the state of at least one local partition changed. */ - public void onExchangeDone(GridDhtPartitionsExchangeFuture fut, AffinityAssignment assignment, boolean updateRebalanceVer); + public boolean onExchangeDone(GridDhtPartitionsExchangeFuture fut, AffinityAssignment assignment, + boolean updateRebalanceVer); /** * Rents a partition and updates a partition map if the partition was switched to RENTING. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index 194b366debc9e..914b9f339b53e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -84,6 +84,7 @@ import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING; +import static org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry.PART_STATE_FROM_OWNER_FEATURE; import static org.apache.ignite.internal.util.lang.ClusterNodeFunc.nodeIds; /** @@ -1606,6 +1607,8 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD return false; } + boolean changed = false; + if (exchangeVer != null) { assert exchangeVer.compareTo(readyTopVer) >= 0 && exchangeVer.compareTo(lastTopChangeVer) >= 0; @@ -1623,7 +1626,7 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD // part.clearFuture lifecycle will be broken after resetting. // New partition should be created instead. if (locPart != null && locPart.state() != EVICTED) { - locPart.markLost(); + changed |= locPart.markLost(); GridDhtPartitionMap locMap = partMap.get(ctx.localNodeId()); @@ -1672,8 +1675,6 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD diffFromAffinityVer = readyTopVer; } - boolean changed = false; - GridDhtPartitionMap nodeMap = partMap.get(ctx.localNodeId()); // Only in real exchange occurred. @@ -2049,9 +2050,11 @@ else if (isStaleUpdate(cur, parts)) { } /** {@inheritDoc} */ - @Override public void onExchangeDone(@Nullable GridDhtPartitionsExchangeFuture fut, + @Override public boolean onExchangeDone(@Nullable GridDhtPartitionsExchangeFuture fut, AffinityAssignment assignment, boolean updateRebalanceVer) { + boolean changed = false; + lock.writeLock().lock(); try { @@ -2082,19 +2085,27 @@ else if (isStaleUpdate(cur, parts)) { // Own orphan moving partitions (having no suppliers). if (fut != null && (fut.events().hasServerJoin() || fut.changedBaseline())) - ownOrphans(); + changed = ownOrphans(); } finally { lock.writeLock().unlock(); } + + return changed; } /** * Check if some local moving partitions have no suitable supplier and own them. * This can happen if a partition has been created by affinity assignment on new node and no supplier exists. * For example, if a node joins topology being a first data node for cache with node filter configured. + * + * @return {@code True} if a local partition was owned. */ - private void ownOrphans() { + private boolean ownOrphans() { + boolean updRmtMaps = !ctx.kernalContext().rollingUpgrade().features().isActive(PART_STATE_FROM_OWNER_FEATURE); + + boolean changed = false; + for (int p = 0; p < grp.affinity().partitions(); p++) { boolean hasOwner = false; @@ -2110,15 +2121,23 @@ private void ownOrphans() { GridDhtLocalPartition locPart = localPartition(p); if (locPart != null && locPart.state() != EVICTED && locPart.state() != LOST) { - locPart.own(); + if (locPart.state() == MOVING && locPart.own()) { + changed = true; - for (GridDhtPartitionMap map : node2part.values()) { - if (map.get(p) != null) - map.put(p, OWNING); + updateLocal(p, OWNING, updateSeq.incrementAndGet(), readyTopVer); + } + + if (updRmtMaps) { + for (GridDhtPartitionMap map : node2part.values()) { + if (map.get(p) != null) + map.put(p, OWNING); + } } } } } + + return changed; } /** @@ -2225,6 +2244,9 @@ private void rebuildDiff(AffinityAssignment affAssignment) { boolean changed = false; + boolean updRmtMaps = + !ctx.kernalContext().rollingUpgrade().features().isActive(PART_STATE_FROM_OWNER_FEATURE); + for (int part = 0; part < grp.affinity().partitions(); part++) { boolean lost = F.contains(lostParts, part); @@ -2276,14 +2298,16 @@ private void rebuildDiff(AffinityAssignment affAssignment) { } // Update remote maps according to policy. - for (Map.Entry entry : node2part.entrySet()) { - if (entry.getKey().equals(ctx.localNodeId())) - continue; + if (updRmtMaps) { + for (Map.Entry entry : node2part.entrySet()) { + if (entry.getKey().equals(ctx.localNodeId())) + continue; - GridDhtPartitionState p0 = entry.getValue().get(part); + GridDhtPartitionState p0 = entry.getValue().get(part); - if (p0 != null && p0 != EVICTED) - entry.getValue().put(part, isLossIgnored ? OWNING : LOST); + if (p0 != null && p0 != EVICTED) + entry.getValue().put(part, isLossIgnored ? OWNING : LOST); + } } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java index 4d00489327fea..ff7a69d72b6ed 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java @@ -111,4 +111,10 @@ public class CoreFeatureRegistry { /** Feature that introduces the snapshot list command. */ public static final IgniteFeature SNAPSHOT_LIST_FEATURE = new IgniteCoreFeature(4); + + /** + * Feature that makes only the node holding a partition write its state after loss detection or after the node owns + * a partition that no node has. The other nodes take the state from that node's partition map. + */ + public static final IgniteFeature PART_STATE_FROM_OWNER_FEATURE = new IgniteCoreFeature(5); } From 63dcf4553dbf7a53118db94d0eff6544b200a8cd Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 28 Oct 2026 15:03:23 +0300 Subject: [PATCH 19/27] IGNITE-29088 Add rolling upgrade test --- ...itionStateFromOwnerRollingUpgradeTest.java | 203 ++++++++++++++++++ .../testsuites/IgniteBasicTestSuite.java | 2 + 2 files changed, 205 insertions(+) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java new file mode 100644 index 0000000000000..52438fb4f6858 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rollingupgrade; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import org.apache.ignite.Ignite; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; +import org.apache.ignite.internal.util.typedef.G; +import org.junit.Test; + +import static org.apache.ignite.cache.PartitionLossPolicy.IGNORE; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED; +import static org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry.PART_STATE_FROM_OWNER_FEATURE; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** + * Checks that the coordinator writes the state of the new owners of lost partitions into their partition maps itself + * only until {@code PART_STATE_FROM_OWNER_FEATURE} is active. An older node doesn't send its map after it owns a lost + * partition, so until every node runs the new version the coordinator can't wait for that map. + *

    + * In both parts a server leaves with the only copies of some partitions under the IGNORE policy, while the maps that + * the other servers send outside an exchange can't reach the coordinator. The servers send their maps as before, so + * dropping the maps stands for an older node that doesn't send them. + */ +public class PartitionStateFromOwnerRollingUpgradeTest extends AbstractRollingUpgradeTest { + /** */ + private static final String CACHE = "cache"; + + /** */ + private static final int PARTS = 64; + + /** Version without {@code PART_STATE_FROM_OWNER_FEATURE}. */ + private static final String OLD_VER = "2.18.0"; + + /** Index of the server that leaves. */ + private static final int LEAVING = 3; + + /** + * Before the upgrade the coordinator sees the new owners right after the exchange, with their maps dropped. After + * all nodes are upgraded and the version is finalized, the feature is active, and the coordinator sees the new + * owners only once their maps arrive. + * + * @throws Exception If failed. + */ + @Test + public void testOwnersReportStateAfterUpgrade() throws Exception { + startCluster(OLD_VER); + startGrid(LEAVING, OLD_VER); + + grid(0).createCache(new CacheConfiguration(CACHE) + .setBackups(0) + .setPartitionLossPolicy(IGNORE) + .setAffinity(new RendezvousAffinityFunction(false, PARTS))); + + awaitPartitionMapExchange(); + + assertFalse(isFeatureActive(grid(0), PART_STATE_FROM_OWNER_FEATURE)); + + IgniteEx crd = stopLeavingWithMapsDropped(); + + assertTrue("The coordinator didn't write the new owners' state: " + mismatches(crd), + mismatches(crd).isEmpty()); + + TestRecordingCommunicationSpi.stopBlockAll(); + + ru(0).enableVersionUpgrade(); + + upgradeNodeVersion(1, OLD_VER, MASTER_VER_REF); + upgradeNodeVersion(2, OLD_VER, MASTER_VER_REF); + upgradeNodeVersion(0, OLD_VER, MASTER_VER_REF); + + finalizeClusterVersion(1, MASTER_VER_REF); + + for (Ignite g : G.allGrids()) + assertTrue(g.name(), isFeatureActive(g, PART_STATE_FROM_OWNER_FEATURE)); + + startGrid(LEAVING, MASTER_VER_REF); + + awaitPartitionMapExchange(); + + crd = stopLeavingWithMapsDropped(); + + assertFalse("The coordinator wrote the new owners' state itself", mismatches(crd).isEmpty()); + + TestRecordingCommunicationSpi.stopBlockAll(); + + IgniteEx crd0 = crd; + + if (!waitForCondition(() -> mismatches(crd0).isEmpty(), 10_000)) + fail("The coordinator's partition map differs from the nodes: " + mismatches(crd)); + } + + /** + * Stops the leaving server while the other servers' partition maps sent outside an exchange are dropped, and waits + * for every node to finish the exchange. + * + * @return Coordinator. + * @throws Exception If failed. + */ + private IgniteEx stopLeavingWithMapsDropped() throws Exception { + IgniteEx crd = coordinator(); + + UUID crdId = crd.localNode().id(); + + for (Ignite g : G.allGrids()) { + if (g != crd && !g.cluster().localNode().isClient()) { + TestRecordingCommunicationSpi.spi(g).blockMessages((node, msg) -> node.id().equals(crdId) + && msg instanceof GridDhtPartitionsSingleMessage + && ((GridDhtPartitionsSingleMessage)msg).exchangeId() == null); + } + } + + long leaveVer = crd.cluster().topologyVersion() + 1; + + stopGrid(LEAVING); + + for (Ignite g : G.allGrids()) { + assertTrue(waitForCondition(() -> { + GridDhtPartitionsExchangeFuture fut = ((IgniteEx)g).context().cache().context().exchange() + .lastFinishedFuture(); + + return fut != null && fut.topologyVersion().topologyVersion() >= leaveVer; + }, getTestTimeout())); + } + + return crd; + } + + /** @return The oldest server node. */ + private IgniteEx coordinator() { + IgniteEx crd = null; + + for (Ignite g : G.allGrids()) { + if (!g.cluster().localNode().isClient() + && (crd == null || g.cluster().localNode().order() < crd.localNode().order())) + crd = (IgniteEx)g; + } + + return crd; + } + + /** + * @param viewer Node whose partition map is checked. + * @return Partitions whose state in the viewer's map differs from the state on the server itself. + */ + private List mismatches(IgniteEx viewer) { + GridDhtPartitionTopology viewerTop = viewer.cachex(CACHE).context().topology(); + + List res = new ArrayList<>(); + + for (Ignite g : G.allGrids()) { + if (g.cluster().localNode().isClient()) + continue; + + GridDhtPartitionMap view = viewerTop.partitionMap(false).get(g.cluster().localNode().id()); + + GridDhtPartitionTopology top = ((IgniteEx)g).cachex(CACHE).context().topology(); + + for (int p = 0; p < PARTS; p++) { + GridDhtLocalPartition locPart = top.localPartition(p, AffinityTopologyVersion.NONE, false, true); + + GridDhtPartitionState loc = locPart == null ? null : locPart.state(); + GridDhtPartitionState seen = view == null ? null : view.get(p); + + // The map keeps evicted partitions, the node doesn't. + if (seen == EVICTED) + seen = null; + + if (loc != seen) + res.add(g.name() + " p=" + p + " local=" + loc + " seen=" + seen); + } + } + + return res; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java index 7273f0d23b243..499e7f7756d73 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java @@ -65,6 +65,7 @@ import org.apache.ignite.internal.processors.odbc.SqlListenerUtilsTest; import org.apache.ignite.internal.processors.query.NoQueryEngineNodeTest; import org.apache.ignite.internal.processors.rollingupgrade.CoreVersionRollingUpgradeTest; +import org.apache.ignite.internal.processors.rollingupgrade.PartitionStateFromOwnerRollingUpgradeTest; import org.apache.ignite.internal.processors.rollingupgrade.PluginVersionRollingUpgradeTest; import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureSetTest; import org.apache.ignite.internal.processors.rollingupgrade.feature.ManagementApiVersionValidationTest; @@ -115,6 +116,7 @@ CoreVersionRollingUpgradeTest.class, PluginVersionRollingUpgradeTest.class, RollingUpgradeMessageSerializationTest.class, + PartitionStateFromOwnerRollingUpgradeTest.class, ManagementApiVersionValidationTest.class, GridProductVersionSelfTest.class, GridAffinityAssignmentV2Test.class, From e04eaaf7143858831106fee2a46e83fb94b3da4e Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Wed, 28 Oct 2026 17:14:20 +0300 Subject: [PATCH 20/27] IGNITE-29088 Fix tests after review --- .../CachePartitionLossStaleMapTest.java | 20 ++--- ...itionStateFromOwnerRollingUpgradeTest.java | 75 +++++++++---------- 2 files changed, 46 insertions(+), 49 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java index 0711efbae1237..5849352faad99 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java @@ -297,14 +297,15 @@ public void testStaleMapBeforeFullMessage() throws Exception { * The coordinator creates these partitions as MOVING, sends the full message, and only then owns them. When its * last RENTING partition is evicted, the map changes and the coordinator schedules a resend. If the resend fires * before the coordinator owns the partitions, it carries them as MOVING. Its version is newer than the full - * message's, so node 1 takes it. + * message's, so node 1 takes it. Node 1 sees the partitions OWNING only once the coordinator sends its map after + * owning them. *

    * The test: *

      *
    1. holds the eviction of one partition that the coordinator gave to node 1;
      2. *
    2. stops node 2 and stalls the coordinator right after it sends the full message;
      3. *
    3. lets the eviction finish once node 1 has finished the exchange;
      4. - *
    4. checks that node 1 took the stale map, then lets the coordinator go on.
      5. + *
    5. lets the coordinator go on once node 1 has taken the resent map.
      6. *
    * * @throws Exception If failed. @@ -354,15 +355,14 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { assertTrue(waitForCondition(() -> exchangeFinished(other, leaveVer), WAIT)); - evictionResume.countDown(); - GridDhtPartitionTopology otherTop = other.cachex(CACHE).context().topology(); - assertTrue("Node 1 didn't take a stale map of the coordinator", waitForCondition(() -> { - GridDhtPartitionMap crdMap = otherTop.partitionMap(false).get(crdId); + long seq = otherTop.partitionMap(false).get(crdId).updateSequence(); + + evictionResume.countDown(); - return crdMap != null && crdMap.hasMovingPartitions(); - }, 10_000)); + assertTrue("Node 1 didn't take the coordinator's resent map", waitForCondition( + () -> otherTop.partitionMap(false).get(crdId).updateSequence() > seq, WAIT)); crdResume.countDown(); @@ -407,8 +407,8 @@ private IgniteEx startNodes() throws Exception { } /** - * Under the IGNORE policy runs a query on each viewer. Then checks that the partition map of each viewer matches the - * local states of the nodes. + * Under the IGNORE policy runs a query on each viewer. Then checks that the partition map of each viewer matches + * the local states of the nodes. * * @param viewers Nodes whose partition maps are checked. * @throws Exception If failed. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java index 52438fb4f6858..51de749d7755e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.CU; import org.junit.Test; import static org.apache.ignite.cache.PartitionLossPolicy.IGNORE; @@ -41,13 +42,14 @@ import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; /** - * Checks that the coordinator writes the state of the new owners of lost partitions into their partition maps itself - * only until {@code PART_STATE_FROM_OWNER_FEATURE} is active. An older node doesn't send its map after it owns a lost - * partition, so until every node runs the new version the coordinator can't wait for that map. + * Checks that the other nodes write the state of the new owners of lost partitions into their partition maps + * themselves only until {@code PART_STATE_FROM_OWNER_FEATURE} is active. An older node doesn't send its map after it + * owns a lost partition, so until every node runs the new version the others can't wait for that map. *

    * In both parts a server leaves with the only copies of some partitions under the IGNORE policy, while the maps that - * the other servers send outside an exchange can't reach the coordinator. The servers send their maps as before, so - * dropping the maps stands for an older node that doesn't send them. + * the other servers send outside an exchange are held. The servers send their maps as before, so holding the maps + * stands for an older node that doesn't send them. The coordinator and the client watch the new owners' partitions. + * The client hasn't started the cache, so it keeps the cache's partition map in a client topology. */ public class PartitionStateFromOwnerRollingUpgradeTest extends AbstractRollingUpgradeTest { /** */ @@ -59,13 +61,16 @@ public class PartitionStateFromOwnerRollingUpgradeTest extends AbstractRollingUp /** Version without {@code PART_STATE_FROM_OWNER_FEATURE}. */ private static final String OLD_VER = "2.18.0"; + /** Index of the client node, as started by {@link #startCluster(String)}. */ + private static final int CLIENT = 2; + /** Index of the server that leaves. */ private static final int LEAVING = 3; /** - * Before the upgrade the coordinator sees the new owners right after the exchange, with their maps dropped. After - * all nodes are upgraded and the version is finalized, the feature is active, and the coordinator sees the new - * owners only once their maps arrive. + * Before the upgrade the coordinator and the client see the new owners right after the exchange, with the owners' + * maps held. Once all nodes are upgraded and the version is finalized, the feature is active, and they see the new + * owners only after the maps arrive. * * @throws Exception If failed. */ @@ -83,49 +88,50 @@ public void testOwnersReportStateAfterUpgrade() throws Exception { assertFalse(isFeatureActive(grid(0), PART_STATE_FROM_OWNER_FEATURE)); - IgniteEx crd = stopLeavingWithMapsDropped(); + IgniteEx oldCrd = stopLeavingWithMapsHeld(); - assertTrue("The coordinator didn't write the new owners' state: " + mismatches(crd), - mismatches(crd).isEmpty()); + for (IgniteEx viewer : new IgniteEx[] {oldCrd, grid(CLIENT)}) { + assertTrue(viewer.name() + " didn't write the new owners' state: " + mismatches(viewer), + mismatches(viewer).isEmpty()); + } TestRecordingCommunicationSpi.stopBlockAll(); ru(0).enableVersionUpgrade(); upgradeNodeVersion(1, OLD_VER, MASTER_VER_REF); - upgradeNodeVersion(2, OLD_VER, MASTER_VER_REF); + upgradeNodeVersion(CLIENT, OLD_VER, MASTER_VER_REF); upgradeNodeVersion(0, OLD_VER, MASTER_VER_REF); + // Also checks that every node has the features of the new version active. finalizeClusterVersion(1, MASTER_VER_REF); - for (Ignite g : G.allGrids()) - assertTrue(g.name(), isFeatureActive(g, PART_STATE_FROM_OWNER_FEATURE)); - startGrid(LEAVING, MASTER_VER_REF); awaitPartitionMapExchange(); - crd = stopLeavingWithMapsDropped(); + IgniteEx crd = stopLeavingWithMapsHeld(); - assertFalse("The coordinator wrote the new owners' state itself", mismatches(crd).isEmpty()); + for (IgniteEx viewer : new IgniteEx[] {crd, grid(CLIENT)}) + assertFalse(viewer.name() + " wrote the new owners' state itself", mismatches(viewer).isEmpty()); TestRecordingCommunicationSpi.stopBlockAll(); - IgniteEx crd0 = crd; - - if (!waitForCondition(() -> mismatches(crd0).isEmpty(), 10_000)) - fail("The coordinator's partition map differs from the nodes: " + mismatches(crd)); + for (IgniteEx viewer : new IgniteEx[] {crd, grid(CLIENT)}) { + if (!waitForCondition(() -> mismatches(viewer).isEmpty(), 10_000)) + fail("The partition map of " + viewer.name() + " differs from the nodes: " + mismatches(viewer)); + } } /** - * Stops the leaving server while the other servers' partition maps sent outside an exchange are dropped, and waits + * Stops the leaving server while the other servers' partition maps sent outside an exchange are held, and waits * for every node to finish the exchange. * * @return Coordinator. * @throws Exception If failed. */ - private IgniteEx stopLeavingWithMapsDropped() throws Exception { - IgniteEx crd = coordinator(); + private IgniteEx stopLeavingWithMapsHeld() throws Exception { + IgniteEx crd = grid(coordinatorIndex()); UUID crdId = crd.localNode().id(); @@ -153,25 +159,16 @@ private IgniteEx stopLeavingWithMapsDropped() throws Exception { return crd; } - /** @return The oldest server node. */ - private IgniteEx coordinator() { - IgniteEx crd = null; - - for (Ignite g : G.allGrids()) { - if (!g.cluster().localNode().isClient() - && (crd == null || g.cluster().localNode().order() < crd.localNode().order())) - crd = (IgniteEx)g; - } - - return crd; - } - /** - * @param viewer Node whose partition map is checked. + * @param viewer Server with the cache started, or the client, which keeps the cache in a client topology. * @return Partitions whose state in the viewer's map differs from the state on the server itself. */ private List mismatches(IgniteEx viewer) { - GridDhtPartitionTopology viewerTop = viewer.cachex(CACHE).context().topology(); + GridDhtPartitionTopology viewerTop = viewer.cluster().localNode().isClient() + ? viewer.context().cache().context().exchange().clientTopologyIfExists(CU.cacheId(CACHE)) + : viewer.cachex(CACHE).context().topology(); + + assertNotNull(viewer.name(), viewerTop); List res = new ArrayList<>(); From dc6da597214fa054ed109e4e4a00b2af2eb7fcbe Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 29 Oct 2026 18:44:56 +0300 Subject: [PATCH 21/27] IGNITE-29088 Remove redundant check --- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index b23b115d5296b..3fc74cb04ba58 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -3526,12 +3526,8 @@ private Collection detectLostPartitions(AffinityTopologyVersi cctx.kernalContext().pools().getSystemExecutorService(), cctx.affinity().cacheGroups().values(), desc -> { - if (partitionTopology(desc.groupId()).detectLostPartitions(resTopVer, this)) { - CacheGroupContext grp = cctx.cache().cacheGroup(desc.groupId()); - - if (grp != null) - changedGrps.add(grp); - } + if (partitionTopology(desc.groupId()).detectLostPartitions(resTopVer, this)) + changedGrps.add(cctx.cache().cacheGroup(desc.groupId())); return null; }); From 5728a93672d1f0d7f82fbccdb5b6d3287d6fdb0a Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Thu, 29 Oct 2026 18:54:56 +0300 Subject: [PATCH 22/27] IGNITE-29088 Skip the SAFE case in the SQL test --- .../cache/IndexingCachePartitionLossStaleMapTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossStaleMapTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossStaleMapTest.java index 50a2ce4437fe0..9adce867c08b9 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossStaleMapTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossStaleMapTest.java @@ -22,6 +22,8 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.cache.distributed.CachePartitionLossStaleMapTest; +import org.junit.Ignore; +import org.junit.Test; import static java.util.concurrent.TimeUnit.SECONDS; @@ -44,4 +46,11 @@ public class IndexingCachePartitionLossStaleMapTest extends CachePartitionLossSt @Override protected void runQuery(Ignite node) { node.cache(CACHE).query(new SqlFieldsQuery("select count(*) from Integer").setTimeout(10, SECONDS)).getAll(); } + + /** {@inheritDoc} */ + @Test + @Ignore("No query runs under READ_WRITE_SAFE, so the test is the same as in CachePartitionLossStaleMapTest") + @Override public void testStaleMapBeforeLostPartitionMarkedLost() throws Exception { + super.testStaleMapBeforeLostPartitionMarkedLost(); + } } From 96c48e71e9b3cde84affc4a1836a92c8136bf4d4 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 2 Nov 2026 18:06:45 +0300 Subject: [PATCH 23/27] IGNITE-29088 Gate owner reports by feature --- .../preloader/GridDhtPartitionsExchangeFuture.java | 9 +++++++-- .../dht/topology/GridDhtPartitionTopology.java | 1 + .../dht/topology/GridDhtPartitionTopologyImpl.java | 13 +++++++++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 3fc74cb04ba58..b8b9c0fb92ccb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -142,6 +142,7 @@ import static org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents.serverJoinEvent; import static org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents.serverLeftEvent; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.isSnapshotOperation; +import static org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry.PART_STATE_FROM_OWNER_FEATURE; import static org.apache.ignite.internal.util.IgniteUtils.doInParallel; import static org.apache.ignite.internal.util.IgniteUtils.doInParallelUninterruptibly; import static org.apache.ignite.internal.util.lang.ClusterNodeFunc.node2id; @@ -2297,6 +2298,8 @@ private String exchangeTimingsLogMessage(String header, List timings) { Collection locChangedGrps = new HashSet<>(); + boolean ownerReports = cctx.kernalContext().rollingUpgrade().features().isActive(PART_STATE_FROM_OWNER_FEATURE); + try { waitUntilNewCachesAreRegistered(); @@ -2328,7 +2331,9 @@ private String exchangeTimingsLogMessage(String header, List timings) { } - if (!grpToRefresh.isEmpty()) { + if (ownerReports) + locChangedGrps.addAll(grpToRefresh); + else if (!grpToRefresh.isEmpty()) { if (log.isDebugEnabled()) log.debug("Refresh partitions due to partitions initialized when affinity ready [" + grpToRefresh.stream().map(CacheGroupContext::name).collect(Collectors.toList()) + ']'); @@ -2398,7 +2403,7 @@ private String exchangeTimingsLogMessage(String header, List timings) { locChangedGrps.add(grp); } - if (!locChangedGrps.isEmpty()) { + if (ownerReports && !locChangedGrps.isEmpty()) { if (log.isDebugEnabled()) { log.debug("Refresh partitions due to local partition changes at the end of exchange [grps=" + locChangedGrps.stream().map(CacheGroupContext::cacheOrGroupName) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java index 88a45125c34a9..224eb41c457e1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java @@ -452,6 +452,7 @@ public Map> resetOwners( /** * Callback on exchange done. * + * @param fut Exchange future, or {@code null} if called outside an exchange. * @param assignment New affinity assignment. * @param updateRebalanceVer {@code True} if need check rebalance state. * @return {@code True} if the state of at least one local partition changed. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index 914b9f339b53e..3b066d17152a4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -1619,6 +1619,9 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD this.lostParts = new HashSet<>(lostParts); if (exchFut != null && !exchFut.activateCluster()) { + boolean ownerReports = + ctx.kernalContext().rollingUpgrade().features().isActive(PART_STATE_FROM_OWNER_FEATURE); + for (Integer part : lostParts) { GridDhtLocalPartition locPart = localPartition(part); @@ -1626,7 +1629,8 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD // part.clearFuture lifecycle will be broken after resetting. // New partition should be created instead. if (locPart != null && locPart.state() != EVICTED) { - changed |= locPart.markLost(); + if (locPart.markLost() && ownerReports) + changed = true; GridDhtPartitionMap locMap = partMap.get(ctx.localNodeId()); @@ -2102,7 +2106,7 @@ else if (isStaleUpdate(cur, parts)) { * @return {@code True} if a local partition was owned. */ private boolean ownOrphans() { - boolean updRmtMaps = !ctx.kernalContext().rollingUpgrade().features().isActive(PART_STATE_FROM_OWNER_FEATURE); + boolean ownerReports = ctx.kernalContext().rollingUpgrade().features().isActive(PART_STATE_FROM_OWNER_FEATURE); boolean changed = false; @@ -2124,10 +2128,11 @@ private boolean ownOrphans() { if (locPart.state() == MOVING && locPart.own()) { changed = true; - updateLocal(p, OWNING, updateSeq.incrementAndGet(), readyTopVer); + if (ownerReports) + updateLocal(p, OWNING, updateSeq.incrementAndGet(), readyTopVer); } - if (updRmtMaps) { + if (!ownerReports) { for (GridDhtPartitionMap map : node2part.values()) { if (map.get(p) != null) map.put(p, OWNING); From 4a60d4850055229b22e2e1962fd6cdc425c32402 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 2 Nov 2026 18:07:11 +0300 Subject: [PATCH 24/27] IGNITE-29088 Fix tests after review --- .../CacheNoSupplierPartitionMapTest.java | 7 +- .../CachePartitionLossStaleMapTest.java | 51 +++++++----- ...itionStateFromOwnerRollingUpgradeTest.java | 81 +++++-------------- 3 files changed, 59 insertions(+), 80 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheNoSupplierPartitionMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheNoSupplierPartitionMapTest.java index 459a35fe3b68d..98efbcd23d26a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheNoSupplierPartitionMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheNoSupplierPartitionMapTest.java @@ -66,6 +66,9 @@ public class CacheNoSupplierPartitionMapTest extends GridCommonAbstractTest { /** Index of the data node. */ private static final int DATA = 3; + /** Time to wait for the partition states, in milliseconds. */ + private static final long WAIT = 10_000; + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { boolean data = igniteInstanceName.equals(getTestIgniteInstanceName(DATA)); @@ -111,14 +114,14 @@ public void testOrphanPartitionsOwned() throws Exception { List parts = data.cachex(CACHE).context().topology().localPartitions(); return parts.size() == PARTS && parts.stream().allMatch(p -> p.state() == OWNING); - }, 10_000)); + }, WAIT)); UUID dataId = data.localNode().id(); for (int idx = 0; idx <= CLIENT; idx++) { IgniteEx watcher = grid(idx); - if (!waitForCondition(() -> notOwning(watcher, dataId).isEmpty(), 10_000)) { + if (!waitForCondition(() -> notOwning(watcher, dataId).isEmpty(), WAIT)) { fail("The partition map of " + watcher.name() + " differs from the data node: " + notOwning(watcher, dataId)); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java index 5849352faad99..7402673d95ad5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java @@ -31,7 +31,6 @@ import org.apache.ignite.cache.affinity.AffinityFunction; import org.apache.ignite.cache.affinity.AffinityFunctionContext; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; -import org.apache.ignite.cache.query.ScanQuery; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -40,6 +39,7 @@ import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.IgniteInternalCache; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage; @@ -51,6 +51,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl; import org.apache.ignite.internal.processors.resource.DependencyResolver; import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.plugin.extensions.communication.Message; @@ -119,7 +120,7 @@ public class CachePartitionLossStaleMapTest extends GridCommonAbstractTest { private final CountDownLatch evictionResume = new CountDownLatch(1); /** When set, the coordinator waits for {@link #crdResume} right after it sends the next exchange full message. */ - private final AtomicBoolean stallAfterFullMessage = new AtomicBoolean(); + private final AtomicBoolean stallAfterFullMsg = new AtomicBoolean(); /** Counted down when the coordinator starts waiting. */ private final CountDownLatch crdStalled = new CountDownLatch(1); @@ -216,9 +217,9 @@ private void checkStaleMapBeforeLostPartitionChanged(PartitionLossPolicy plc) th stopGrid(LEAVING); - assertTrue("Node 1 didn't change a lost partition", sndSpi.waitForBlocked(1, WAIT)); + assertTrue("Node 1 didn't change a lost partition", waitForCondition(() -> !sendBeforeChange.get(), WAIT)); - assertFalse("Node 1 didn't change a lost partition", sendBeforeChange.get()); + assertTrue("Node 1's map wasn't held", sndSpi.waitForBlocked(1, WAIT)); assertTrue(waitForCondition(() -> exchangeFinished(crd, leaveVer), WAIT)); @@ -345,7 +346,7 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { UUID crdId = crd.localNode().id(); - stallAfterFullMessage.set(true); + stallAfterFullMsg.set(true); long leaveVer = crd.cluster().topologyVersion() + 1; @@ -372,12 +373,13 @@ public void testStaleCoordinatorMapAfterEviction() throws Exception { } /** - * Runs a query over the cache on a node. Called while the node may still see a lost partition without an owner. + * Runs a query over the cache on a node that may still see a lost partition without an owner. Does nothing here: + * scans and key operations don't fail in that case. The SQL subclass runs a SQL query. * * @param node Node. */ protected void runQuery(Ignite node) { - node.cache(CACHE).query(new ScanQuery<>()).getAll(); + // No-op. } /** @@ -420,8 +422,10 @@ private void checkNodes(IgniteEx... viewers) throws Exception { } for (IgniteEx viewer : viewers) { - if (!waitForCondition(() -> mismatches(viewer).isEmpty(), 10_000)) - fail("The partition map of " + viewer.name() + " differs from the nodes: " + mismatches(viewer)); + if (!waitForCondition(() -> mismatches(viewer, CACHE).isEmpty(), WAIT)) { + fail("The partition map of " + viewer.name() + " differs from the nodes: " + + mismatches(viewer, CACHE)); + } } } @@ -491,22 +495,31 @@ private boolean exchangeFinished(IgniteEx node, long topVer) { } /** - * @param viewer Node whose partition map is checked. - * @return Partitions whose state in the viewer's map differs from the state on the node itself. + * @param viewer Server or client whose partition map is checked. A client that hasn't started the cache keeps it in + * a client topology. + * @param cacheName Cache name. + * @return Partitions whose state in the viewer's map differs from the state on the server that holds them. */ - private List mismatches(IgniteEx viewer) { - GridDhtPartitionTopology viewerTop = viewer.cachex(CACHE).context().topology(); + public static List mismatches(IgniteEx viewer, String cacheName) { + IgniteInternalCache cache = viewer.cachex(cacheName); + + GridDhtPartitionTopology viewerTop = cache != null + ? cache.context().topology() + : viewer.context().cache().context().exchange().clientTopologyIfExists(CU.cacheId(cacheName)); + + assertNotNull(viewer.name(), viewerTop); List res = new ArrayList<>(); for (Ignite g : G.allGrids()) { - UUID id = g.cluster().localNode().id(); + if (g.cluster().localNode().isClient()) + continue; - GridDhtPartitionMap view = viewerTop.partitionMap(false).get(id); + GridDhtPartitionMap view = viewerTop.partitionMap(false).get(g.cluster().localNode().id()); - GridDhtPartitionTopology top = ((IgniteEx)g).cachex(CACHE).context().topology(); + GridDhtPartitionTopology top = ((IgniteEx)g).cachex(cacheName).context().topology(); - for (int p = 0; p < PARTS; p++) { + for (int p = 0; p < top.partitions(); p++) { GridDhtLocalPartition locPart = top.localPartition(p, AffinityTopologyVersion.NONE, false, true); GridDhtPartitionState loc = locPart == null ? null : locPart.state(); @@ -525,7 +538,7 @@ private List mismatches(IgniteEx viewer) { } /** - * Once {@link #stallAfterFullMessage} is set, holds the coordinator's thread right after it sends an exchange full + * Once {@link #stallAfterFullMsg} is set, holds the coordinator's thread right after it sends an exchange full * message, before it owns its lost partitions. The thread waits for {@link #crdResume}. */ private class StallingCommunicationSpi extends TestRecordingCommunicationSpi { @@ -537,7 +550,7 @@ private class StallingCommunicationSpi extends TestRecordingCommunicationSpi { if (msg instanceof GridIoMessage && ((GridIoMessage)msg).message() instanceof GridDhtPartitionsFullMessage && ((GridDhtPartitionsFullMessage)((GridIoMessage)msg).message()).exchangeId() != null - && stallAfterFullMessage.compareAndSet(true, false)) { + && stallAfterFullMsg.compareAndSet(true, false)) { crdStalled.countDown(); U.awaitQuiet(crdResume); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java index 51de749d7755e..22850b950a8aa 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.rollingupgrade; -import java.util.ArrayList; import java.util.List; import java.util.UUID; import org.apache.ignite.Ignite; @@ -25,31 +24,25 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.TestRecordingCommunicationSpi; -import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; -import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.util.typedef.G; -import org.apache.ignite.internal.util.typedef.internal.CU; import org.junit.Test; import static org.apache.ignite.cache.PartitionLossPolicy.IGNORE; -import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED; +import static org.apache.ignite.internal.processors.cache.distributed.CachePartitionLossStaleMapTest.mismatches; import static org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry.PART_STATE_FROM_OWNER_FEATURE; import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; /** - * Checks that the other nodes write the state of the new owners of lost partitions into their partition maps - * themselves only until {@code PART_STATE_FROM_OWNER_FEATURE} is active. An older node doesn't send its map after it - * owns a lost partition, so until every node runs the new version the others can't wait for that map. + * Checks that until {@code PART_STATE_FROM_OWNER_FEATURE} is active the nodes handle lost partitions as the previous + * version does: the other nodes write the new owners' state into their partition maps themselves. Once the feature is + * active, they take it from the owners' maps. *

    * In both parts a server leaves with the only copies of some partitions under the IGNORE policy, while the maps that - * the other servers send outside an exchange are held. The servers send their maps as before, so holding the maps - * stands for an older node that doesn't send them. The coordinator and the client watch the new owners' partitions. - * The client hasn't started the cache, so it keeps the cache's partition map in a client topology. + * the other servers send outside an exchange are held, so the coordinator sees only the state it wrote itself. The + * coordinator and the client watch the new owners' partitions. The client hasn't started the cache, so it keeps the + * cache's partition map in a client topology. */ public class PartitionStateFromOwnerRollingUpgradeTest extends AbstractRollingUpgradeTest { /** */ @@ -67,6 +60,9 @@ public class PartitionStateFromOwnerRollingUpgradeTest extends AbstractRollingUp /** Index of the server that leaves. */ private static final int LEAVING = 3; + /** Time to wait for the partition maps to match, in milliseconds. */ + private static final long WAIT = 10_000; + /** * Before the upgrade the coordinator and the client see the new owners right after the exchange, with the owners' * maps held. Once all nodes are upgraded and the version is finalized, the feature is active, and they see the new @@ -91,8 +87,9 @@ public void testOwnersReportStateAfterUpgrade() throws Exception { IgniteEx oldCrd = stopLeavingWithMapsHeld(); for (IgniteEx viewer : new IgniteEx[] {oldCrd, grid(CLIENT)}) { - assertTrue(viewer.name() + " didn't write the new owners' state: " + mismatches(viewer), - mismatches(viewer).isEmpty()); + List mismatches = mismatches(viewer, CACHE); + + assertTrue(viewer.name() + " didn't write the new owners' state: " + mismatches, mismatches.isEmpty()); } TestRecordingCommunicationSpi.stopBlockAll(); @@ -112,14 +109,19 @@ public void testOwnersReportStateAfterUpgrade() throws Exception { IgniteEx crd = stopLeavingWithMapsHeld(); - for (IgniteEx viewer : new IgniteEx[] {crd, grid(CLIENT)}) - assertFalse(viewer.name() + " wrote the new owners' state itself", mismatches(viewer).isEmpty()); + for (IgniteEx viewer : new IgniteEx[] {crd, grid(CLIENT)}) { + List mismatches = mismatches(viewer, CACHE); + + // Only the new owners' partitions may differ: the viewer must not have written OWNING for them. + assertTrue(viewer.name() + " wrote the new owners' state itself: " + mismatches, + !mismatches.isEmpty() && mismatches.stream().allMatch(m -> m.contains(" local=OWNING "))); + } TestRecordingCommunicationSpi.stopBlockAll(); for (IgniteEx viewer : new IgniteEx[] {crd, grid(CLIENT)}) { - if (!waitForCondition(() -> mismatches(viewer).isEmpty(), 10_000)) - fail("The partition map of " + viewer.name() + " differs from the nodes: " + mismatches(viewer)); + if (!waitForCondition(() -> mismatches(viewer, CACHE).isEmpty(), WAIT)) + fail("The partition map of " + viewer.name() + " differs from the nodes: " + mismatches(viewer, CACHE)); } } @@ -158,43 +160,4 @@ private IgniteEx stopLeavingWithMapsHeld() throws Exception { return crd; } - - /** - * @param viewer Server with the cache started, or the client, which keeps the cache in a client topology. - * @return Partitions whose state in the viewer's map differs from the state on the server itself. - */ - private List mismatches(IgniteEx viewer) { - GridDhtPartitionTopology viewerTop = viewer.cluster().localNode().isClient() - ? viewer.context().cache().context().exchange().clientTopologyIfExists(CU.cacheId(CACHE)) - : viewer.cachex(CACHE).context().topology(); - - assertNotNull(viewer.name(), viewerTop); - - List res = new ArrayList<>(); - - for (Ignite g : G.allGrids()) { - if (g.cluster().localNode().isClient()) - continue; - - GridDhtPartitionMap view = viewerTop.partitionMap(false).get(g.cluster().localNode().id()); - - GridDhtPartitionTopology top = ((IgniteEx)g).cachex(CACHE).context().topology(); - - for (int p = 0; p < PARTS; p++) { - GridDhtLocalPartition locPart = top.localPartition(p, AffinityTopologyVersion.NONE, false, true); - - GridDhtPartitionState loc = locPart == null ? null : locPart.state(); - GridDhtPartitionState seen = view == null ? null : view.get(p); - - // The map keeps evicted partitions, the node doesn't. - if (seen == EVICTED) - seen = null; - - if (loc != seen) - res.add(g.name() + " p=" + p + " local=" + loc + " seen=" + seen); - } - } - - return res; - } } From 0be92648f172e4bdc9cb24262149484767bb9f8a Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 2 Nov 2026 20:49:22 +0300 Subject: [PATCH 25/27] IGNITE-29088 Revert refresh folding --- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 9 +++------ .../dht/topology/GridClientPartitionTopology.java | 6 +++--- .../dht/topology/GridDhtPartitionTopologyImpl.java | 6 +++--- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index b8b9c0fb92ccb..e3017499d49db 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -2298,8 +2298,6 @@ private String exchangeTimingsLogMessage(String header, List timings) { Collection locChangedGrps = new HashSet<>(); - boolean ownerReports = cctx.kernalContext().rollingUpgrade().features().isActive(PART_STATE_FROM_OWNER_FEATURE); - try { waitUntilNewCachesAreRegistered(); @@ -2331,9 +2329,7 @@ private String exchangeTimingsLogMessage(String header, List timings) { } - if (ownerReports) - locChangedGrps.addAll(grpToRefresh); - else if (!grpToRefresh.isEmpty()) { + if (!grpToRefresh.isEmpty()) { if (log.isDebugEnabled()) log.debug("Refresh partitions due to partitions initialized when affinity ready [" + grpToRefresh.stream().map(CacheGroupContext::name).collect(Collectors.toList()) + ']'); @@ -2403,7 +2399,8 @@ else if (!grpToRefresh.isEmpty()) { locChangedGrps.add(grp); } - if (ownerReports && !locChangedGrps.isEmpty()) { + if (!locChangedGrps.isEmpty() + && cctx.kernalContext().rollingUpgrade().features().isActive(PART_STATE_FROM_OWNER_FEATURE)) { if (log.isDebugEnabled()) { log.debug("Refresh partitions due to local partition changes at the end of exchange [grps=" + locChangedGrps.stream().map(CacheGroupContext::cacheOrGroupName) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java index e64ce2ab24ada..01deb754adcd0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java @@ -1044,8 +1044,8 @@ else if (isStaleUpdate(cur, parts)) { boolean changed = false; - boolean updRmtMaps = - !cctx.kernalContext().rollingUpgrade().features().isActive(PART_STATE_FROM_OWNER_FEATURE); + boolean ownerReports = + cctx.kernalContext().rollingUpgrade().features().isActive(PART_STATE_FROM_OWNER_FEATURE); for (int part = 0; part < parts; part++) { boolean lost = F.contains(lostParts, part); @@ -1075,7 +1075,7 @@ else if (isStaleUpdate(cur, parts)) { } } - if (lost && updRmtMaps) { + if (lost && !ownerReports) { // Update remote maps according to policy. for (Map.Entry entry : node2part.entrySet()) { if (entry.getValue().get(part) != null) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index 3b066d17152a4..02deef309f248 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -2249,8 +2249,8 @@ private void rebuildDiff(AffinityAssignment affAssignment) { boolean changed = false; - boolean updRmtMaps = - !ctx.kernalContext().rollingUpgrade().features().isActive(PART_STATE_FROM_OWNER_FEATURE); + boolean ownerReports = + ctx.kernalContext().rollingUpgrade().features().isActive(PART_STATE_FROM_OWNER_FEATURE); for (int part = 0; part < grp.affinity().partitions(); part++) { boolean lost = F.contains(lostParts, part); @@ -2303,7 +2303,7 @@ private void rebuildDiff(AffinityAssignment affAssignment) { } // Update remote maps according to policy. - if (updRmtMaps) { + if (!ownerReports) { for (Map.Entry entry : node2part.entrySet()) { if (entry.getKey().equals(ctx.localNodeId())) continue; From 4b63b0f75afa050976bfe591d3f421fbe87409d3 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 2 Nov 2026 20:49:44 +0300 Subject: [PATCH 26/27] IGNITE-29088 Resend on feature activation --- .../cache/GridCachePartitionExchangeManager.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index c63df8c543b75..5c7fdf9e1c089 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -104,6 +104,7 @@ import org.apache.ignite.internal.processors.metric.impl.BooleanMetricImpl; import org.apache.ignite.internal.processors.metric.impl.HistogramMetricImpl; import org.apache.ignite.internal.processors.query.schema.SchemaNodeLeaveExchangeWorkerTask; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureManager; import org.apache.ignite.internal.processors.timeout.GridTimeoutObject; import org.apache.ignite.internal.thread.context.OperationContext; import org.apache.ignite.internal.thread.context.Scope; @@ -161,6 +162,7 @@ import static org.apache.ignite.internal.processors.metric.GridMetricManager.PME_OPS_BLOCKED_DURATION; import static org.apache.ignite.internal.processors.metric.GridMetricManager.PME_OPS_BLOCKED_DURATION_HISTOGRAM; import static org.apache.ignite.internal.processors.metric.GridMetricManager.REBALANCED; +import static org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry.PART_STATE_FROM_OWNER_FEATURE; import static org.apache.ignite.internal.processors.task.TaskExecutionOptions.options; import static org.apache.ignite.internal.util.lang.ClusterNodeFunc.nodeIds; @@ -524,6 +526,16 @@ public void onCoordinatorInitialized() { * @param cache Cache. */ public void onLocalJoin(DiscoveryEvent evt, DiscoCache cache) { + IgniteFeatureManager features = cctx.kernalContext().rollingUpgrade().features(); + + boolean activationAhead = features.localVersionFeatures().contains(PART_STATE_FROM_OWNER_FEATURE) + && !features.isActive(PART_STATE_FROM_OWNER_FEATURE); + + // On activation every node resends its maps once: an owner whose last exchange ended before it sent none, + // and the others no longer write the owner's state. + if (activationAhead) + features.listenActivation(PART_STATE_FROM_OWNER_FEATURE, this::scheduleResendPartitions); + discoLsnr.onEvent(evt, cache); } From 0c87dd664f0405611db4590c82af58b4886b7040 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Mon, 2 Nov 2026 20:51:57 +0300 Subject: [PATCH 27/27] IGNITE-29088 Move and tighten RU test --- .../CachePartitionLossStaleMapTest.java | 8 +++++--- ...itionStateFromOwnerRollingUpgradeTest.java | 20 ++++++++++++------- .../testsuites/IgniteBasicTestSuite.java | 2 -- .../testsuites/IgniteCacheTestSuite15.java | 2 ++ 4 files changed, 20 insertions(+), 12 deletions(-) rename modules/core/src/test/java/org/apache/ignite/internal/processors/{rollingupgrade => cache/distributed}/PartitionStateFromOwnerRollingUpgradeTest.java (89%) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java index 7402673d95ad5..a1f154e188127 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionLossStaleMapTest.java @@ -495,12 +495,14 @@ private boolean exchangeFinished(IgniteEx node, long topVer) { } /** - * @param viewer Server or client whose partition map is checked. A client that hasn't started the cache keeps it in - * a client topology. + * Compares the viewer's partition map with the local partition states of the servers. A client that hasn't started + * the cache keeps the map in a client topology. + * + * @param viewer Server or client whose partition map is checked. * @param cacheName Cache name. * @return Partitions whose state in the viewer's map differs from the state on the server that holds them. */ - public static List mismatches(IgniteEx viewer, String cacheName) { + static List mismatches(IgniteEx viewer, String cacheName) { IgniteInternalCache cache = viewer.cachex(cacheName); GridDhtPartitionTopology viewerTop = cache != null diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/PartitionStateFromOwnerRollingUpgradeTest.java similarity index 89% rename from modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java rename to modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/PartitionStateFromOwnerRollingUpgradeTest.java index 22850b950a8aa..a150e2ffbaf27 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/PartitionStateFromOwnerRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/PartitionStateFromOwnerRollingUpgradeTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.rollingupgrade; +package org.apache.ignite.internal.processors.cache.distributed; import java.util.List; import java.util.UUID; @@ -26,12 +26,12 @@ import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; +import org.apache.ignite.internal.processors.rollingupgrade.AbstractRollingUpgradeTest; import org.apache.ignite.internal.util.typedef.G; import org.junit.Test; import static org.apache.ignite.cache.PartitionLossPolicy.IGNORE; import static org.apache.ignite.internal.processors.cache.distributed.CachePartitionLossStaleMapTest.mismatches; -import static org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry.PART_STATE_FROM_OWNER_FEATURE; import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; /** @@ -82,8 +82,6 @@ public void testOwnersReportStateAfterUpgrade() throws Exception { awaitPartitionMapExchange(); - assertFalse(isFeatureActive(grid(0), PART_STATE_FROM_OWNER_FEATURE)); - IgniteEx oldCrd = stopLeavingWithMapsHeld(); for (IgniteEx viewer : new IgniteEx[] {oldCrd, grid(CLIENT)}) { @@ -92,6 +90,13 @@ public void testOwnersReportStateAfterUpgrade() throws Exception { assertTrue(viewer.name() + " didn't write the new owners' state: " + mismatches, mismatches.isEmpty()); } + for (Ignite g : G.allGrids()) { + if (g != oldCrd && !g.cluster().localNode().isClient()) { + assertFalse(g.name() + " sent its partition map before the feature is active", + TestRecordingCommunicationSpi.spi(g).hasBlockedMessages()); + } + } + TestRecordingCommunicationSpi.stopBlockAll(); ru(0).enableVersionUpgrade(); @@ -112,9 +117,10 @@ public void testOwnersReportStateAfterUpgrade() throws Exception { for (IgniteEx viewer : new IgniteEx[] {crd, grid(CLIENT)}) { List mismatches = mismatches(viewer, CACHE); - // Only the new owners' partitions may differ: the viewer must not have written OWNING for them. - assertTrue(viewer.name() + " wrote the new owners' state itself: " + mismatches, - !mismatches.isEmpty() && mismatches.stream().allMatch(m -> m.contains(" local=OWNING "))); + assertFalse(viewer.name() + " wrote the new owners' state itself", mismatches.isEmpty()); + + assertTrue(viewer.name() + " differs from the nodes in more than the new owners' partitions: " + mismatches, + mismatches.stream().allMatch(m -> m.contains(" local=OWNING "))); } TestRecordingCommunicationSpi.stopBlockAll(); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java index 499e7f7756d73..7273f0d23b243 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java @@ -65,7 +65,6 @@ import org.apache.ignite.internal.processors.odbc.SqlListenerUtilsTest; import org.apache.ignite.internal.processors.query.NoQueryEngineNodeTest; import org.apache.ignite.internal.processors.rollingupgrade.CoreVersionRollingUpgradeTest; -import org.apache.ignite.internal.processors.rollingupgrade.PartitionStateFromOwnerRollingUpgradeTest; import org.apache.ignite.internal.processors.rollingupgrade.PluginVersionRollingUpgradeTest; import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureSetTest; import org.apache.ignite.internal.processors.rollingupgrade.feature.ManagementApiVersionValidationTest; @@ -116,7 +115,6 @@ CoreVersionRollingUpgradeTest.class, PluginVersionRollingUpgradeTest.class, RollingUpgradeMessageSerializationTest.class, - PartitionStateFromOwnerRollingUpgradeTest.class, ManagementApiVersionValidationTest.class, GridProductVersionSelfTest.class, GridAffinityAssignmentV2Test.class, diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java index 5bfb5ad021d12..66e500005459a 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite15.java @@ -47,6 +47,7 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheGroupsPartitionLossPolicySelfTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePartitionLossPolicySelfTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheTxIteratorSelfTest; +import org.apache.ignite.internal.processors.cache.distributed.PartitionStateFromOwnerRollingUpgradeTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.CacheManualRebalancingTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.RebalanceMetricsTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest; @@ -102,6 +103,7 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, IgniteCachePartitionLossPolicySelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CachePartitionLossStaleMapTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheNoSupplierPartitionMapTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, PartitionStateFromOwnerRollingUpgradeTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteCacheGroupsPartitionLossPolicySelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheRebalancingSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheManualRebalancingTest.class, ignoredTests);