From 201163bf5c2aeab8eab68a3163381ea47f8cb1b2 Mon Sep 17 00:00:00 2001 From: deardeng Date: Tue, 4 Aug 2026 20:46:06 +0800 Subject: [PATCH 1/3] [improvement](fe) Presize global cloud tablet route sets ### What problem does this PR solve? Issue Number: None Related PR: #66378, #66389 Problem Summary: Cloud tablet route rebuilding creates new current and future global per-backend tablet sets on every `statRouteInfo()` round. Even when route cardinalities are stable, each `ConcurrentHashMap`-backed set starts without a sizing hint and repeatedly replaces its backing table while growing. Use the corresponding previous-generation per-backend tablet count as the initial-capacity hint for each rebuilt global set. The previous maps already remain live until the temporary routes are complete, so the hint does not extend their lifetime. First-round and newly seen backends retain the default unsized `newKeySet()` behavior. Table- and partition-level indexes, routing contents, scheduling decisions, and incremental update semantics are unchanged. A single-threaded JDK 17 multi-scale allocation model representing 4 million tablets across 4 clusters and both current/future global memberships estimates cumulative allocation on this path at 1.94 GiB before and 1.62 GiB after the change, a 16.14% reduction. Approximate post-GC retained heap is unchanged in the stable-cardinality model; these are path-level estimates, not production RSS measurements. ### Release note None ### Check List (For Author) - Test: Unit Test - `./run-fe-ut.sh --run org.apache.doris.cloud.catalog.CloudTabletRebalancerTest` (17 tests passed) - `mvn checkstyle:check -pl fe-core` (0 violations) - Single-threaded JDK 17 multi-scale allocation model - Behavior changed: No - Does this need documentation: No --- .../cloud/catalog/CloudTabletRebalancer.java | 40 +++++++++++- .../catalog/CloudTabletRebalancerTest.java | 64 +++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java index fde9c0d4850741..4da0a53715ece8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java @@ -52,6 +52,7 @@ import org.apache.doris.thrift.TWarmUpCacheAsyncRequest; import org.apache.doris.thrift.TWarmUpCacheAsyncResponse; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Sets; @@ -77,10 +78,13 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.stream.Collectors; public class CloudTabletRebalancer extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(CloudTabletRebalancer.class); + private static final Function> DEFAULT_GLOBAL_TABLET_SET_FACTORY = + ignored -> ConcurrentHashMap.newKeySet(); private volatile ConcurrentHashMap> beToTabletsGlobal = new ConcurrentHashMap>(); @@ -1067,8 +1071,18 @@ void fillBeToTablets(Long be, Long tableId, Long partId, Long indexId, Long tabl ConcurrentHashMap>> beToTabletsInTable, ConcurrentHashMap>>> partToTablets) { + fillBeToTablets(be, tableId, partId, indexId, tabletId, DEFAULT_GLOBAL_TABLET_SET_FACTORY, + globalBeToTablets, beToTabletsInTable, partToTablets); + } + + private void fillBeToTablets(Long be, Long tableId, Long partId, Long indexId, Long tabletId, + Function> globalTabletSetFactory, + ConcurrentHashMap> globalBeToTablets, + ConcurrentHashMap>> beToTabletsInTable, + ConcurrentHashMap>>> + partToTablets) { // global - globalBeToTablets.computeIfAbsent(be, ignored -> ConcurrentHashMap.newKeySet()).add(tabletId); + globalBeToTablets.computeIfAbsent(be, globalTabletSetFactory).add(tabletId); // table ConcurrentHashMap> beToTabletsOfTable = @@ -1083,6 +1097,22 @@ void fillBeToTablets(Long be, Long tableId, Long partId, Long indexId, Long tabl beToTabletsOfIndex.computeIfAbsent(be, ignored -> ConcurrentHashMap.newKeySet()).add(tabletId); } + private Function> newGlobalTabletSetFactory(Map> previousBeToTablets) { + Map> previousRoute = previousBeToTablets == null + ? Collections.emptyMap() : previousBeToTablets; + return be -> { + Set previousTablets = previousRoute.get(be); + int initialCapacity = previousTablets == null ? 0 : previousTablets.size(); + return newGlobalTabletSet(initialCapacity); + }; + } + + @VisibleForTesting + protected Set newGlobalTabletSet(int initialCapacity) { + return initialCapacity == 0 + ? ConcurrentHashMap.newKeySet() : ConcurrentHashMap.newKeySet(initialCapacity); + } + private void enqueueWarmupTask(WarmupTabletTask task) { WarmupBatchKey key = new WarmupBatchKey(task.srcBe, task.destBe); WarmupBatch batch = warmupBatches.computeIfAbsent(key, WarmupBatch::new); @@ -1158,6 +1188,12 @@ private void flushExpiredWarmupBatches() { } public void statRouteInfo() { + // The previous generation remains live until the temporary global routes are complete, so reuse its + // per-backend cardinalities as allocation hints without extending its lifetime. + Function> currentGlobalTabletSetFactory = + newGlobalTabletSetFactory(beToTabletsGlobal); + Function> futureGlobalTabletSetFactory = + newGlobalTabletSetFactory(futureBeToTabletsGlobal); ConcurrentHashMap> tmpBeToTabletsGlobal = new ConcurrentHashMap>(); ConcurrentHashMap> tmpFutureBeToTabletsGlobal = new ConcurrentHashMap>(); ConcurrentHashMap> tmpBeToTabletsGlobalInSecondary @@ -1237,9 +1273,11 @@ public void statRouteInfo() { Long futureBeId = task == null ? beId : Long.valueOf(task.destBe); Long routeTabletId = task == null ? tabletId : task.pickedTabletId; fillBeToTablets(beId, tableId, partitionId, indexId, routeTabletId, + currentGlobalTabletSetFactory, tmpBeToTabletsGlobal, beToTabletsInTable, this.partitionToTablets); fillBeToTablets(futureBeId, tableId, partitionId, indexId, routeTabletId, + futureGlobalTabletSetFactory, tmpFutureBeToTabletsGlobal, futureBeToTabletsInTable, futurePartitionToTablets); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java index 520932b5fb6283..cb5ccb7ff894ae 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java @@ -97,6 +97,16 @@ protected boolean isInternalDbId(Long dbId) { } } + private static class CapacityTrackingRebalancer extends TestRebalancer { + private final List globalTabletSetInitialCapacities = new ArrayList<>(); + + @Override + protected Set newGlobalTabletSet(int initialCapacity) { + globalTabletSetInitialCapacities.add(initialCapacity); + return ConcurrentHashMap.newKeySet(); + } + } + private static class CountingConcurrentHashMap extends ConcurrentHashMap { private int computeIfAbsentCalls; private int getCalls; @@ -349,6 +359,60 @@ public void testWarmupRollbackReusesInflightBoxedTabletIdAfterRouteRebuild() thr } } + @Test + public void testStatRouteInfoPresizesGlobalTabletSetsFromPreviousRoute() throws Exception { + CapacityTrackingRebalancer rebalancer = new CapacityTrackingRebalancer(); + Long dbId = 10_001L; + Long tableId = 20_001L; + Long partitionId = 30_001L; + Long indexId = 40_001L; + Long tabletId = 50_001L; + Long beId = 60_001L; + String clusterId = "cluster-a"; + + ConcurrentHashMap> previousCurrent = new ConcurrentHashMap<>(); + previousCurrent.put(beId, Set.of(1L, 2L, 3L)); + ConcurrentHashMap> previousFuture = new ConcurrentHashMap<>(); + previousFuture.put(beId, Set.of(1L, 2L, 3L, 4L, 5L)); + setField(rebalancer, "beToTabletsGlobal", previousCurrent); + setField(rebalancer, "futureBeToTabletsGlobal", previousFuture); + setField(rebalancer, "clusterToBes", Collections.singletonMap(clusterId, List.of(beId))); + setField(rebalancer, "allBes", Set.of(beId)); + + try (MockedStatic ignored = mockRouteEnvironment( + dbId, tableId, partitionId, indexId, tabletId, clusterId, beId)) { + rebalancer.statRouteInfo(); + } + + Assertions.assertEquals(List.of(3, 5), rebalancer.globalTabletSetInitialCapacities); + ConcurrentHashMap> current = getField(rebalancer, "beToTabletsGlobal"); + ConcurrentHashMap> future = getField(rebalancer, "futureBeToTabletsGlobal"); + Assertions.assertEquals(Set.of(tabletId), current.get(beId)); + Assertions.assertEquals(Set.of(tabletId), future.get(beId)); + } + + @Test + public void testStatRouteInfoUsesZeroCapacityForNewBackend() throws Exception { + CapacityTrackingRebalancer rebalancer = new CapacityTrackingRebalancer(); + Long dbId = 10_001L; + Long tableId = 20_001L; + Long partitionId = 30_001L; + Long indexId = 40_001L; + Long tabletId = 50_001L; + Long beId = 60_001L; + String clusterId = "cluster-a"; + + setField(rebalancer, "clusterToBes", Collections.singletonMap(clusterId, List.of(beId))); + setField(rebalancer, "allBes", Set.of(beId)); + + try (MockedStatic ignored = mockRouteEnvironment( + dbId, tableId, partitionId, indexId, tabletId, clusterId, beId)) { + rebalancer.statRouteInfo(); + } + + Assertions.assertEquals(List.of(0, 0), rebalancer.globalTabletSetInitialCapacities); + } + private static void initializeRouteMaps(TestRebalancer rebalancer, RouteMaps current, RouteMaps future, Long srcBe, Long tableId, Long partitionId, Long indexId, Long tabletId) throws Exception { rebalancer.fillBeToTablets(srcBe, tableId, partitionId, indexId, tabletId, From d31788b80ed711d58d2bec72859e5683031ce186 Mon Sep 17 00:00:00 2001 From: deardeng Date: Wed, 5 Aug 2026 12:18:49 +0800 Subject: [PATCH 2/3] [test](fe) Add sharp-shrink reproducer for global route set sizing ### What problem does this PR solve? Issue Number: None Related PR: #66447 Problem Summary: Global cloud tablet route rebuilding uses the previous per-backend tablet count as an allocation hint. Add a focused unit-test reproducer showing that a stale count of two million is currently forwarded unchanged even when the rebuilt current and future routes each contain only one tablet. The test expects the hint to be bounded at 1,048,576 while preserving the rebuilt memberships. ### Release note None ### Check List (For Author) - Test: Unit Test - `./run-fe-ut.sh --run org.apache.doris.cloud.catalog.CloudTabletRebalancerTest` (expected RED: 18 tests run, the new sharp-shrink assertion is the only failure) - Behavior changed: No - Does this need documentation: No --- .../catalog/CloudTabletRebalancerTest.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java index cb5ccb7ff894ae..5b3f636846db9a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java @@ -391,6 +391,42 @@ public void testStatRouteInfoPresizesGlobalTabletSetsFromPreviousRoute() throws Assertions.assertEquals(Set.of(tabletId), future.get(beId)); } + @Test + @SuppressWarnings("unchecked") + public void testStatRouteInfoBoundsStaleGlobalTabletSetCapacity() throws Exception { + CapacityTrackingRebalancer rebalancer = new CapacityTrackingRebalancer(); + Long dbId = 10_001L; + Long tableId = 20_001L; + Long partitionId = 30_001L; + Long indexId = 40_001L; + Long tabletId = 50_001L; + Long beId = 60_001L; + String clusterId = "cluster-a"; + + Set stalePreviousTablets = Mockito.mock(Set.class); + Mockito.when(stalePreviousTablets.size()).thenReturn(2_000_000); + ConcurrentHashMap> previousCurrent = new ConcurrentHashMap<>(); + previousCurrent.put(beId, stalePreviousTablets); + ConcurrentHashMap> previousFuture = new ConcurrentHashMap<>(); + previousFuture.put(beId, stalePreviousTablets); + setField(rebalancer, "beToTabletsGlobal", previousCurrent); + setField(rebalancer, "futureBeToTabletsGlobal", previousFuture); + setField(rebalancer, "clusterToBes", Collections.singletonMap(clusterId, List.of(beId))); + setField(rebalancer, "allBes", Set.of(beId)); + + try (MockedStatic ignored = mockRouteEnvironment( + dbId, tableId, partitionId, indexId, tabletId, clusterId, beId)) { + rebalancer.statRouteInfo(); + } + + Assertions.assertEquals(List.of(1_048_576, 1_048_576), + rebalancer.globalTabletSetInitialCapacities); + ConcurrentHashMap> current = getField(rebalancer, "beToTabletsGlobal"); + ConcurrentHashMap> future = getField(rebalancer, "futureBeToTabletsGlobal"); + Assertions.assertEquals(Set.of(tabletId), current.get(beId)); + Assertions.assertEquals(Set.of(tabletId), future.get(beId)); + } + @Test public void testStatRouteInfoUsesZeroCapacityForNewBackend() throws Exception { CapacityTrackingRebalancer rebalancer = new CapacityTrackingRebalancer(); From 493b2a5f14004cfe80ca26e7cad787101b42532c Mon Sep 17 00:00:00 2001 From: deardeng Date: Wed, 5 Aug 2026 12:28:37 +0800 Subject: [PATCH 3/3] [fix](fe) Bound cloud tablet route set allocation hints ### What problem does this PR solve? Issue Number: None Related PR: #66447 Problem Summary: Cloud tablet route rebuilding uses the previous generation per-backend tablet count to presize current and future global route sets. After a sharp catalog shrink, an obsolete multi-million-tablet count could force both replacement sets to allocate oversized ConcurrentHashMap backing tables even when they retain only a few tablets. Bound the previous-generation hint at 1,048,576 entries while preserving exact hints below that threshold and unchanged route memberships. A single-threaded JDK 17 model matching the production uncompressed-oops mode estimates that rebuilding two one-tablet sets from a four-million-tablet previous count reduces allocation from 128.01 MiB to 32.01 MiB (75.00%) and approximate post-full-GC retained heap from 130.00 MiB to 34.00 MiB (73.85%); these are path-level model estimates, not production RSS measurements. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.cloud.catalog.CloudTabletRebalancerTest (18 tests passed) - cd fe && mvn checkstyle:check -pl fe-core (pass) - Single-threaded JDK 17 allocation model with compressed and uncompressed object pointers - Behavior changed: No - Does this need documentation: No --- .../org/apache/doris/cloud/catalog/CloudTabletRebalancer.java | 4 +++- .../apache/doris/cloud/catalog/CloudTabletRebalancerTest.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java index 4da0a53715ece8..935c6fe321fd00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java @@ -83,6 +83,7 @@ public class CloudTabletRebalancer extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(CloudTabletRebalancer.class); + private static final int MAX_GLOBAL_TABLET_SET_INITIAL_CAPACITY = 1 << 20; private static final Function> DEFAULT_GLOBAL_TABLET_SET_FACTORY = ignored -> ConcurrentHashMap.newKeySet(); @@ -1102,7 +1103,8 @@ private Function> newGlobalTabletSetFactory(Map> ? Collections.emptyMap() : previousBeToTablets; return be -> { Set previousTablets = previousRoute.get(be); - int initialCapacity = previousTablets == null ? 0 : previousTablets.size(); + int initialCapacity = previousTablets == null ? 0 + : Math.min(previousTablets.size(), MAX_GLOBAL_TABLET_SET_INITIAL_CAPACITY); return newGlobalTabletSet(initialCapacity); }; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java index 5b3f636846db9a..efbde66d38e963 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudTabletRebalancerTest.java @@ -103,7 +103,7 @@ private static class CapacityTrackingRebalancer extends TestRebalancer { @Override protected Set newGlobalTabletSet(int initialCapacity) { globalTabletSetInitialCapacities.add(initialCapacity); - return ConcurrentHashMap.newKeySet(); + return super.newGlobalTabletSet(initialCapacity); } }