From 2c504fe5d105db787ef60b4103aad7dae88fdee4 Mon Sep 17 00:00:00 2001 From: Di Wang Date: Tue, 22 Sep 2026 23:08:53 +0800 Subject: [PATCH 1/2] Make PROACTIVE the default and grow groups by active series slots up to the maximum RegionGroup number. Cover allocation, recovery, and policy switching with unit and integration tests. --- ...TDBProactiveRegionGroupDistributionIT.java | 1471 +++++++++++++++++ .../IoTDBProactiveRegionGroupExtensionIT.java | 368 +++++ .../confignode/conf/ConfigNodeConfig.java | 70 +- .../confignode/manager/ConfigManager.java | 33 +- .../partition/DataPartitionPolicyTable.java | 18 +- .../manager/partition/PartitionManager.java | 103 +- .../partition/RegionGroupExtensionPolicy.java | 4 +- .../partition/DatabasePartitionTable.java | 16 + .../persistence/partition/PartitionInfo.java | 12 + .../RegionGroupExtensionPolicyConfigTest.java | 106 ++ ...figManagerRegionGroupPolicyReloadTest.java | 150 ++ ...DataPartitionPolicyTableProactiveTest.java | 190 +++ ...artitionManagerProactiveExtensionTest.java | 811 +++++++++ .../conf/iotdb-system.properties.template | 77 +- 14 files changed, 3364 insertions(+), 65 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupDistributionIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupExtensionIT.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/conf/RegionGroupExtensionPolicyConfigTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerRegionGroupPolicyReloadTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTableProactiveTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/partition/PartitionManagerProactiveExtensionTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupDistributionIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupDistributionIT.java new file mode 100644 index 0000000000000..980f07b2d53c1 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupDistributionIT.java @@ -0,0 +1,1471 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.partition; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.cluster.RegionStatus; +import org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchemaResp; +import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; +import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionReq; +import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionTableResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_SCOPE_BINARY; +import static org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils.generatePatternTreeBuffer; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBProactiveRegionGroupDistributionIT { + + private static final String AUTO_DATABASE = "root.auto"; + private static final String PROACTIVE_DATABASE = "root.proactive"; + private static final String CUSTOM_DATABASE = "root.custom"; + // 1000 possible hash slots, but the six devices deliberately activate only slots 0 through 5. + // Each device has one measurement; a new time partition adds no new series slot. + private static final int SERIES_SLOT_NUM = 1000; + private static final int DEVICE_COUNT = 6; + // Group counts below are logical groups: G groups correspond to 3 * G replica rows. + private static final int REPLICATION_FACTOR = 3; + private static final long TIME_PARTITION_INTERVAL = 10; + private static final BKDRHashExecutor PARTITION_EXECUTOR = new BKDRHashExecutor(SERIES_SLOT_NUM); + + /** + * Clean up the test cluster; for example, stop one ConfigNode and three DataNodes after a 1C3D + * case. + */ + @After + public void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + /** + * Verify six distinct slots on 1C3D: PROACTIVE uses three groups per type with two slots each; + * AUTO uses one Schema group and two Data groups. + */ + @Test + public void testSixDevicesOnThreeDataNodes() throws Exception { + // Two databases: cap = 3 groups per type per database. PROACTIVE: 6 slots / 3 groups = 2. + checkSparseDeviceDistribution(3); + } + + /** + * Verify six distinct slots on 1C6D: PROACTIVE uses six groups per type with one slot each, + * placing 18 replicas as three per node. + */ + @Test + public void testSixDevicesOnSixDataNodes() throws Exception { + // Cap = 6; PROACTIVE uses 6 groups per type, each owning one of the six slots. + checkSparseDeviceDistribution(6); + } + + /** + * Verify sparse allocation on 1C9D: six active slots create only six groups despite a cap of + * nine, placing 18 replicas as two per node. + */ + @Test + public void testSixDevicesOnNineDataNodes() throws Exception { + // Cap = 9, but only 6 slots are active: PROACTIVE stops at 6 groups per type. + // Each type has 6 * 3 = 18 replicas, balanced across 9 DataNodes: 2 replicas per node. + checkSparseDeviceDistribution(9); + } + + /** + * Verify AUTO-to-PROACTIVE redistribution: two groups with three slots each become six groups + * with one slot each. Each old group retains one slot and each of four new groups receives one. + * Also verify historical mappings and SQL values. + */ + @Test + public void testAutoToProactiveHotReloadRedistributesSixSlots() throws Exception { + // Step 1: one database on 1C3D; AUTO starts with minimums of 1 schema group / 2 data groups. + initAutoCluster(3); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + // Expect successful creation (200): Schema/Data minimums 1/2 are valid, and three nodes + // support three replicas. + assertSuccess( + client.setDatabase( + new TDatabaseSchema(AUTO_DATABASE) + .setMinSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2))); + // Cap per type = ceil(6 groups per node * 3 nodes / (1 database * 3 replicas)) = 6. + // Expect both caps to be 6: ceil(6*3/(1 database*3 replicas))=6. + assertMaximum(client, AUTO_DATABASE, DEVICE_COUNT); + // Creates 6 devices / measurements / series slots and writes t=0 and t=10. + // Result: 1 schema group, 2 data groups, 6 schema partitions and 6 * 2 = 12 data partitions. + checkSixDevices(client, statement, AUTO_DATABASE, 3, false); + List devices = generateDeviceNamesForSlots(AUTO_DATABASE, DEVICE_COUNT); + Set oldGroups = + // Expect 2 Data groups and 6 replicas: AUTO has reached its minimum; six sparse slots + // do not trigger further growth. + assertRegionGroupsAndReplicas(client, AUTO_DATABASE, TConsensusGroupType.DataRegion, 2) + .keySet(); + Map>> original = + readDataPartitions(client, AUTO_DATABASE, devices, 0); + List originalAssignments = groupsAtTime(original, 0); + oldGroups.forEach( + // Expect 3 slots per old group: six distinct slots are evenly split between two AUTO + // groups, 6/2=3. + group -> Assert.assertEquals(3, Collections.frequency(originalAssignments, group))); + + // Step 2: t=20 raises the data-partition count to 6 * 3 = 18, still only 6 series slots. + // AUTO keeps 2 data groups with 3 slots each; verify the same slot-to-group routing as t=0. + writeAtTime(statement, devices, 2 * TIME_PARTITION_INTERVAL, 200); + // Expect the same two group IDs: adding only a time partition does not make AUTO extend + // or replace groups. + Assert.assertEquals( + oldGroups, + // Expect 2 Data groups: t=20 adds no series slots, so the active-slot count remains + // 6. + assertRegionGroupsAndReplicas(client, AUTO_DATABASE, TConsensusGroupType.DataRegion, 2) + .keySet()); + Map>> beforeReload = + readDataPartitions(client, AUTO_DATABASE, devices, 2 * TIME_PARTITION_INTERVAL); + original.forEach( + (slot, times) -> + // Expect the same owner at t=20 and t=0: AUTO has not extended, so the routing + // stays unchanged. + Assert.assertEquals( + times.get(new TTimePartitionSlot(0)), + beforeReload.get(slot).get(new TTimePartitionSlot(2 * TIME_PARTITION_INTERVAL)))); + + // Step 3: hot-reload only the data policy. Schema stays AUTO; no node is restarted. + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + // Reload alone leaves 2 data groups. The first missing partition at t=30 triggers extension. + // Expect the original two IDs: reload alone creates no groups; a later missing-partition + // request triggers extension. + Assert.assertEquals( + oldGroups, + // Expect 2 Data groups: only the policy has changed; no new time partition has been + // written. + assertRegionGroupsAndReplicas(client, AUTO_DATABASE, TConsensusGroupType.DataRegion, 2) + .keySet()); + writeAtTime(statement, devices, 3 * TIME_PARTITION_INTERVAL, 300); + // Target = min(6 active slots, cap 6) = 6: keep the old 2 groups and create 6 - 2 = 4. + // t=0/10/20/30 now give 24 data partitions in total, still derived from 6 series slots. + Set allGroups = + // Expect 6 Data groups and 18 replicas: PROACTIVE targets min(6 active slots, cap + // 6)=6. + assertRegionGroupsAndReplicas( + client, AUTO_DATABASE, TConsensusGroupType.DataRegion, DEVICE_COUNT) + .keySet(); + // Expect true: all six groups must include the two old groups; extension must not delete + // or replace them. + Assert.assertTrue(allGroups.containsAll(oldGroups)); + Set newGroups = new HashSet<>(allGroups); + newGroups.removeAll(oldGroups); + // Expect 4 new groups: target 6 minus the existing 2 gives 6-2=4. + Assert.assertEquals(4, newGroups.size()); + Map>> afterReload = + readDataPartitions(client, AUTO_DATABASE, devices, 3 * TIME_PARTITION_INTERVAL); + // Expect all six groups to be used, with one slot each: 6 slots / 6 groups=1. + assertSlotDistribution(allGroups, groupsAtTime(afterReload, 3 * TIME_PARTITION_INTERVAL)); + + // Step 4: verify ownership, not just the final group count. At rebalance, + // PROACTIVE retains mu = max(1, floor(6 active slots / 6 groups)) = 1 slot per old group; + // AUTO would retain 166. + // Each old group unassigns 3 - 1 = 2 slots: 2 * 2 = 4 slots activate four distinct new + // groups. + // Which slot is retained is randomized, so compare its original owner rather than its ID. + Map retainedByOldGroup = new HashMap<>(); + Set reassignedGroups = new HashSet<>(); + original.forEach( + (slot, times) -> { + TConsensusGroupId previous = times.get(new TTimePartitionSlot(0)).get(0); + TConsensusGroupId current = + afterReload + .get(slot) + .get(new TTimePartitionSlot(3 * TIME_PARTITION_INTERVAL)) + .get(0); + if (oldGroups.contains(current)) { + // Expect a retained slot to keep its previous owner: an old group may retain only + // its own slots. + Assert.assertEquals(previous, current); + retainedByOldGroup.merge(current, 1, Integer::sum); + } else { + // Expect true: each reassigned slot uses a distinct new group, giving one slot to + // each of four new groups. + Assert.assertTrue(reassignedGroups.add(current)); + } + }); + // Expect exactly the original two groups to retain slots: both old groups must retain + // one. + Assert.assertEquals(oldGroups, retainedByOldGroup.keySet()); + // Expect 1 retained slot per old group: PROACTIVE uses mu=max(1,floor(6/6))=1. + retainedByOldGroup.values().forEach(count -> Assert.assertEquals(1, count.intValue())); + // Expect reassignment to use exactly the four new groups, without leaving any unused or + // including old groups. + Assert.assertEquals(newGroups, reassignedGroups); + + // Step 5: repeat the reload and write t=40: 6 data groups, 6 slots, 6 * 5 = 30 partitions. + // Verify stable future routing, unchanged historical mappings, and all five values per + // device. + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + writeAtTime(statement, devices, 4 * TIME_PARTITION_INTERVAL, 400); + Map>> nextPartitions = + readDataPartitions(client, AUTO_DATABASE, devices, 4 * TIME_PARTITION_INTERVAL); + afterReload.forEach( + (slot, times) -> + // Expect identical routing at t=40 and t=30: reapplying the same policy preserves + // the balanced mapping. + Assert.assertEquals( + times.get(new TTimePartitionSlot(3 * TIME_PARTITION_INTERVAL)), + nextPartitions + .get(slot) + .get(new TTimePartitionSlot(4 * TIME_PARTITION_INTERVAL)))); + // Expect the six historical t=0 mappings to match the snapshot: rebalancing affects + // future allocation only. + Assert.assertEquals(original, readDataPartitions(client, AUTO_DATABASE, devices, 0)); + // Expect the six t=20 mappings to remain unchanged, preserving the last time partition + // before reload. + Assert.assertEquals( + beforeReload, + readDataPartitions(client, AUTO_DATABASE, devices, 2 * TIME_PARTITION_INTERVAL)); + // Expect the same six group IDs after another reload and time partition, with no further + // group creation. + Assert.assertEquals( + allGroups, + // Expect 6 Data groups: there are still six active series slots, and the database cap + // has been reached. + assertRegionGroupsAndReplicas( + client, AUTO_DATABASE, TConsensusGroupType.DataRegion, DEVICE_COUNT) + .keySet()); + // Expect 1 Schema group and 3 replicas: only the data policy changed, with no new schema + // slots. + assertRegionGroupsAndReplicas(client, AUTO_DATABASE, TConsensusGroupType.SchemaRegion, 1); + // Expect both caps to remain 6: node count, database count, replication and quotas are + // unchanged. + assertMaximum(client, AUTO_DATABASE, DEVICE_COUNT); + for (int time = 0; time <= 4; time++) { + // Expect one row per device at t=0/10/20/30/40, with slot+0/100/200/300/400 from the + // five writes. + assertValuesAtTime(statement, devices, time * TIME_PARTITION_INTERVAL, time * 100); + } + } + } + + /** + * Verify switching a low CUSTOM cap to PROACTIVE: refresh the cap from 2 to 6 and use six groups + * with one slot each for the next time partition. + */ + @Test + public void testCustomBelowProactiveHotReloadRedistributesSixSlots() throws Exception { + checkCustomToResourcePolicy("PROACTIVE"); + } + + /** + * Verify CUSTOM-to-AUTO cap refresh: increase the cap from 2 to 6 while the six sparse slots + * remain in the original two groups. + */ + @Test + public void testCustomToAutoHotReloadRefreshesMaximum() throws Exception { + checkCustomToResourcePolicy("AUTO"); + } + + /** + * Compare switching CUSTOM to resource-based policies. Starting with two groups of three slots, + * PROACTIVE uses six groups of one slot, while AUTO keeps two groups of three. Both caps must + * refresh from 2 to 6. + */ + private void checkCustomToResourcePolicy(String policy) throws Exception { + // Step 1: 1C3D / one database gives a resource cap of 6, but CUSTOM explicitly limits data to + // 2. + // At t=0, six devices activate six slots: 1 schema group, 2 data groups, 3 data slots per + // group. + initAutoCluster(3); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='CUSTOM'"); + // Expect successful creation (200): CUSTOM cap 2 equals the Data minimum, and three nodes + // support three replicas. + assertSuccess( + client.setDatabase( + new TDatabaseSchema(CUSTOM_DATABASE) + .setMinSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2) + .setMaxDataRegionGroupNum(2))); + // Expect Schema cap 6 and Data cap 2: Schema uses ceil(6*3/(1*3)); Data uses the explicit + // CUSTOM cap. + assertMaximum(client, CUSTOM_DATABASE, 6, 2); + List devices = createSixDevices(statement, CUSTOM_DATABASE); + Set oldGroups = + // Expect 2 Data groups and 6 replicas: CUSTOM allocates directly to its configured + // cap of 2. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 2) + .keySet(); + Map original = + readDataAssignments(client, CUSTOM_DATABASE, devices, 0); + oldGroups.forEach( + // Expect 3 slots per group: six active slots are evenly split between two CUSTOM + // groups. + group -> Assert.assertEquals(3, Collections.frequency(original.values(), group))); + + // Step 2: expect the data cap to refresh from 2 to 6 when leaving CUSTOM; groups remain at 2. + // Change only the policy: no quota edits, database creation or node registration may + // accidentally refresh the CUSTOM cap before the next partition request. + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='" + policy + "'"); + // Expect the same two IDs: the policy switch refreshes the cap but has not yet triggered + // partition allocation. + Assert.assertEquals( + oldGroups, + // Expect 2 Data groups immediately after reload: t=10 has not been written, so + // extension has not run. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 2) + .keySet()); + writeAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + // Step 3: t=10 adds six data partitions (12 total), but still only six active series slots. + // PROACTIVE grows to min(6 slots, cap 6)=6 groups; AUTO stays at its minimum of 2. + Set currentGroups = + // Expect 6 groups for PROACTIVE and 2 for AUTO: only PROACTIVE grows for the six + // active slots. + assertRegionGroupsAndReplicas( + client, + CUSTOM_DATABASE, + TConsensusGroupType.DataRegion, + "PROACTIVE".equals(policy) ? DEVICE_COUNT : 2) + .keySet(); + // Expect both caps to be 6: leaving CUSTOM recalculates the Data cap from resources + // instead of retaining 2. + assertMaximum(client, CUSTOM_DATABASE, 6); + Map current = + readDataAssignments(client, CUSTOM_DATABASE, devices, TIME_PARTITION_INTERVAL); + if ("PROACTIVE".equals(policy)) { + // Retain mu=max(1,6/6)=1 original slot in each old group; reassign 2*(3-1)=4 to new groups. + // Expect one slot in each of six groups: PROACTIVE must actually use the newly created + // groups. + assertSlotDistribution(currentGroups, new ArrayList<>(current.values())); + Set newGroups = new HashSet<>(currentGroups); + newGroups.removeAll(oldGroups); + // Expect 4 new groups: PROACTIVE target 6 minus the original 2 CUSTOM groups. + Assert.assertEquals(4, newGroups.size()); + Map retained = new HashMap<>(); + Set reassigned = new HashSet<>(); + current.forEach( + (slot, group) -> { + if (oldGroups.contains(group)) { + // Expect a retained slot to keep its original owner, without swapping slots + // between old groups. + Assert.assertEquals(original.get(slot), group); + retained.merge(group, 1, Integer::sum); + } else { + // Expect true: the four unassigned slots must each enter a different new group. + Assert.assertTrue(reassigned.add(group)); + } + }); + // Expect both original groups to retain slots; their ID set must match the pre-switch + // snapshot. + Assert.assertEquals(oldGroups, retained.keySet()); + // Expect one retained slot per old group: mu=max(1,6/6)=1; the other four slots enter + // new groups. + retained.values().forEach(count -> Assert.assertEquals(1, count.intValue())); + // Expect exactly the four new groups to receive reassigned slots, verifying that every + // new group is used. + Assert.assertEquals(newGroups, reassigned); + } else { + // AUTO has the same resource cap as PROACTIVE, but six sparse slots still need two groups. + // Expect AUTO routing to remain unchanged: raising the cap from 2 to 6 does not require + // more groups for sparse load. + Assert.assertEquals(original, current); + } + // Step 4: data-policy reload leaves schema at 1 group and preserves t=0 mappings and values. + // Expect the six t=0 mappings to match the snapshot: neither target policy may rewrite + // historical partitions. + Assert.assertEquals(original, readDataAssignments(client, CUSTOM_DATABASE, devices, 0)); + // Expect 1 Schema group and 3 replicas: only the data policy changed; schema load and + // minimum are unchanged. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.SchemaRegion, 1); + // Expect one row per device at t=0 with values 0..5, verifying that pre-switch data + // remains readable. + assertValuesAtTime(statement, devices, 0, 0); + // Expect one row per device at t=10 with values 100..105, verifying the new time + // partition. + assertValuesAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + } + } + + /** + * Verify that existing CUSTOM groups exceeding the PROACTIVE target are preserved. For example, + * eight groups with six active slots and a resource cap of six retain all eight groups and + * existing slot owners. + */ + @Test + public void testCustomAboveProactiveHotReloadPreservesGroups() throws Exception { + // Step 1: CUSTOM creates 8 data groups although six active slots only use six of them. + // There are 8*3=24 data replicas; two logical groups initially have no series slots. + initAutoCluster(3); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='CUSTOM'"); + // Expect successful creation (200): CUSTOM permits an explicit Data cap of 8 above the + // minimum of 2. + assertSuccess( + client.setDatabase( + new TDatabaseSchema(CUSTOM_DATABASE) + .setMinSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2) + .setMaxDataRegionGroupNum(8))); + List devices = createSixDevices(statement, CUSTOM_DATABASE); + Set originalGroups = + // Expect 8 Data groups and 24 replicas: CUSTOM creates up to its cap even with only + // six active slots. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 8) + .keySet(); + Map original = + readDataAssignments(client, CUSTOM_DATABASE, devices, 0); + // Expect 6 used groups: each slot occupies a distinct group, leaving two of the eight + // groups unused. + Assert.assertEquals(DEVICE_COUNT, new HashSet<>(original.values()).size()); + + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + // Step 2: recalculated data cap=max(minimum 2, resource quota 6, existing groups 8)=8. + // Switching to PROACTIVE does not shrink to the six-group active-slot target. + // Expect Schema cap 6 and Data cap 8: max(minimum 2, resource cap 6, existing groups 8)=8 + // prevents shrinking. + assertMaximum(client, CUSTOM_DATABASE, 6, 8); + // Expect all eight IDs to remain: switching to PROACTIVE must not delete groups to match + // six active slots. + Assert.assertEquals( + originalGroups, + // Expect 8 Data groups and 24 replicas: a policy reload must not shrink existing + // groups. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 8) + .keySet()); + writeAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + // Step 3: t=10 gives 12 data partitions; all 8 group IDs and all six slot owners stay + // unchanged. + // Expect the same eight IDs after t=10: existing groups already outnumber the six active + // slots. + Assert.assertEquals( + originalGroups, + // Expect 8 Data groups: a new time partition adds no active series slots and does not + // trigger shrinking. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 8) + .keySet()); + // Expect t=10 routing to match t=0: without new groups, assigned slots keep their + // original owners. + Assert.assertEquals( + original, readDataAssignments(client, CUSTOM_DATABASE, devices, TIME_PARTITION_INTERVAL)); + // Expect the six historical t=0 mappings to remain unchanged: switching policies does not + // migrate partitions. + Assert.assertEquals(original, readDataAssignments(client, CUSTOM_DATABASE, devices, 0)); + // Expect 1 Schema group: only the data policy changed; schema load and minimum are + // unchanged. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.SchemaRegion, 1); + // Expect one row per device at t=0 with values 0..5, confirming historical reads after + // retaining eight groups. + assertValuesAtTime(statement, devices, 0, 0); + // Expect one row per device at t=10 with values 100..105, verifying writes through the + // retained routing. + assertValuesAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + } + } + + /** + * Verify that entering PROACTIVE balances slots even when CUSTOM has already created enough + * groups. Existing time partitions and repeated policy reloads retain their assignments. + */ + @Test + public void testExpandedCustomGroupsRebalanceOnProactiveSwitchWithoutFurtherGrowth() + throws Exception { + initAutoCluster(3); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='CUSTOM'"); + assertSuccess( + client.setDatabase( + new TDatabaseSchema(CUSTOM_DATABASE) + .setMinSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2) + .setMaxDataRegionGroupNum(2))); + List devices = createSixDevices(statement, CUSTOM_DATABASE); + Map original = + readDataAssignments(client, CUSTOM_DATABASE, devices, 0); + Assert.assertEquals(2, new HashSet<>(original.values()).size()); + + statement.execute("ALTER DATABASE " + CUSTOM_DATABASE + " WITH MAX_DATA_REGION_GROUP_NUM=6"); + writeAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + Set expandedGroups = + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 6) + .keySet(); + // CUSTOM keeps all six slots on the original two groups despite creating four more groups. + Assert.assertEquals( + original, readDataAssignments(client, CUSTOM_DATABASE, devices, TIME_PARTITION_INTERVAL)); + + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + writeAtTime(statement, devices, 2 * TIME_PARTITION_INTERVAL, 200); + Assert.assertEquals( + expandedGroups, + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 6) + .keySet()); + Map rebalanced = + readDataAssignments(client, CUSTOM_DATABASE, devices, 2 * TIME_PARTITION_INTERVAL); + assertSlotDistribution(expandedGroups, new ArrayList<>(rebalanced.values())); + + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + writeAtTime(statement, devices, 3 * TIME_PARTITION_INTERVAL, 300); + Assert.assertEquals( + rebalanced, + readDataAssignments(client, CUSTOM_DATABASE, devices, 3 * TIME_PARTITION_INTERVAL)); + Assert.assertEquals(original, readDataAssignments(client, CUSTOM_DATABASE, devices, 0)); + Assert.assertEquals( + original, readDataAssignments(client, CUSTOM_DATABASE, devices, TIME_PARTITION_INTERVAL)); + for (int time = 0; time <= 3; time++) { + assertValuesAtTime(statement, devices, time * TIME_PARTITION_INTERVAL, time * 100); + } + } + } + + /** + * Verify PROACTIVE-to-CUSTOM growth to the cap: grow from six to nine groups, then to ten after + * raising the cap. The six slots keep their owners; also verify rejection of a lower cap and SQL + * values. + */ + @Test + public void testProactiveToCustomHotReloadAllocatesMaximum() throws Exception { + // Step 1: quota 9 per node gives cap=ceil(9*3/(1*3))=9; PROACTIVE activates only 6 groups. + // At t=0, six distinct slots each own one group: 6 data partitions and 6*3=18 data replicas. + initAutoCluster(3, 9); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + // Expect successful creation (200): minimums 1/2 are valid, and three nodes support three + // replicas. + assertSuccess( + client.setDatabase( + new TDatabaseSchema(PROACTIVE_DATABASE) + .setMinSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2))); + // Expect both caps to be 9: the explicit per-node quota is 9, so ceil(9*3/(1*3))=9. + assertMaximum(client, PROACTIVE_DATABASE, 9); + List devices = createSixDevices(statement, PROACTIVE_DATABASE); + Set oldGroups = + // Expect 6 Data groups and 18 replicas: PROACTIVE creates groups for six active + // slots, below the cap of 9. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, DEVICE_COUNT) + .keySet(); + Map original = + readDataAssignments(client, PROACTIVE_DATABASE, devices, 0); + // Expect one slot in each of six groups: six active slots are evenly assigned to six + // PROACTIVE groups. + assertSlotDistribution(oldGroups, new ArrayList<>(original.values())); + + // Step 2: CUSTOM inherits the saved cap of 9; changing the policy alone leaves 6 groups. + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='CUSTOM'"); + // Expect both caps to remain 9: CUSTOM uses the database's current cap rather than + // resetting to a default minimum. + assertMaximum(client, PROACTIVE_DATABASE, 9); + // Expect the original six IDs: changing the policy alone does not create the three + // additional groups. + Assert.assertEquals( + oldGroups, + // Expect 6 Data groups: the missing t=10 partition has not yet been requested, so + // CUSTOM has not extended. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, DEVICE_COUNT) + .keySet()); + writeAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + Set customGroups = + // Expect 9 Data groups and 27 replicas: the first missing t=10 partition makes CUSTOM + // fill the cap of 9. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, 9) + .keySet(); + // Expect true: the nine groups must include the original six, adding exactly 9-6=3 + // groups. + Assert.assertTrue(customGroups.containsAll(oldGroups)); + // Step 3: t=10 triggers 9-6=3 new groups (27 replicas total), giving 12 data partitions. + // CUSTOM retains floor(1000/9)=111 slots per group, so each old group's single slot stays. + // All six slots keep their original groups, leaving the three new groups unused for now. + // Expect all six slots to retain their owners: CUSTOM's threshold floor(1000/9)=111 + // exceeds one slot per old group. + Assert.assertEquals( + original, + readDataAssignments(client, PROACTIVE_DATABASE, devices, TIME_PARTITION_INTERVAL)); + + // Step 4: repeating CUSTOM and writing t=20 preserves 9 groups / 6 slots / the same owners. + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='CUSTOM'"); + writeAtTime(statement, devices, 2 * TIME_PARTITION_INTERVAL, 200); + // Expect the same nine IDs: reapplying CUSTOM finds both the cap and current group count + // already satisfied. + Assert.assertEquals( + customGroups, + // Expect 9 Data groups: t=20 only adds a time partition without raising the cap. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, 9) + .keySet()); + // Expect t=20 routing to match the original six groups: one slot per group remains below + // threshold 111. + Assert.assertEquals( + original, + readDataAssignments(client, PROACTIVE_DATABASE, devices, 2 * TIME_PARTITION_INTERVAL)); + + // Step 5: explicitly raise the CUSTOM cap to 10; t=30 creates one more group (30 replicas). + // mu=1000/10=100 still retains all six slot owners; 6 slots * 4 times = 24 data partitions. + statement.execute( + "ALTER DATABASE " + PROACTIVE_DATABASE + " WITH MAX_DATA_REGION_GROUP_NUM=10"); + writeAtTime(statement, devices, 3 * TIME_PARTITION_INTERVAL, 300); + // Expect true: extending the cap from 9 to 10 must preserve all nine existing groups. + Assert.assertTrue( + // Expect 10 Data groups and 30 replicas: allocation at t=30 fills the new explicit + // CUSTOM cap. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, 10) + .keySet() + .containsAll(customGroups)); + // Expect unchanged routing: the new threshold 1000/10=100 still exceeds one slot per old + // group. + Assert.assertEquals( + original, + readDataAssignments(client, PROACTIVE_DATABASE, devices, 3 * TIME_PARTITION_INTERVAL)); + // Step 6: lowering the cap to 2 must fail; check 10 groups remain and all four times are + // readable. + // Expect SQLException: requested cap 2 is below the current cap and existing group count + // of 10. + Assert.assertThrows( + SQLException.class, + () -> + statement.execute( + "ALTER DATABASE " + PROACTIVE_DATABASE + " WITH MAX_DATA_REGION_GROUP_NUM=2")); + // Expect Schema cap 9 and Data cap 10: only the Data cap increased, and the rejected + // decrease changes nothing. + assertMaximum(client, PROACTIVE_DATABASE, 9, 10); + // Expect 10 Data groups and 30 replicas: rejecting the lower cap must not delete existing + // groups. + assertRegionGroupsAndReplicas(client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, 10); + // Expect the six t=0 mappings to stay unchanged through growth to 9/10 groups and the + // rejected decrease. + Assert.assertEquals(original, readDataAssignments(client, PROACTIVE_DATABASE, devices, 0)); + for (int time = 0; time <= 3; time++) { + // Expect one row per device at t=0/10/20/30, with slot+0/100/200/300 from the four + // writes. + assertValuesAtTime(statement, devices, time * TIME_PARTITION_INTERVAL, time * 100); + } + } + } + + /** + * Verify independent Schema policy reload. Six CUSTOM slots initially share one group; after + * switching to PROACTIVE, new slot 6 triggers growth to cap 6. Existing schema mappings and the + * two Data groups remain unchanged. + */ + @Test + public void testSchemaCustomToProactiveHotReloadRefreshesMaximum() throws Exception { + // Step 1: schema CUSTOM cap=1, data AUTO cap=6; six schema slots all belong to one schema + // group. + initAutoCluster(3); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("SET CONFIGURATION 'schema_region_group_extension_policy'='CUSTOM'"); + // Expect successful creation (200): CUSTOM Schema cap 1 equals its minimum; Data minimum + // 2 is valid. + assertSuccess( + client.setDatabase( + new TDatabaseSchema(CUSTOM_DATABASE) + .setMinSchemaRegionGroupNum(1) + .setMaxSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2))); + // Expect Schema cap 1 and Data cap 6: the former is explicit CUSTOM; the latter uses + // ceil(6*3/(1*3)). + assertMaximum(client, CUSTOM_DATABASE, 1, 6); + List devices = createSixDevices(statement, CUSTOM_DATABASE); + TSchemaPartitionTableResp original = client.getSchemaPartitionTable(schemaRequest(devices)); + // Expect successful query (200): schema partitions for all six existing measurements must + // be readable. + assertSuccess(original.getStatus()); + Set oldGroups = + // Expect 1 Schema group and 3 replicas: CUSTOM cap 1 places all six schema slots in + // that group. + assertRegionGroupsAndReplicas( + client, CUSTOM_DATABASE, TConsensusGroupType.SchemaRegion, 1) + .keySet(); + statement.execute("SET CONFIGURATION 'schema_region_group_extension_policy'='PROACTIVE'"); + // Expect the original Schema group ID: reloading the policy has not yet requested a new + // schema slot. + Assert.assertEquals( + oldGroups, + // Expect 1 Schema group: extension waits for the seventh device's new slot. + assertRegionGroupsAndReplicas( + client, CUSTOM_DATABASE, TConsensusGroupType.SchemaRegion, 1) + .keySet()); + // Step 2: reload only the schema policy, then create a seventh device in a seventh slot. + // Target=min(7 schema slots, cap 6)=6 groups: add 5 groups, while the original six slots stay + // put. + // The new device has no data write, so data stays at 6 active slots and 2 groups. + String newDevice = findDeviceInSlot(CUSTOM_DATABASE, DEVICE_COUNT); + statement.execute("CREATE TIMESERIES " + newDevice + ".s WITH DATATYPE=INT32, ENCODING=RLE"); + // Expect true: the six Schema groups must include the original group, preserving its + // existing mappings. + Assert.assertTrue( + // Expect 6 Schema groups and 18 replicas: the new device raises active slots to 7, so + // min(7,cap 6)=6. + assertRegionGroupsAndReplicas( + client, CUSTOM_DATABASE, TConsensusGroupType.SchemaRegion, 6) + .keySet() + .containsAll(oldGroups)); + // Expect both caps to be 6: Schema resumes resource-based calculation; the Data cap stays + // unchanged. + assertMaximum(client, CUSTOM_DATABASE, 6); + TSchemaPartitionTableResp restored = client.getSchemaPartitionTable(schemaRequest(devices)); + // Step 3: verify the original schema mappings and SQL values survive the schema-only change. + // Expect successful query (200): all six original devices' schema partitions remain + // readable after extension. + assertSuccess(restored.getStatus()); + // Expect unchanged mappings for the original six schema slots: extension assigns new + // slots without moving old ones. + Assert.assertEquals(original.getSchemaPartitionTable(), restored.getSchemaPartitionTable()); + // Expect 2 Data groups and 6 replicas: Data stays AUTO, and no data was written to the + // seventh device. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 2); + // Expect one t=0 row per original device with values 0..5: Schema reload must preserve + // existing reads. + assertValuesAtTime(statement, devices, 0, 0); + } + } + + /** + * Initialize AUTO with an explicit per-node quota of 6. For example, dataNodeCount=3 starts 1C3D + * with three replicas per Region type and 1000 series slots. + */ + private void initAutoCluster(int dataNodeCount) throws Exception { + initAutoCluster(dataNodeCount, 6); + } + + /** + * Initialize AUTO with a specified node quota. For example, three nodes, regionPerDataNode=9 and + * one database with three replicas yield ceil(9*3/3)=9 groups. The quota is not a simulated CPU + * count. + */ + private void initAutoCluster(int dataNodeCount, int regionPerDataNode) throws Exception { + // Shared setup: 1 ConfigNode, N DataNodes, 3 replicas, INHERIT, and 10 ms time partitions. + // Per-type cap depends on regionPerDataNode (normally 6), node count, databases and replicas. + // A nonzero Data quota is used directly without the 0.5 CPU multiplier; quota 6 matches + // automatic sizing with 12 reported cores per node. + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaReplicationFactor(REPLICATION_FACTOR) + .setDataReplicationFactor(REPLICATION_FACTOR) + .setSchemaRegionGroupExtensionPolicy("AUTO") + .setDataRegionGroupExtensionPolicy("AUTO") + .setDataPartitionAllocationStrategy("INHERIT") + .setSchemaRegionPerDataNode(regionPerDataNode) + .setDataRegionPerDataNode(regionPerDataNode) + .setSeriesSlotNum(SERIES_SLOT_NUM) + .setSeriesPartitionExecutorClass(BKDRHashExecutor.class.getName()) + .setTimePartitionInterval(TIME_PARTITION_INTERVAL); + EnvFactory.getEnv().initClusterEnvironment(1, dataNodeCount); + } + + /** + * Run the two-database distribution, reload, cap and recovery scenarios. For example, on 3DN each + * cap is 3: AUTO uses two Data groups of three slots, then PROACTIVE uses three groups of two + * while preserving historical mappings. + */ + private void checkSparseDeviceDistribution(int dataNodeCount) throws Exception { + initAutoCluster(dataNodeCount); + Map>> + originalAutoPartitions; + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + // Step 1: create both databases before allocating any regions, ensuring the same cap for + // each. + // Cap per type = ceil(6 groups per node * N nodes / (2 databases * 3 replicas)) = N. + // Thus the 3D / 6D / 9D cases have caps of 3 / 6 / 9; minimums are schema=1 and data=2. + for (String database : Arrays.asList(AUTO_DATABASE, PROACTIVE_DATABASE)) { + // Expect both creations to succeed (200): minimums 1/2 are valid, and 3/6/9DN all + // support three replicas. + assertSuccess( + client.setDatabase( + new TDatabaseSchema(database) + .setMinSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2))); + } + // Expect both root.auto caps to be 3/6/9 on 3/6/9DN: ceil(6*N/(2 databases*3 + // replicas))=N. + assertMaximum(client, AUTO_DATABASE, dataNodeCount); + // Expect root.proactive caps to be 3/6/9 as well: both databases share the same resource + // calculation. + assertMaximum(client, PROACTIVE_DATABASE, dataNodeCount); + + // Step 2: AUTO database, t=0/10: 6 slots, 1 schema group, 2 data groups, 12 data partitions. + checkSixDevices(client, statement, AUTO_DATABASE, dataNodeCount, false); + + // Step 3: switch both policies, then populate the second database with the same workload. + // K = min(6, N) gives 3 / 6 / 6 groups per type for 3D / 6D / 9D, respectively. + statement.execute("SET CONFIGURATION 'schema_region_group_extension_policy'='PROACTIVE'"); + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + // Expect caps to remain 3/6/9: AUTO and PROACTIVE share the resource formula; nodes and + // database count are unchanged. + assertMaximum(client, PROACTIVE_DATABASE, dataNodeCount); + checkSixDevices(client, statement, PROACTIVE_DATABASE, dataNodeCount, true); + + // Step 4: write the existing AUTO database's six devices at t=20 after the global switch. + // Its data groups grow 2 -> K (3 / 6 / 6); six slots spread as 2 / 1 / 1 slots per group. + // Its schema group stays at 1 because these devices need no new schema partitions. + // Verify all K data groups are used and t=0 keeps its original group IDs (no data migration). + List autoDevices = generateDeviceNamesForSlots(AUTO_DATABASE, DEVICE_COUNT); + originalAutoPartitions = readDataPartitions(client, AUTO_DATABASE, autoDevices, 0); + writeAtTime(statement, autoDevices, 2 * TIME_PARTITION_INTERVAL, 200); + // Expect 3/6/6 Data groups on 3/6/9DN, with 2/1/1 slots per group: target=min(6,N). + // Expect replicas on all 3/6/9 nodes, with 3/3/2 per node: divide 9/18/18 total replicas + // by node count. + assertNewTimeDistribution( + client, AUTO_DATABASE, autoDevices, 2 * TIME_PARTITION_INTERVAL, dataNodeCount); + // Expect one t=20 row per device with slot+200 (200..205): identical writes on 3/6/9DN. + assertValuesAtTime(statement, autoDevices, 2 * TIME_PARTITION_INTERVAL, 200); + // Expect the six t=0 mappings to match the snapshot: growing to three or six groups must + // preserve history. + Assert.assertEquals( + originalAutoPartitions, readDataPartitions(client, AUTO_DATABASE, autoDevices, 0)); + + // Step 5: add distinct slots starting at 6 to the PROACTIVE database only. + // The 3D / 6D / 9D cases add 2 / 2 / 5 slots, giving 8 / 8 / 11 active slots in total. + // Both region types reach/stay at their caps of 3 / 6 / 9 despite exceeding the cap in slots. + List extraDevices = new ArrayList<>(); + for (int slot = DEVICE_COUNT; slot < Math.max(DEVICE_COUNT, dataNodeCount) + 2; slot++) { + extraDevices.add(findDeviceInSlot(PROACTIVE_DATABASE, slot)); + } + TSchemaPartitionTableResp schemaResponse = + client.getOrCreateSchemaPartitionTable(schemaRequest(extraDevices)); + // Expect schema allocation to succeed (200): assign the extra distinct slots, reusing + // groups at the cap. + assertSuccess(schemaResponse.getStatus()); + TDataPartitionTableResp dataResponse = + client.getOrCreateDataPartitionTable(dataRequest(PROACTIVE_DATABASE, extraDevices, 0)); + // Expect data allocation to succeed (200): once capped, new slots must still be assigned + // to existing groups. + assertSuccess(dataResponse.getStatus()); + // Expect 2/2/5 series slots on 3/6/9DN: this request contains only the extra devices, not + // the original six. + Assert.assertEquals( + extraDevices.size(), dataResponse.getDataPartitionTable().get(PROACTIVE_DATABASE).size()); + // Expect 3/6/9 Schema groups: 8/8/11 active slots exceed the respective caps of 3/6/9. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.SchemaRegion, dataNodeCount); + // Expect 3/6/9 Data groups: the cap prevents growing to match all 8/8/11 active slots. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, dataNodeCount); + } + + // Step 6: restart the ConfigNode and rebuild its in-memory allot map from persisted partitions. + // In root.auto, t=30 must still use K=3/6/6 data groups for the same 6 slots (24 partitions + // total). + // Wait for all K * 3 replicas to become Running, then check routing, SQL values and t=0 + // history. + EnvFactory.getEnv().shutdownConfigNode(0); + EnvFactory.getEnv().startConfigNode(0); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + // Expect all replicas of 3/6/6 groups to recover on 3/6/9DN, yielding 9/18/18 Running + // records. + awaitRunningDataRegions(client, AUTO_DATABASE, Math.min(DEVICE_COUNT, dataNodeCount)); + List autoDevices = generateDeviceNamesForSlots(AUTO_DATABASE, DEVICE_COUNT); + writeAtTime(statement, autoDevices, 3 * TIME_PARTITION_INTERVAL, 300); + // Expect 3/6/6 Data groups with 2/1/1 slots each after restart: root.auto still has six + // active slots. + // Expect replicas on all 3/6/9 nodes, with 3/3/2 per node: divide 9/18/18 total replicas + // by node count. + assertNewTimeDistribution( + client, AUTO_DATABASE, autoDevices, 3 * TIME_PARTITION_INTERVAL, dataNodeCount); + // Expect one t=30 row per device with slot+300 (300..305), verifying routing and reads + // after recovery. + assertValuesAtTime(statement, autoDevices, 3 * TIME_PARTITION_INTERVAL, 300); + // Expect the six t=0 mappings to remain unchanged: rebuilding the allocation map must + // preserve persisted history. + Assert.assertEquals( + originalAutoPartitions, readDataPartitions(client, AUTO_DATABASE, autoDevices, 0)); + } + } + + /** + * Create six devices in distinct slots one by one and write t=0 and t=10. Verify groups, + * replicas, routing and SQL values. For example, on 3DN PROACTIVE ends with three groups of two + * slots; AUTO Data uses two groups of three. + */ + private void checkSixDevices( + SyncConfigNodeIServiceClient client, + Statement statement, + String database, + int dataNodeCount, + boolean proactive) + throws Exception { + // Insert devices one at a time so each iteration activates exactly one distinct series slot. + // At iteration i (1..6): AUTO schema=1, data=min(i,2); PROACTIVE schema=data=min(i,N). + List devices = new ArrayList<>(); + for (int slot = 0; slot < DEVICE_COUNT; slot++) { + String device = findDeviceInSlot(database, slot); + devices.add(device); + statement.execute("CREATE TIMESERIES " + device + ".s WITH DATATYPE=INT32, ENCODING=RLE"); + statement.execute("INSERT INTO " + device + "(time,s) VALUES (0," + slot + ")"); + int expectedSchemaGroups = proactive ? Math.min(slot + 1, dataNodeCount) : 1; + int expectedDataGroups = + proactive ? Math.min(slot + 1, dataNodeCount) : Math.min(slot + 1, 2); + // Let i=slot+1. Expect one AUTO Schema group: its minimum is 1 and six slots do not + // trigger proportional growth. + // Expect PROACTIVE counts 1,2,3,3,3,3 on 3DN and 1,2,3,4,5,6 on 6/9DN: min(i,N). + assertRegionGroupsAndReplicas( + client, database, TConsensusGroupType.SchemaRegion, expectedSchemaGroups); + // Expect AUTO Data counts 1,2,2,2,2,2: successive requests gradually reach the minimum of + // 2. + // Expect PROACTIVE counts 1,2,3,3,3,3 on 3DN and 1,2,3,4,5,6 on 6/9DN, bounded by active + // slots and the cap. + assertRegionGroupsAndReplicas( + client, database, TConsensusGroupType.DataRegion, expectedDataGroups); + } + + // After t=0: 6 measurements, 6 schema slots and 6 (series slot, time slot) data partitions. + // For each type, G logical groups produce G * 3 replica rows on distinct nodes within a group. + int expectedSchemaGroups = proactive ? Math.min(DEVICE_COUNT, dataNodeCount) : 1; + int expectedDataGroups = proactive ? Math.min(DEVICE_COUNT, dataNodeCount) : 2; + Map> schemaReplicas = + // Expect Schema counts: AUTO 1 on all 3/6/9DN cases; PROACTIVE 3/6/6, or min(6 + // slots,N). + assertRegionGroupsAndReplicas( + client, database, TConsensusGroupType.SchemaRegion, expectedSchemaGroups); + Map> dataReplicas = + // Expect Data counts: AUTO 2 on all 3/6/9DN cases; PROACTIVE 3/6/6 after activating all + // six slots. + assertRegionGroupsAndReplicas( + client, database, TConsensusGroupType.DataRegion, expectedDataGroups); + // Expect Schema replicas to cover 3/3/3 nodes for AUTO (one three-replica group), and 3/6/9 + // for PROACTIVE. + // Expect 3/3/2 replicas of this type per node for PROACTIVE: 9/18/18 replicas divided by + // 3/6/9 nodes. + assertReplicaDistribution( + schemaReplicas, proactive ? dataNodeCount : REPLICATION_FACTOR, proactive); + // Expect Data replicas to cover 3/6/6 nodes for AUTO (six replicas across two groups), and + // 3/6/9 for PROACTIVE. + // Expect 3/3/2 replicas per node for PROACTIVE; AUTO checks node coverage here without + // requiring balance. + assertReplicaDistribution( + dataReplicas, + proactive ? dataNodeCount : Math.min(dataNodeCount, 2 * REPLICATION_FACTOR), + proactive); + + // Check actual partition owners as well as created groups: no expected group may be unused. + // AUTO data distribution is 3+3; PROACTIVE is 2+2+2 (3D), or six groups with one slot (6D/9D). + TSchemaPartitionTableResp schemaResponse = + client.getSchemaPartitionTable(schemaRequest(devices)); + // Expect successful schema query (200): all six measurements exist on every 3/6/9DN + // configuration. + assertSuccess(schemaResponse.getStatus()); + Map schemaPartitions = + schemaResponse.getSchemaPartitionTable().get(database); + // Expect exactly 6 schema slots: hash-selected device names avoid collisions regardless of + // node count. + Assert.assertEquals(DEVICE_COUNT, schemaPartitions.size()); + // Expect Schema slots: AUTO has one group of 6; PROACTIVE has three groups of 2 on 3DN or + // six of 1 on 6/9DN. + assertSlotDistribution(schemaReplicas.keySet(), new ArrayList<>(schemaPartitions.values())); + Map>> firstPartitions = + readDataPartitions(client, database, devices, 0); + // Expect Data slots: AUTO has two groups of 3; PROACTIVE has three groups of 2 on 3DN or + // six of 1 on 6/9DN. + assertSlotDistribution(dataReplicas.keySet(), groupsAtTime(firstPartitions, 0)); + + // At t=10: 6 series slots * 2 time slots = 12 data partitions; schema partitions stay at 6. + // No extra groups are needed here; verify unchanged replica placement and per-slot routing. + writeAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + // Expect identical Schema groups and replica placements to t=0: writing t=10 adds no + // devices or schema slots. + Assert.assertEquals( + schemaReplicas, + // Expect Schema counts AUTO=1 or PROACTIVE=3/6/6 on 3/6/9DN: schema partitions are + // unchanged. + assertRegionGroupsAndReplicas( + client, database, TConsensusGroupType.SchemaRegion, expectedSchemaGroups)); + // Expect identical Data groups and replica placements to t=0: the same six series slots + // need no further growth. + Assert.assertEquals( + dataReplicas, + // Expect Data counts AUTO=2 or PROACTIVE=3/6/6 on 3/6/9DN: only the time slot is new. + assertRegionGroupsAndReplicas( + client, database, TConsensusGroupType.DataRegion, expectedDataGroups)); + Map>> secondPartitions = + readDataPartitions(client, database, devices, TIME_PARTITION_INTERVAL); + firstPartitions.forEach( + (slot, times) -> + // Expect each slot's owner at t=10 to match t=0: no new group triggers rebalancing, + // so INHERIT keeps its mapping. + Assert.assertEquals( + times.get(new TTimePartitionSlot(0)), + secondPartitions.get(slot).get(new TTimePartitionSlot(TIME_PARTITION_INTERVAL)))); + + // Finally verify SQL reads: exactly two rows per device, values slotId and slotId + 100. + for (int slot = 0; slot < DEVICE_COUNT; slot++) { + try (ResultSet rows = statement.executeQuery("SELECT s FROM " + devices.get(slot))) { + // Expect true: each device has a t=0 row on all 3/6/9DN configurations. + Assert.assertTrue(rows.next()); + // Expect first timestamp 0: default ascending time order returns the earliest write + // first. + Assert.assertEquals(0, rows.getLong(1)); + // Expect first value slot (0..5), matching this device's INSERT at t=0. + Assert.assertEquals(slot, rows.getInt(2)); + // Expect true: each device also has a t=10 write, so a second row must exist. + Assert.assertTrue(rows.next()); + // Expect second timestamp 10: TIME_PARTITION_INTERVAL=10 places the second write in the + // next partition. + Assert.assertEquals(TIME_PARTITION_INTERVAL, rows.getLong(1)); + // Expect second value slot+100 (100..105), matching the INSERT at t=10. + Assert.assertEquals(slot + 100, rows.getInt(2)); + // Expect false: only t=0 and t=10 were written, so no third or duplicate row may exist. + Assert.assertFalse(rows.next()); + } + } + } + + /** + * Select device names with the real BKDR hash; this only generates names, without creating + * measurements or partitions or mocking the executor. + * + *

For database=root.auto and slotCount=3, return three names whose indices i hash to slots i: + * 0, 1 and 2. Their d suffixes need not be 0, 1 and 2. + * + * @param database database containing the device paths + * @param slotCount number of distinct series slots to cover, from 0 to SERIES_SLOT_NUM; target + * slots are 0 through slotCount-1 + * @return names ordered by target slot, for subsequent SQL/RPC calls to allocate actual + * partitions + */ + private static List generateDeviceNamesForSlots(String database, int slotCount) { + List devices = new ArrayList<>(); + for (int slot = 0; slot < slotCount; slot++) { + devices.add(findDeviceInSlot(database, slot)); + } + return devices; + } + + /** + * Create six devices with one INT32 measurement s each and write initial data. For example, the + * devices hash to slots 0..5 and receive values 0..5 at t=0. + */ + private static List createSixDevices(Statement statement, String database) + throws Exception { + List devices = generateDeviceNamesForSlots(database, DEVICE_COUNT); + for (String device : devices) { + statement.execute("CREATE TIMESERIES " + device + ".s WITH DATATYPE=INT32, ENCODING=RLE"); + } + writeAtTime(statement, devices, 0, 0); + return devices; + } + + /** + * Read the unique logical owner of each series slot at a given time partition. For example, six + * devices at t=10 yield six slot-to-DataRegionGroup mappings; three replicas do not count as + * three groups. + */ + private static Map readDataAssignments( + SyncConfigNodeIServiceClient client, String database, List devices, long time) + throws Exception { + Map assignments = new HashMap<>(); + readDataPartitions(client, database, devices, time) + .forEach( + (slot, times) -> { + List groups = times.get(new TTimePartitionSlot(time)); + // Expect one logical Data group per (series slot,time slot), not three replica + // records. + Assert.assertEquals(1, groups.size()); + assignments.put(slot, groups.get(0)); + }); + return assignments; + } + + /** + * Write one value per device, using its list index plus valueOffset. For example, six devices + * with time=10 and valueOffset=100 receive values 100..105. + */ + private static void writeAtTime( + Statement statement, List devices, long time, int valueOffset) throws Exception { + for (int slot = 0; slot < devices.size(); slot++) { + statement.execute( + "INSERT INTO " + + devices.get(slot) + + "(time,s) VALUES (" + + time + + "," + + (slot + valueOffset) + + ")"); + } + } + + /** + * Verify exactly one row per device at the specified time, including timestamp and value. For + * example, six devices with time=20 and valueOffset=200 must return values 200..205; callers + * describe each scenario. + */ + private static void assertValuesAtTime( + Statement statement, List devices, long time, int valueOffset) throws Exception { + for (int slot = 0; slot < devices.size(); slot++) { + try (ResultSet rows = + statement.executeQuery("SELECT s FROM " + devices.get(slot) + " WHERE time=" + time)) { + // Expect true: filtering by the exact time must return the row written for each device, + // regardless of node count. + Assert.assertTrue(rows.next()); + // Expect the supplied timestamp: WHERE time selects the exact write batch requested by + // the caller. + Assert.assertEquals(time, rows.getLong(1)); + // Expect device index slot + valueOffset, matching the writeAtTime rule. + Assert.assertEquals(slot + valueOffset, rows.getInt(2)); + // Expect false: one value was written per device at this time, so no second row may + // exist. + Assert.assertFalse(rows.next()); + } + } + } + + /** + * Check a new time partition in the two-database PROACTIVE scenario, assuming six distinct slots + * and a per-database cap equal to dataNodeCount. For example, 3DN uses three groups of two slots + * and nine replicas across three nodes. + */ + private static void assertNewTimeDistribution( + SyncConfigNodeIServiceClient client, + String database, + List devices, + long time, + int dataNodeCount) + throws Exception { + Map> replicas = + // Expect min(DEVICE_COUNT, dataNodeCount) Data groups; callers list the concrete counts + // for each cluster size. + assertRegionGroupsAndReplicas( + client, + database, + TConsensusGroupType.DataRegion, + Math.min(DEVICE_COUNT, dataNodeCount)); + // Expect replicas on dataNodeCount nodes, with counts differing by at most one. + assertReplicaDistribution(replicas, dataNodeCount, true); + // Expect this time partition to use every expected group, with slot counts differing by at + // most one. + assertSlotDistribution( + replicas.keySet(), groupsAtTime(readDataPartitions(client, database, devices, time), time)); + } + + /** + * Wait for all expected Data replicas in the database to become Running. For example, + * expectedRegionGroupCount=6 with three replicas requires 18 Running records; query at most 30 + * times. + */ + private static void awaitRunningDataRegions( + SyncConfigNodeIServiceClient client, String database, int expectedRegionGroupCount) + throws Exception { + TShowRegionResp response = null; + for (int retry = 0; retry < 30; retry++) { + response = + client.showRegion( + new TShowRegionReq().setConsensusGroupType(TConsensusGroupType.DataRegion)); + // Expect successful SHOW REGIONS (200): statuses may change during recovery, but + // readiness requires a valid query. + assertSuccess(response.getStatus()); + long runningReplicas = + response.getRegionInfoList().stream() + .filter(region -> database.equals(region.getDatabase())) + .filter(region -> RegionStatus.Running.getStatus().equals(region.getStatus())) + .count(); + if (runningReplicas == (long) expectedRegionGroupCount * REPLICATION_FACTOR) { + return; + } + TimeUnit.SECONDS.sleep(1); + } + // Expect expectedRegionGroupCount * REPLICATION_FACTOR Running replicas within 30 attempts. + Assert.fail( + "DataRegion replicas did not become Running after ConfigNode recovery: " + response); + } + + /** + * Enumerate names and use the real BKDR hash to find the target slot. For example, slot=2 returns + * a root.auto.dN that hashes to 2, without assuming d2 hashes to slot 2. + */ + private static String findDeviceInSlot(String database, int slot) { + // Control hashing instead of assuming d0...d5 occupy different slots; measurements share a + // slot. + for (int candidate = 0; candidate < SERIES_SLOT_NUM * 100; candidate++) { + String device = database + ".d" + candidate; + if (PARTITION_EXECUTOR.getSeriesPartitionSlot(device).getSlotId() == slot) { + return device; + } + } + throw new AssertionError("No device found in series slot " + slot); + } + + /** + * Build a schema partition request for measurement s of each device. For example, root.auto.d8 + * becomes root.auto.d8.s; this method only constructs the request. + */ + private static TSchemaPartitionReq schemaRequest(List devices) throws Exception { + return new TSchemaPartitionReq( + generatePatternTreeBuffer( + devices.stream().map(device -> device + ".s").toArray(String[]::new))); + } + + /** + * Build a request for the database, series slots and time partition start. For example, two + * devices in distinct slots and time=10 produce two entries requesting t=10; time must be a + * partition start. + */ + private static TDataPartitionReq dataRequest(String database, List devices, long time) { + Map slots = new HashMap<>(); + for (String device : devices) { + slots.put( + PARTITION_EXECUTOR.getSeriesPartitionSlot(device), + new TTimeSlotList(Collections.singletonList(new TTimePartitionSlot(time)), false, false)); + } + return new TDataPartitionReq(Collections.singletonMap(database, slots)); + } + + /** + * Query allocated data partitions, requiring devices in distinct series slots. For example, six + * distinct slots at time=10 must return six slot entries; callers verify their owners. + */ + private static Map>> + readDataPartitions( + SyncConfigNodeIServiceClient client, String database, List devices, long time) + throws Exception { + TDataPartitionTableResp response = + client.getDataPartitionTable(dataRequest(database, devices, time)); + // Expect successful partition query (200): the requested partitions must already have been + // allocated. + assertSuccess(response.getStatus()); + Map>> partitions = + response.getDataPartitionTable().get(database); + // Expect devices.size() distinct slots: callers must provide devices in different slots + // with existing partitions. + Assert.assertEquals(devices.size(), partitions.size()); + return partitions; + } + + /** + * Extract group IDs at the requested time, retaining duplicates for slot counting. For example, + * six slots owned by A and B may produce three A entries and three B entries. + */ + private static List groupsAtTime( + Map>> partitions, + long time) { + List groups = new ArrayList<>(); + partitions + .values() + .forEach( + times -> { + List regionIds = times.get(new TTimePartitionSlot(time)); + // Expect one group ID per slot/time: this checks logical ownership; the three + // replicas are checked separately. + Assert.assertEquals(1, regionIds.size()); + groups.add(regionIds.get(0)); + }); + return groups; + } + + /** + * Verify that every expected group is used and slot counts differ by at most one. For example, + * expected groups A/B with assignments A,A,A,B,B,B pass; an unused B fails. + */ + private static void assertSlotDistribution( + Set groups, List assigned) { + // One assignment per series slot at a fixed time: all groups must be used, counts differ by + // <=1. + Map counts = new HashMap<>(); + assigned.forEach(group -> counts.merge(group, 1, Integer::sum)); + // Expect exactly the caller's group set to be used: every expected group must receive a + // slot. + Assert.assertEquals(groups, counts.keySet()); + // Expect slot counts to differ by at most one; callers describe the concrete slot and group + // counts. + Assert.assertTrue( + counts.toString(), + Collections.max(counts.values()) - Collections.min(counts.values()) <= 1); + } + + /** + * Check the number of distinct DataNodes hosting replicas and optionally require counts to differ + * by at most one. For example, six groups with three replicas balanced across nine nodes yield + * two replicas per node. + * + * @param replicas group IDs mapped to the DataNode IDs hosting their replicas + * @param expectedHostingDataNodeCount expected number of distinct DataNodes actually hosting + * these replicas + * @param requireBalancedReplicaCounts whether to require balanced replica counts among hosting + * nodes + */ + private static void assertReplicaDistribution( + Map> replicas, + int expectedHostingDataNodeCount, + boolean requireBalancedReplicaCounts) { + Map counts = new HashMap<>(); + replicas.values().forEach(nodes -> nodes.forEach(node -> counts.merge(node, 1, Integer::sum))); + // Expect expectedHostingDataNodeCount distinct DataNodes to host these replicas. + Assert.assertEquals(counts.toString(), expectedHostingDataNodeCount, counts.size()); + if (requireBalancedReplicaCounts) { + // Expect replica counts among hosting nodes to differ by at most one, only when balance + // is requested. + Assert.assertTrue( + counts.toString(), + Collections.max(counts.values()) - Collections.min(counts.values()) <= 1); + } + } + + /** + * Check logical group count, total replica rows and distinct DataNodes per group for one database + * and Region type. + * + *

For root.auto, DataRegion and expectedRegionGroupCount=2, expect two logical groups, 2*3=6 + * replica rows and three distinct DataNodes per group. Slot allocation and balance across groups + * are checked separately. + * + * @param database database whose Regions are counted + * @param regionType SchemaRegion or DataRegion to check + * @param expectedRegionGroupCount expected number of created logical groups for this database and + * type, not nodes or slots + * @return group IDs mapped to the DataNode IDs hosting their replicas + */ + private static Map> assertRegionGroupsAndReplicas( + SyncConfigNodeIServiceClient client, + String database, + TConsensusGroupType regionType, + int expectedRegionGroupCount) + throws Exception { + // SHOW REGIONS returns replicas, not logical groups: deduplicate IDs before checking G and 3*G. + TShowRegionResp response = + client.showRegion(new TShowRegionReq().setConsensusGroupType(regionType)); + // Expect successful SHOW REGIONS (200) before counting replicas by database, type and group + // ID. + assertSuccess(response.getStatus()); + Map> replicas = new HashMap<>(); + int replicaCount = 0; + for (TRegionInfo region : response.getRegionInfoList()) { + if (database.equals(region.getDatabase())) { + replicas + .computeIfAbsent(region.getConsensusGroupId(), ignored -> new HashSet<>()) + .add(region.getDataNodeId()); + replicaCount++; + } + } + // Expect expectedRegionGroupCount distinct groups for the specified database and Region + // type. + Assert.assertEquals(database + " " + regionType, expectedRegionGroupCount, replicas.size()); + // Expect expectedRegionGroupCount * REPLICATION_FACTOR replica rows. + Assert.assertEquals(expectedRegionGroupCount * REPLICATION_FACTOR, replicaCount); + // Expect each group's replicas on REPLICATION_FACTOR distinct DataNodes, preventing + // colocated replicas of one group. + replicas.values().forEach(nodes -> Assert.assertEquals(REPLICATION_FACTOR, nodes.size())); + return replicas; + } + + /** + * Check that both database group caps equal the same expected value. For example, 6 means both + * Schema and Data caps are 6, without requiring six groups to have been created. + */ + private static void assertMaximum( + SyncConfigNodeIServiceClient client, String database, int expectedRegionGroupLimit) + throws Exception { + // Expect both Schema and Data caps to equal the caller's expectedRegionGroupLimit. + assertMaximum(client, database, expectedRegionGroupLimit, expectedRegionGroupLimit); + } + + /** + * Check database group caps independently. For example, expectedSchemaRegionGroupLimit=6 and + * expectedDataRegionGroupLimit=2 require Schema cap 6 and Data cap 2; actual group counts are + * checked separately. + */ + private static void assertMaximum( + SyncConfigNodeIServiceClient client, + String database, + int expectedSchemaRegionGroupLimit, + int expectedDataRegionGroupLimit) + throws Exception { + TDatabaseSchemaResp response = + client.getMatchedDatabaseSchemas( + new TGetDatabaseReq(Arrays.asList(database.split("\\.")), ALL_MATCH_SCOPE_BINARY)); + // Expect successful database metadata query (200) before checking both group caps. + assertSuccess(response.getStatus()); + TDatabaseSchema schema = response.getDatabaseSchemaMap().get(database); + // Expect Schema cap expectedSchemaRegionGroupLimit; a cap is not the number of groups + // already created. + Assert.assertEquals(expectedSchemaRegionGroupLimit, schema.getMaxSchemaRegionGroupNum()); + // Expect Data cap expectedDataRegionGroupLimit, checked independently of the Schema cap. + Assert.assertEquals(expectedDataRegionGroupLimit, schema.getMaxDataRegionGroupNum()); + } + + /** + * Check RPC success; for example, normal database creation returns SUCCESS_STATUS (200). Callers + * assert expected failures separately. + */ + private static void assertSuccess(TSStatus status) { + // Expect SUCCESS_STATUS (200) for normal database creation, allocation and queries; failure + // cases use assertThrows. + Assert.assertEquals( + status.getMessage(), TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupExtensionIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupExtensionIT.java new file mode 100644 index 0000000000000..1f2238d29f32b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupExtensionIT.java @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.partition; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.cluster.RegionStatus; +import org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor; +import org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchemaResp; +import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; +import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionReq; +import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionTableResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_SCOPE_BINARY; +import static org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils.generatePatternTreeBuffer; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBProactiveRegionGroupExtensionIT { + + private static final String DATABASE = "root.proactive"; + private static final int SERIES_SLOT_NUM = 32; + private static final int MAX_REGION_GROUP_NUM = 4; + private static final long TIME_PARTITION_INTERVAL = 10; + private static final BKDRHashExecutor PARTITION_EXECUTOR = new BKDRHashExecutor(SERIES_SLOT_NUM); + + @After + public void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testMinimumGrowthCountsOnlyDistinctPendingSlots() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS) + .setSchemaReplicationFactor(1) + .setDataReplicationFactor(1) + .setSchemaRegionGroupExtensionPolicy("PROACTIVE") + .setDataRegionGroupExtensionPolicy("PROACTIVE") + .setSchemaRegionPerDataNode(4) + .setDataRegionPerDataNode(4) + .setTimePartitionInterval(TIME_PARTITION_INTERVAL); + EnvFactory.getEnv().initClusterEnvironment(1, 1); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + assertSuccess( + client.setDatabase( + new TDatabaseSchema(DATABASE) + .setMinSchemaRegionGroupNum(4) + .setMinDataRegionGroupNum(4))); + Map> schemaRequest = new HashMap<>(); + schemaRequest.put( + DATABASE, Arrays.asList(new TSeriesPartitionSlot(0), new TSeriesPartitionSlot(0))); + assertSuccess(client.getOrCreateSchemaPartitionTableWithSlots(schemaRequest).getStatus()); + assertRegionGroups(client, TConsensusGroupType.SchemaRegion, 1, 1); + + createDataPartitions(client, 0, 2, 0, 1); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 2, 1); + // Slots 0 and 1 already have this time partition; only slot 2 contributes to minimum growth. + createDataPartitions(client, 0, 3, 0, 1); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 3, 1); + createDataPartitions(client, 0, 3, 0, 1); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 3, 1); + // A genuinely new time partition for an existing series slot still grows toward the minimum. + createDataPartitions(client, 2, 3, 1, 2); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 4, 1); + } + } + + @Test + public void testSingleConfigNodeAndDataNode() throws Exception { + checkProactiveExtension( + 1, 1, ConsensusFactory.SIMPLE_CONSENSUS, ConsensusFactory.SIMPLE_CONSENSUS); + } + + @Test + public void testTwoConfigNodesAndDataNodes() throws Exception { + checkProactiveExtension( + 2, 2, ConsensusFactory.RATIS_CONSENSUS, ConsensusFactory.RATIS_CONSENSUS); + } + + @Test + public void testThreeConfigNodesAndDataNodes() throws Exception { + checkProactiveExtension( + 3, 3, ConsensusFactory.RATIS_CONSENSUS, ConsensusFactory.RATIS_CONSENSUS); + } + + @Test + public void testSingleConfigNodeAndThreeDataNodes() throws Exception { + checkProactiveExtension(1, 3, ConsensusFactory.RATIS_CONSENSUS, ConsensusFactory.IOT_CONSENSUS); + } + + @Test + public void testThreeConfigNodesAndSingleDataNode() throws Exception { + checkProactiveExtension( + 3, 1, ConsensusFactory.SIMPLE_CONSENSUS, ConsensusFactory.SIMPLE_CONSENSUS); + } + + private void checkProactiveExtension( + int configNodeCount, int dataNodeCount, String schemaConsensus, String dataConsensus) + throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass( + configNodeCount == 1 && dataNodeCount == 1 + ? ConsensusFactory.SIMPLE_CONSENSUS + : ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(schemaConsensus) + .setDataRegionConsensusProtocolClass(dataConsensus) + .setSchemaReplicationFactor(dataNodeCount) + .setDataReplicationFactor(dataNodeCount) + .setSchemaRegionGroupExtensionPolicy("PROACTIVE") + .setDataRegionGroupExtensionPolicy("PROACTIVE") + .setSchemaRegionPerDataNode(MAX_REGION_GROUP_NUM) + .setDataRegionPerDataNode(MAX_REGION_GROUP_NUM) + .setSeriesSlotNum(SERIES_SLOT_NUM) + .setSeriesPartitionExecutorClass(BKDRHashExecutor.class.getName()) + .setTimePartitionInterval(TIME_PARTITION_INTERVAL); + EnvFactory.getEnv().initClusterEnvironment(configNodeCount, dataNodeCount); + + String[] paths = new String[MAX_REGION_GROUP_NUM + 2]; + for (int slot = 0; slot < paths.length; slot++) { + paths[slot] = findPathInSlot(slot, null); + } + + Set schemaGroups; + Set dataGroups; + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + assertSuccess( + client.setDatabase( + new TDatabaseSchema(DATABASE) + .setMinSchemaRegionGroupNum(2) + .setMinDataRegionGroupNum(2))); + assertMaxRegionGroupNum(client); + assertRegionGroups(client, TConsensusGroupType.SchemaRegion, 0, dataNodeCount); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 0, dataNodeCount); + + // A minimum of two is reached gradually, adding one group per pending schema slot. + for (int slot = 0; slot < 3; slot++) { + createSchemaPartitions(client, paths[slot]); + assertRegionGroups(client, TConsensusGroupType.SchemaRegion, slot + 1, dataNodeCount); + } + // Measurements, repeated requests and another device hashing to the same slot do not grow. + createSchemaPartitions(client, paths[0], paths[0] + "2", findPathInSlot(0, paths[0])); + schemaGroups = assertRegionGroups(client, TConsensusGroupType.SchemaRegion, 3, dataNodeCount); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 0, dataNodeCount); + + // Activating schema slots must not count as activating data slots. + createDataPartitions(client, 0, 1, 0, 1); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 1, dataNodeCount); + createDataPartitions(client, 0, 1, 0, 1); + // Already assigned partitions do not trigger growth toward the minimum. + assertRegionGroups(client, TConsensusGroupType.DataRegion, 1, dataNodeCount); + createDataPartitions(client, 0, 1, 1, 4); + // New times for the existing series slot complete the minimum, just as with AUTO. + assertRegionGroups(client, TConsensusGroupType.DataRegion, 2, dataNodeCount); + createDataPartitions(client, 0, 2, 4, 5); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 2, dataNodeCount); + // New time partitions for two existing slots are still only two activated series slots. + createDataPartitions(client, 0, 2, 5, 8); + dataGroups = assertRegionGroups(client, TConsensusGroupType.DataRegion, 2, dataNodeCount); + } + + int oldLeader = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + EnvFactory.getEnv().shutdownConfigNode(oldLeader); + if (configNodeCount < 3) { + // A two-member ConfigNode group needs both members for quorum. + EnvFactory.getEnv().startConfigNode(oldLeader); + } else { + Assert.assertNotEquals(oldLeader, EnvFactory.getEnv().getLeaderConfigNodeIndex()); + } + + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + // Restart / leader election preserves the groups, counts and resource-derived cap. + assertMaxRegionGroupNum(client); + Assert.assertEquals( + schemaGroups, + assertRegionGroups(client, TConsensusGroupType.SchemaRegion, 3, dataNodeCount)); + Assert.assertEquals( + dataGroups, assertRegionGroups(client, TConsensusGroupType.DataRegion, 2, dataNodeCount)); + // Leader discovery precedes the heartbeat statistics that make recovered groups available. + awaitRunningRegions(client, (schemaGroups.size() + dataGroups.size()) * dataNodeCount); + createSchemaPartitions(client, paths[0], paths[1], paths[2]); + createDataPartitions(client, 0, 2, 8, 9); + assertRegionGroups(client, TConsensusGroupType.SchemaRegion, 3, dataNodeCount); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 2, dataNodeCount); + + // Batched requests mix existing and new slots, then reach and stay at AUTO's same cap. + createDataPartitions(client, 0, 3, 9, 10); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 3, dataNodeCount); + createSchemaPartitions(client, paths); + createDataPartitions(client, 0, paths.length, 10, 11); + assertRegionGroups( + client, TConsensusGroupType.SchemaRegion, MAX_REGION_GROUP_NUM, dataNodeCount); + assertRegionGroups( + client, TConsensusGroupType.DataRegion, MAX_REGION_GROUP_NUM, dataNodeCount); + createSchemaPartitions(client, findPathInSlot(paths.length, null)); + createDataPartitions(client, 0, paths.length + 1, 11, 12); + assertRegionGroups( + client, TConsensusGroupType.SchemaRegion, MAX_REGION_GROUP_NUM, dataNodeCount); + assertRegionGroups( + client, TConsensusGroupType.DataRegion, MAX_REGION_GROUP_NUM, dataNodeCount); + } + } + + private static void awaitRunningRegions( + SyncConfigNodeIServiceClient client, int expectedReplicaCount) throws Exception { + TShowRegionResp response = null; + for (int retry = 0; retry < 30; retry++) { + response = client.showRegion(new TShowRegionReq()); + assertSuccess(response.getStatus()); + int replicaCount = 0; + int runningReplicaCount = 0; + for (TRegionInfo region : response.getRegionInfoList()) { + if (DATABASE.equals(region.getDatabase())) { + replicaCount++; + if (RegionStatus.Running.getStatus().equals(region.getStatus())) { + runningReplicaCount++; + } + } + } + if (replicaCount == expectedReplicaCount && runningReplicaCount == expectedReplicaCount) { + return; + } + TimeUnit.SECONDS.sleep(1); + } + Assert.fail("Regions did not become Running after ConfigNode recovery: " + response); + } + + private static String findPathInSlot(int slot, String excludedPath) { + for (int device = 0; device < SERIES_SLOT_NUM * 100; device++) { + String devicePath = DATABASE + ".d" + device; + String path = devicePath + ".s"; + if (!path.equals(excludedPath) + && PARTITION_EXECUTOR.getSeriesPartitionSlot(devicePath).getSlotId() == slot) { + return path; + } + } + throw new AssertionError("No device found in series slot " + slot); + } + + private static void createSchemaPartitions(SyncConfigNodeIServiceClient client, String... paths) + throws Exception { + TSchemaPartitionTableResp response = + client.getOrCreateSchemaPartitionTable( + new TSchemaPartitionReq(generatePatternTreeBuffer(paths))); + assertSuccess(response.getStatus()); + Set expectedSlots = new HashSet<>(); + for (String path : paths) { + expectedSlots.add( + PARTITION_EXECUTOR.getSeriesPartitionSlot(path.substring(0, path.lastIndexOf('.')))); + } + Assert.assertEquals(expectedSlots, response.getSchemaPartitionTable().get(DATABASE).keySet()); + } + + private static void createDataPartitions( + SyncConfigNodeIServiceClient client, int slotStart, int slotEnd, int timeStart, int timeEnd) + throws Exception { + TDataPartitionTableResp response = + client.getOrCreateDataPartitionTable( + new TDataPartitionReq( + ConfigNodeTestUtils.constructPartitionSlotsMap( + DATABASE, slotStart, slotEnd, timeStart, timeEnd, TIME_PARTITION_INTERVAL))); + assertSuccess(response.getStatus()); + ConfigNodeTestUtils.checkDataPartitionTable( + DATABASE, + slotStart, + slotEnd, + timeStart, + timeEnd, + TIME_PARTITION_INTERVAL, + response.getDataPartitionTable()); + } + + private static void assertMaxRegionGroupNum(SyncConfigNodeIServiceClient client) + throws Exception { + TDatabaseSchemaResp response = + client.getMatchedDatabaseSchemas( + new TGetDatabaseReq(Arrays.asList("root", "proactive"), ALL_MATCH_SCOPE_BINARY)); + assertSuccess(response.getStatus()); + TDatabaseSchema schema = response.getDatabaseSchemaMap().get(DATABASE); + // One database: per-node quota * DataNode count / replication factor = 4, as with AUTO. + Assert.assertEquals(MAX_REGION_GROUP_NUM, schema.getMaxSchemaRegionGroupNum()); + Assert.assertEquals(MAX_REGION_GROUP_NUM, schema.getMaxDataRegionGroupNum()); + } + + private static Set assertRegionGroups( + SyncConfigNodeIServiceClient client, + TConsensusGroupType type, + int expectedGroupCount, + int replicationFactor) + throws Exception { + TShowRegionResp response = client.showRegion(new TShowRegionReq().setConsensusGroupType(type)); + assertSuccess(response.getStatus()); + Map> replicas = new HashMap<>(); + int replicaCount = 0; + for (TRegionInfo region : response.getRegionInfoList()) { + if (DATABASE.equals(region.getDatabase())) { + replicas + .computeIfAbsent(region.getConsensusGroupId(), ignored -> new HashSet<>()) + .add(region.getDataNodeId()); + replicaCount++; + } + } + Assert.assertEquals(type.toString(), expectedGroupCount, replicas.size()); + Assert.assertEquals(expectedGroupCount * replicationFactor, replicaCount); + replicas.values().forEach(nodes -> Assert.assertEquals(replicationFactor, nodes.size())); + return replicas.keySet(); + } + + private static void assertSuccess(TSStatus status) { + Assert.assertEquals( + status.getMessage(), TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 16a183ecf1ed6..c5facfb371ece 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -95,34 +95,82 @@ public class ConfigNodeConfig { private String dataPartitionAllocationStrategy = "INHERIT"; - /** The policy of extension SchemaRegionGroup for each Database. */ + /** + * The policy of extending SchemaRegionGroups for each Database: CUSTOM, AUTO, or PROACTIVE. + * + *

CUSTOM suits known workloads that need manual resource allocation, creating the configured + * target number of groups per Database when schema partitions are first allocated. AUTO suits + * most routine workloads (roughly 80% as an approximate planning guideline), gradually meeting + * the configured minimum and expanding with slot occupancy up to the resource-based maximum to + * balance parallelism and group management overhead. PROACTIVE complements AUTO for workloads + * such as very few devices with many measurements and a high load, creating groups earlier for + * more parallelism at the cost of additional group management overhead. + * + *

AUTO and PROACTIVE use the same per-Database maximum calculation, including resource sharing + * across Databases; their group counts match when both reach the same maximum. PROACTIVE targets + * one group per distinct schema series slot, not per measurement, and grows toward the configured + * minimum incrementally like AUTO. When all groups of this type are disabled, it may add one + * group within the maximum. Switching policies does not remove existing groups. + */ private volatile RegionGroupExtensionPolicy schemaRegionGroupExtensionPolicy = - RegionGroupExtensionPolicy.AUTO; + RegionGroupExtensionPolicy.PROACTIVE; /** * When set schema_region_group_extension_policy=CUSTOM, this parameter is the default number of - * SchemaRegionGroups for each Database. When set schema_region_group_extension_policy=AUTO, this - * parameter is the default minimal number of SchemaRegionGroups for each Database. + * SchemaRegionGroups for each Database. For AUTO and PROACTIVE, this parameter is the default + * minimum number of SchemaRegionGroups and a lower bound for the per-Database maximum. Both + * policies grow toward this minimum incrementally, adding at most the number of series slots in + * the pending request to satisfy the minimum. PROACTIVE also grows with the number of active + * series slots. */ private volatile int defaultSchemaRegionGroupNumPerDatabase = 1; - /** The maximum number of SchemaRegions expected to be managed by each DataNode. */ + /** + * The expected number of SchemaRegions per DataNode, used to calculate the same per-Database + * maximum for AUTO and PROACTIVE. This is not a hard limit on node or cluster totals: + * per-Database minimums, rounding, and existing groups can raise the total above this resource + * estimate. + */ private volatile int schemaRegionPerDataNode = 1; - /** The policy of extension DataRegionGroup for each Database. */ + /** + * The policy of extending DataRegionGroups for each Database: CUSTOM, AUTO, or PROACTIVE. + * + *

CUSTOM suits known workloads that need manual resource allocation, creating the configured + * target number of groups per Database when data partitions are first allocated. AUTO suits most + * routine workloads (roughly 80% as an approximate planning guideline), gradually meeting the + * configured minimum and expanding with slot occupancy up to the resource-based maximum to + * balance parallelism and group management overhead. PROACTIVE complements AUTO for workloads + * such as very few devices with many measurements and a high load, creating groups earlier for + * more parallelism at the cost of additional group management overhead. + * + *

AUTO and PROACTIVE use the same per-Database maximum calculation, including resource sharing + * across Databases; their group counts match when both reach the same maximum. PROACTIVE targets + * one group per distinct data series slot, not per measurement, and grows toward the configured + * minimum incrementally like AUTO. New time partitions in an existing slot can also trigger this + * growth toward the minimum. When all groups of this type are disabled, it may add one group + * within the maximum. Its allocation policy table balances active slots so new time partitions + * can use new groups; existing time-partition assignments are retained. Switching policies does + * not remove existing groups. + */ private volatile RegionGroupExtensionPolicy dataRegionGroupExtensionPolicy = - RegionGroupExtensionPolicy.AUTO; + RegionGroupExtensionPolicy.PROACTIVE; /** * When set data_region_group_extension_policy=CUSTOM, this parameter is the default number of - * DataRegionGroups for each Database. When set data_region_group_extension_policy=AUTO, this - * parameter is the default minimal number of DataRegionGroups for each Database. + * DataRegionGroups for each Database. For AUTO and PROACTIVE, this parameter is the default + * minimum number of DataRegionGroups and a lower bound for the per-Database maximum. Both + * policies grow toward this minimum incrementally, adding at most the number of series slots in + * the pending request to satisfy the minimum. PROACTIVE also grows with the number of active + * series slots. */ private volatile int defaultDataRegionGroupNumPerDatabase = 2; /** - * The maximum number of DataRegions expected to be managed by each DataNode. Set to 0 means that - * each dataNode automatically has the number of CPU cores / 2 regions. + * The expected number of DataRegions per DataNode, used to calculate the same per-Database + * maximum for AUTO and PROACTIVE. Set to 0 to use half the CPU core count as the expected number + * for each DataNode. This is not a hard limit on node or cluster totals: per-Database minimums, + * rounding, and existing groups can raise the total above this resource estimate. */ private volatile int dataRegionPerDataNode = 0; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 5041b77136b9a..5fe329af8d48a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -125,6 +125,7 @@ import org.apache.iotdb.confignode.manager.node.NodeMetrics; import org.apache.iotdb.confignode.manager.partition.PartitionManager; import org.apache.iotdb.confignode.manager.partition.PartitionMetrics; +import org.apache.iotdb.confignode.manager.partition.RegionGroupExtensionPolicy; import org.apache.iotdb.confignode.manager.pipe.agent.PipeConfigNodeAgent; import org.apache.iotdb.confignode.manager.pipe.coordinator.PipeManager; import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; @@ -1836,6 +1837,10 @@ public TSStatus setConfiguration(TSetConfigurationReq req) { long previousHeartbeatIntervalInMs = CONF.getHeartbeatIntervalInMs(); int previousSchemaRegionPerDataNode = CONF.getSchemaRegionPerDataNode(); int previousDataRegionPerDataNode = CONF.getDataRegionPerDataNode(); + RegionGroupExtensionPolicy previousSchemaRegionGroupExtensionPolicy = + CONF.getSchemaRegionGroupExtensionPolicy(); + RegionGroupExtensionPolicy previousDataRegionGroupExtensionPolicy = + CONF.getDataRegionGroupExtensionPolicy(); boolean wasTopologyProbingEnabled = CONF.isEnableTopologyProbing(); int previousProcedureCompletedCleanInterval = CONF.getProcedureCompletedCleanInterval(); int previousProcedureCompletedEvictTTL = CONF.getProcedureCompletedEvictTTL(); @@ -1866,8 +1871,11 @@ public TSStatus setConfiguration(TSetConfigurationReq req) { return tsStatus; } handleHeartbeatIntervalHotReload(previousHeartbeatIntervalInMs); - handleRegionPerDataNodeHotReload( - previousSchemaRegionPerDataNode, previousDataRegionPerDataNode); + handleRegionGroupConfigHotReload( + previousSchemaRegionPerDataNode, + previousDataRegionPerDataNode, + previousSchemaRegionGroupExtensionPolicy, + previousDataRegionGroupExtensionPolicy); handleTopologyProbingHotReload(wasTopologyProbingEnabled); handleProcedureCleanerHotReload( previousProcedureCompletedCleanInterval, previousProcedureCompletedEvictTTL); @@ -1914,16 +1922,31 @@ private void handleHeartbeatIntervalHotReload(long previousHeartbeatIntervalInMs getRetryFailedTasksThread().reloadHeartbeatInterval(); } - private void handleRegionPerDataNodeHotReload( - int previousSchemaRegionPerDataNode, int previousDataRegionPerDataNode) { + private void handleRegionGroupConfigHotReload( + int previousSchemaRegionPerDataNode, + int previousDataRegionPerDataNode, + RegionGroupExtensionPolicy previousSchemaRegionGroupExtensionPolicy, + RegionGroupExtensionPolicy previousDataRegionGroupExtensionPolicy) { if (previousSchemaRegionPerDataNode == CONF.getSchemaRegionPerDataNode() - && previousDataRegionPerDataNode == CONF.getDataRegionPerDataNode()) { + && previousDataRegionPerDataNode == CONF.getDataRegionPerDataNode() + && previousSchemaRegionGroupExtensionPolicy == CONF.getSchemaRegionGroupExtensionPolicy() + && previousDataRegionGroupExtensionPolicy == CONF.getDataRegionGroupExtensionPolicy()) { return; } if (!getConsensusManager().isLeader()) { return; } + // Leaving CUSTOM must replace its stored cap with the resource-derived cap, even when + // per-node quotas are unchanged. This preserves existing groups and skips types still CUSTOM. getClusterSchemaManager().adjustMaxRegionGroupNum(); + if (previousDataRegionGroupExtensionPolicy != RegionGroupExtensionPolicy.PROACTIVE + && CONF.getDataRegionGroupExtensionPolicy() == RegionGroupExtensionPolicy.PROACTIVE) { + // Existing groups may already meet the new target, so no creation procedure would rebalance + // their slot assignments. Refresh the policy table for future time partitions on this leader. + getClusterSchemaManager() + .getDatabaseNames(null) + .forEach(database -> getLoadManager().reBalanceDataPartitionPolicy(database)); + } } private void handleTopologyProbingHotReload(boolean wasEnabled) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTable.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTable.java index 2cce461478c49..bc80ef62014c0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTable.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTable.java @@ -25,6 +25,7 @@ import org.apache.iotdb.confignode.conf.ConfigNodeConfig; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.i18n.ManagerMessages; +import org.apache.iotdb.confignode.manager.partition.RegionGroupExtensionPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +35,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.locks.ReentrantLock; public class DataPartitionPolicyTable { @@ -111,7 +113,7 @@ public void reBalanceDataPartitionPolicy(List dataRegionGroup } Collections.shuffle(seriesPartitionSlots); - int mu = SERIES_SLOT_NUM / dataRegionGroups.size(); + int mu = getRetainedSlotLimit(dataAllotMap, dataRegionGroups.size()); for (TSeriesPartitionSlot seriesPartitionSlot : seriesPartitionSlots) { if (!dataAllotMap.containsKey(seriesPartitionSlot)) { // Skip unallocated SeriesPartitionSlot @@ -141,7 +143,7 @@ public void setDataAllotMap(Map dataAll } dataAllotTableLock.lock(); try { - int mu = SERIES_SLOT_NUM / seriesPartitionSlotCounter.size(); + int mu = getRetainedSlotLimit(dataAllotMap, seriesPartitionSlotCounter.size()); dataAllotMap.forEach( (seriesPartitionSlot, regionGroupId) -> { if (regionGroupId != null && seriesPartitionSlotCounter.get(regionGroupId) < mu) { @@ -158,6 +160,18 @@ public void setDataAllotMap(Map dataAll } } + private int getRetainedSlotLimit( + Map assignments, int regionGroupCount) { + if (CONF.getDataRegionGroupExtensionPolicy() == RegionGroupExtensionPolicy.PROACTIVE) { + // Empty DataPartition entries have no last group and must not inflate the recovery limit. + long activeSlotCount = assignments.values().stream().filter(Objects::nonNull).count(); + // New groups are created before the pending slots are activated. Keep at least one slot + // per existing group so that incremental growth does not discard balanced assignments. + return Math.max(1, (int) (activeSlotCount / regionGroupCount)); + } + return SERIES_SLOT_NUM / regionGroupCount; + } + public void logDataAllotTable(String database) { seriesPartitionSlotCounter .keySet() diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java index 2ee0bcdb60bbc..249a4f9bd0e05 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java @@ -109,6 +109,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -283,15 +284,9 @@ public SchemaPartitionResp getOrCreateSchemaPartition(final GetOrCreateSchemaPar // Here we ensure that each StorageGroup has at least one SchemaRegion. // And if some StorageGroups own too many slots, extend SchemaRegion for them. - // Map - final Map unassignedSchemaPartitionSlotsCountMap = new ConcurrentHashMap<>(); - unassignedSchemaPartitionSlotsMap.forEach( - (storageGroup, unassignedSchemaPartitionSlots) -> - unassignedSchemaPartitionSlotsCountMap.put( - storageGroup, unassignedSchemaPartitionSlots.size())); TSStatus status = extendRegionGroupIfNecessary( - unassignedSchemaPartitionSlotsCountMap, TConsensusGroupType.SchemaRegion); + unassignedSchemaPartitionSlotsMap, TConsensusGroupType.SchemaRegion); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // Return an error code if Region extension failed resp.setStatus(status); @@ -438,15 +433,20 @@ public DataPartitionResp getOrCreateDataPartition(final GetOrCreateDataPartition // Here we ensure that each StorageGroup has at least one DataRegion. // And if some StorageGroups own too many slots, extend DataRegion for them. - // Map - Map unassignedDataPartitionSlotsCountMap = new ConcurrentHashMap<>(); + // Keep the series slots so PROACTIVE can distinguish new slots from new time partitions. + Map> unassignedDataSeriesPartitionSlotsMap = + new HashMap<>(); unassignedDataPartitionSlotsMap.forEach( (storageGroup, unassignedDataPartitionSlots) -> - unassignedDataPartitionSlotsCountMap.put( - storageGroup, unassignedDataPartitionSlots.size())); + unassignedDataSeriesPartitionSlotsMap.put( + storageGroup, + unassignedDataPartitionSlots.entrySet().stream() + .filter(entry -> !entry.getValue().getTimePartitionSlots().isEmpty()) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()))); TSStatus status = extendRegionGroupIfNecessary( - unassignedDataPartitionSlotsCountMap, TConsensusGroupType.DataRegion); + unassignedDataSeriesPartitionSlotsMap, TConsensusGroupType.DataRegion); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // Return an error code if Region extension failed resp.setStatus(status); @@ -599,13 +599,13 @@ private TSStatus consensusWritePartitionResult(ConfigPhysicalPlan plan) { /** * Allocate more RegionGroup to the specified StorageGroups if necessary. * - * @param unassignedPartitionSlotsCountMap Map + * @param unassignedPartitionSlotsMap Map * @param consensusGroupType SchemaRegion or DataRegion * @return SUCCESS_STATUS when RegionGroup extension successful; NOT_ENOUGH_DATA_NODE when there * are not enough DataNodes; STORAGE_GROUP_NOT_EXIST when some StorageGroups don't exist */ - private TSStatus extendRegionGroupIfNecessary( - final Map unassignedPartitionSlotsCountMap, + TSStatus extendRegionGroupIfNecessary( + final Map> unassignedPartitionSlotsMap, final TConsensusGroupType consensusGroupType) { final TSStatus result = new TSStatus(); @@ -613,23 +613,29 @@ private TSStatus extendRegionGroupIfNecessary( try { if (TConsensusGroupType.SchemaRegion.equals(consensusGroupType)) { switch (CONF.getSchemaRegionGroupExtensionPolicy()) { + case PROACTIVE: + return proactiveExtendRegionGroupIfNecessary( + unassignedPartitionSlotsMap, consensusGroupType); case CUSTOM: return customExtendRegionGroupIfNecessary( - unassignedPartitionSlotsCountMap, consensusGroupType); + unassignedPartitionSlotsMap, consensusGroupType); case AUTO: default: return autoExtendRegionGroupIfNecessary( - unassignedPartitionSlotsCountMap, consensusGroupType); + unassignedPartitionSlotsMap, consensusGroupType); } } else { switch (CONF.getDataRegionGroupExtensionPolicy()) { + case PROACTIVE: + return proactiveExtendRegionGroupIfNecessary( + unassignedPartitionSlotsMap, consensusGroupType); case CUSTOM: return customExtendRegionGroupIfNecessary( - unassignedPartitionSlotsCountMap, consensusGroupType); + unassignedPartitionSlotsMap, consensusGroupType); case AUTO: default: return autoExtendRegionGroupIfNecessary( - unassignedPartitionSlotsCountMap, consensusGroupType); + unassignedPartitionSlotsMap, consensusGroupType); } } } catch (NotEnoughDataNodeException e) { @@ -645,16 +651,62 @@ private TSStatus extendRegionGroupIfNecessary( return result; } + private TSStatus proactiveExtendRegionGroupIfNecessary( + final Map> unassignedPartitionSlotsMap, + final TConsensusGroupType consensusGroupType) + throws DatabaseNotExistsException, NotEnoughDataNodeException { + final Map allotmentMap = new HashMap<>(); + for (Map.Entry> entry : + unassignedPartitionSlotsMap.entrySet()) { + final String database = entry.getKey(); + final int maxRegionGroupCount = + getClusterSchemaManager().getMaxRegionGroupNum(database, consensusGroupType); + final int minRegionGroupCount = + getClusterSchemaManager().getMinRegionGroupNum(database, consensusGroupType); + final int allocatedRegionGroupCount = + partitionInfo.getRegionGroupCount(database, consensusGroupType); + // As with AUTO, grow toward the minimum by at most the number of pending series slots. + // An existing data slot with new time partitions also contributes to this incremental growth. + final int minimumRegionGroupTarget = + Math.min( + minRegionGroupCount, + allocatedRegionGroupCount + new HashSet<>(entry.getValue()).size()); + final int targetRegionGroupCount = + Math.min( + maxRegionGroupCount, + Math.max( + minimumRegionGroupTarget, + partitionInfo.getSeriesPartitionSlotsCount( + database, consensusGroupType, entry.getValue()))); + // Also allocate one group per active series slot, up to the same maximum as AUTO. + // Existing groups are retained when the maximum decreases or the policy changes. + if (allocatedRegionGroupCount < targetRegionGroupCount) { + allotmentMap.put(database, targetRegionGroupCount - allocatedRegionGroupCount); + } else if (!entry.getValue().isEmpty() + && allocatedRegionGroupCount > 0 + && allocatedRegionGroupCount < maxRegionGroupCount + && partitionInfo.getAllRegionGroupIds(database, consensusGroupType).stream() + .allMatch( + regionGroupId -> + RegionGroupStatus.Disabled.equals( + getLoadManager().getRegionGroupStatus(regionGroupId)))) { + // As with AUTO, preserve availability when all groups are disabled. This may exceed + // the active-slot target, but not the database's maximum for this region type. + allotmentMap.put(database, 1); + } + } + return generateAndAllocateRegionGroups(allotmentMap, consensusGroupType); + } + private TSStatus customExtendRegionGroupIfNecessary( - final Map unassignedPartitionSlotsCountMap, + final Map> unassignedPartitionSlotsMap, final TConsensusGroupType consensusGroupType) throws DatabaseNotExistsException, NotEnoughDataNodeException { // Map final Map allotmentMap = new ConcurrentHashMap<>(); - for (final Map.Entry entry : unassignedPartitionSlotsCountMap.entrySet()) { - final String database = entry.getKey(); + for (final String database : unassignedPartitionSlotsMap.keySet()) { final int maxRegionGroupNum = getClusterSchemaManager().getMaxRegionGroupNum(database, consensusGroupType); final int allocatedRegionGroupCount = @@ -670,16 +722,17 @@ private TSStatus customExtendRegionGroupIfNecessary( } private TSStatus autoExtendRegionGroupIfNecessary( - final Map unassignedPartitionSlotsCountMap, + final Map> unassignedPartitionSlotsMap, final TConsensusGroupType consensusGroupType) throws NotEnoughDataNodeException, DatabaseNotExistsException { // Map final Map allotmentMap = new ConcurrentHashMap<>(); - for (Map.Entry entry : unassignedPartitionSlotsCountMap.entrySet()) { + for (Map.Entry> entry : + unassignedPartitionSlotsMap.entrySet()) { final String database = entry.getKey(); - final int unassignedPartitionSlotsCount = entry.getValue(); + final int unassignedPartitionSlotsCount = entry.getValue().size(); float allocatedRegionGroupCount = partitionInfo.getRegionGroupCount(database, consensusGroupType); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/RegionGroupExtensionPolicy.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/RegionGroupExtensionPolicy.java index 5f69a5d3525a5..fca087933ddab 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/RegionGroupExtensionPolicy.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/RegionGroupExtensionPolicy.java @@ -25,7 +25,9 @@ public enum RegionGroupExtensionPolicy { CUSTOM("CUSTOM"), - AUTO("AUTO"); + AUTO("AUTO"), + + PROACTIVE("PROACTIVE"); private final String policy; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java index 1ed35210afcd2..d3528c40ef835 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java @@ -46,6 +46,7 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -267,6 +268,21 @@ public int getAssignedSeriesPartitionSlotsCount() { dataPartitionTable.getDataPartitionMap().size()); } + /** Count distinct series slots after allocation, independently for schema and data partitions. */ + public int getSeriesPartitionSlotsCount( + TConsensusGroupType type, Collection unassignedSlots) { + final Set assignedSlots = + type == TConsensusGroupType.SchemaRegion + ? schemaPartitionTable.getSchemaPartitionMap().keySet() + : dataPartitionTable.getDataPartitionMap().entrySet().stream() + .filter(entry -> !entry.getValue().getSeriesPartitionMap().isEmpty()) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + final Set newSlots = new HashSet<>(unassignedSlots); + newSlots.removeAll(assignedSlots); + return assignedSlots.size() + newSlots.size(); + } + /** * Thread-safely get SchemaPartition within the specific StorageGroup. * diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java index 71fef7c21d017..7459510f4a23c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java @@ -89,6 +89,7 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -911,6 +912,17 @@ public int getAssignedSeriesPartitionSlotsCount(String database) { .orElse(0); } + /** Count the series slots of one region type, including those in the pending allocation. */ + public int getSeriesPartitionSlotsCount( + String database, TConsensusGroupType type, Collection unassignedSlots) + throws DatabaseNotExistsException { + final DatabasePartitionTable partitionTable = databasePartitionTables.get(database); + if (partitionTable == null || !partitionTable.isNotPreDeleted()) { + throw new DatabaseNotExistsException(database); + } + return partitionTable.getSeriesPartitionSlotsCount(type, unassignedSlots); + } + /** * Only leader use this interface. * diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/conf/RegionGroupExtensionPolicyConfigTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/conf/RegionGroupExtensionPolicyConfigTest.java new file mode 100644 index 0000000000000..dbda9eb9c35e8 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/conf/RegionGroupExtensionPolicyConfigTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.conf; + +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.TrimProperties; +import org.apache.iotdb.confignode.manager.partition.RegionGroupExtensionPolicy; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +public class RegionGroupExtensionPolicyConfigTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testParsePolicies() throws Exception { + for (RegionGroupExtensionPolicy policy : RegionGroupExtensionPolicy.values()) { + assertEquals(policy, RegionGroupExtensionPolicy.parse(policy.getPolicy())); + } + assertEquals( + RegionGroupExtensionPolicy.PROACTIVE, RegionGroupExtensionPolicy.parse("PROACTIVE")); + assertThrows(IOException.class, () -> RegionGroupExtensionPolicy.parse("UNKNOWN")); + } + + @Test + public void testProactiveStartupAndHotReloadForBothTypes() throws Exception { + String originalConf = System.getProperty(ConfigNodeConstant.CONFIGNODE_CONF); + File confDir = temporaryFolder.newFolder(); + Files.writeString( + confDir.toPath().resolve(CommonConfig.SYSTEM_CONFIG_NAME), + "cn_seed_config_node=127.0.0.1:10710\n" + + "schema_region_group_extension_policy=PROACTIVE\n" + + "data_region_group_extension_policy=PROACTIVE\n", + StandardCharsets.UTF_8); + System.setProperty(ConfigNodeConstant.CONFIGNODE_CONF, confDir.getAbsolutePath()); + try { + Constructor constructor = + ConfigNodeDescriptor.class.getDeclaredConstructor(); + constructor.setAccessible(true); + ConfigNodeDescriptor descriptor = constructor.newInstance(); + ConfigNodeConfig conf = descriptor.getConf(); + assertEquals( + RegionGroupExtensionPolicy.PROACTIVE, conf.getSchemaRegionGroupExtensionPolicy()); + assertEquals(RegionGroupExtensionPolicy.PROACTIVE, conf.getDataRegionGroupExtensionPolicy()); + + TrimProperties properties = new TrimProperties(); + properties.setProperty("schema_region_group_extension_policy", "CUSTOM"); + properties.setProperty("data_region_group_extension_policy", "AUTO"); + descriptor.loadHotModifiedProps(properties); + assertEquals(RegionGroupExtensionPolicy.CUSTOM, conf.getSchemaRegionGroupExtensionPolicy()); + assertEquals(RegionGroupExtensionPolicy.AUTO, conf.getDataRegionGroupExtensionPolicy()); + + properties.setProperty("schema_region_group_extension_policy", "PROACTIVE"); + properties.setProperty("data_region_group_extension_policy", "PROACTIVE"); + descriptor.loadHotModifiedProps(properties); + assertEquals( + RegionGroupExtensionPolicy.PROACTIVE, conf.getSchemaRegionGroupExtensionPolicy()); + assertEquals(RegionGroupExtensionPolicy.PROACTIVE, conf.getDataRegionGroupExtensionPolicy()); + + TrimProperties schemaOnly = new TrimProperties(); + schemaOnly.setProperty("schema_region_group_extension_policy", "AUTO"); + descriptor.loadHotModifiedProps(schemaOnly); + assertEquals(RegionGroupExtensionPolicy.AUTO, conf.getSchemaRegionGroupExtensionPolicy()); + assertEquals(RegionGroupExtensionPolicy.PROACTIVE, conf.getDataRegionGroupExtensionPolicy()); + + TrimProperties dataOnly = new TrimProperties(); + dataOnly.setProperty("data_region_group_extension_policy", "CUSTOM"); + descriptor.loadHotModifiedProps(dataOnly); + assertEquals(RegionGroupExtensionPolicy.AUTO, conf.getSchemaRegionGroupExtensionPolicy()); + assertEquals(RegionGroupExtensionPolicy.CUSTOM, conf.getDataRegionGroupExtensionPolicy()); + } finally { + if (originalConf == null) { + System.clearProperty(ConfigNodeConstant.CONFIGNODE_CONF); + } else { + System.setProperty(ConfigNodeConstant.CONFIGNODE_CONF, originalConf); + } + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerRegionGroupPolicyReloadTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerRegionGroupPolicyReloadTest.java new file mode 100644 index 0000000000000..8b8e4bd5da917 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerRegionGroupPolicyReloadTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.manager; + +import org.apache.iotdb.confignode.conf.ConfigNodeConfig; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.partition.RegionGroupExtensionPolicy; +import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.util.Arrays; + +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +public class ConfigManagerRegionGroupPolicyReloadTest { + private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); + private RegionGroupExtensionPolicy originalSchemaPolicy; + private RegionGroupExtensionPolicy originalDataPolicy; + private ConfigManager manager; + private ConsensusManager consensusManager; + private ClusterSchemaManager schemaManager; + private LoadManager loadManager; + + @Before + public void setUp() { + originalSchemaPolicy = CONF.getSchemaRegionGroupExtensionPolicy(); + originalDataPolicy = CONF.getDataRegionGroupExtensionPolicy(); + CONF.setSchemaRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.AUTO); + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.PROACTIVE); + manager = mock(ConfigManager.class); + consensusManager = mock(ConsensusManager.class); + schemaManager = mock(ClusterSchemaManager.class); + loadManager = mock(LoadManager.class); + when(manager.getConsensusManager()).thenReturn(consensusManager); + when(manager.getClusterSchemaManager()).thenReturn(schemaManager); + when(manager.getLoadManager()).thenReturn(loadManager); + when(consensusManager.isLeader()).thenReturn(true); + when(schemaManager.getDatabaseNames(null)).thenReturn(Arrays.asList("root.one", "root.two")); + } + + @After + public void tearDown() { + CONF.setSchemaRegionGroupExtensionPolicy(originalSchemaPolicy); + CONF.setDataRegionGroupExtensionPolicy(originalDataPolicy); + } + + @Test + public void testEnteringProactiveRebalancesEveryDatabase() throws Exception { + for (RegionGroupExtensionPolicy previous : + Arrays.asList(RegionGroupExtensionPolicy.AUTO, RegionGroupExtensionPolicy.CUSTOM)) { + clearInvocations(schemaManager, loadManager); + reload(RegionGroupExtensionPolicy.AUTO, previous, CONF.getDataRegionPerDataNode()); + verify(schemaManager).adjustMaxRegionGroupNum(); + verify(loadManager).reBalanceDataPartitionPolicy("root.one"); + verify(loadManager).reBalanceDataPartitionPolicy("root.two"); + } + } + + @Test + public void testRepeatedPolicyDoesNotRebalance() throws Exception { + reload( + RegionGroupExtensionPolicy.AUTO, + RegionGroupExtensionPolicy.PROACTIVE, + CONF.getDataRegionPerDataNode()); + verifyZeroInteractions(loadManager, schemaManager); + } + + @Test + public void testSchemaAndQuotaChangesDoNotRebalanceData() throws Exception { + reload( + RegionGroupExtensionPolicy.CUSTOM, + RegionGroupExtensionPolicy.PROACTIVE, + CONF.getDataRegionPerDataNode()); + reload( + RegionGroupExtensionPolicy.AUTO, + RegionGroupExtensionPolicy.PROACTIVE, + CONF.getDataRegionPerDataNode() + 1); + verifyZeroInteractions(loadManager); + } + + @Test + public void testLeavingProactiveDoesNotRebalance() throws Exception { + for (RegionGroupExtensionPolicy next : + Arrays.asList(RegionGroupExtensionPolicy.AUTO, RegionGroupExtensionPolicy.CUSTOM)) { + CONF.setDataRegionGroupExtensionPolicy(next); + reload( + RegionGroupExtensionPolicy.AUTO, + RegionGroupExtensionPolicy.PROACTIVE, + CONF.getDataRegionPerDataNode()); + } + verifyZeroInteractions(loadManager); + } + + @Test + public void testFollowerDoesNotRebalance() throws Exception { + when(consensusManager.isLeader()).thenReturn(false); + reload( + RegionGroupExtensionPolicy.AUTO, + RegionGroupExtensionPolicy.CUSTOM, + CONF.getDataRegionPerDataNode()); + verifyZeroInteractions(loadManager, schemaManager); + } + + private void reload( + RegionGroupExtensionPolicy previousSchema, + RegionGroupExtensionPolicy previousData, + int previousDataQuota) + throws Exception { + Method reload = + ConfigManager.class.getDeclaredMethod( + "handleRegionGroupConfigHotReload", + int.class, + int.class, + RegionGroupExtensionPolicy.class, + RegionGroupExtensionPolicy.class); + reload.setAccessible(true); + reload.invoke( + manager, + CONF.getSchemaRegionPerDataNode(), + previousDataQuota, + previousSchema, + previousData); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTableProactiveTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTableProactiveTest.java new file mode 100644 index 0000000000000..9b0c6117c9877 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTableProactiveTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.load.balancer.partition; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.confignode.conf.ConfigNodeConfig; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; +import org.apache.iotdb.confignode.manager.partition.RegionGroupExtensionPolicy; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +public class DataPartitionPolicyTableProactiveTest { + + private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); + private RegionGroupExtensionPolicy originalPolicy; + private int originalSlotCount; + + @Before + public void setUp() { + originalPolicy = CONF.getDataRegionGroupExtensionPolicy(); + originalSlotCount = CONF.getSeriesSlotNum(); + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.PROACTIVE); + CONF.setSeriesSlotNum(1000); + } + + @After + public void tearDown() { + CONF.setDataRegionGroupExtensionPolicy(originalPolicy); + CONF.setSeriesSlotNum(originalSlotCount); + } + + @Test + public void testSixExistingSlotsUseNewGroupsAfterSwitchingFromAuto() { + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.AUTO); + table.reBalanceDataPartitionPolicy(groups(2)); + Assert.assertEquals(2, new HashSet<>(activate(table, 6).values()).size()); + + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.PROACTIVE); + table.reBalanceDataPartitionPolicy(groups(6)); + assertBalanced(groups(6), activate(table, 6)); + } + + @Test + public void testCustomExpansionIsRebalancedWhenEnteringProactive() { + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.CUSTOM); + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + table.reBalanceDataPartitionPolicy(groups(2)); + Map original = activate(table, 6); + table.reBalanceDataPartitionPolicy(groups(6)); + Assert.assertEquals(original, activate(table, 6)); + + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.PROACTIVE); + table.reBalanceDataPartitionPolicy(groups(6)); + assertBalanced(groups(6), activate(table, 6)); + } + + @Test + public void testUnevenSlotCountStillUsesEveryNewGroup() { + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + table.reBalanceDataPartitionPolicy(groups(2)); + activate(table, 5); + table.reBalanceDataPartitionPolicy(groups(4)); + assertBalanced(groups(4), activate(table, 5)); + } + + @Test + public void testGrowingAheadOfPendingSlotsPreservesExistingAssignments() { + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + for (int slot = 0; slot < 6; slot++) { + Map existing = activate(table, slot); + table.reBalanceDataPartitionPolicy(groups(slot + 1)); + Assert.assertEquals(existing, activate(table, slot)); + assertBalanced(groups(slot + 1), activate(table, slot + 1)); + } + } + + @Test + public void testLeaderRecoveryRebalancesSparseSlotsAndIgnoresEmptyEntries() { + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + table.reBalanceDataPartitionPolicy(groups(6)); + Map previous = new HashMap<>(); + for (int slot = 0; slot < 6; slot++) { + previous.put(new TSeriesPartitionSlot(slot), groups(2).get(slot % 2)); + } + // DataPartitionTable can retain empty entries after filtering or time-partition cleanup. + for (int slot = 6; slot < 100; slot++) { + previous.put(new TSeriesPartitionSlot(slot), null); + } + table.setDataAllotMap(previous); + assertBalanced(groups(6), activate(table, 6)); + } + + @Test + public void testLeaderRecoveryPreservesAlreadyBalancedAssignments() { + DataPartitionPolicyTable original = new DataPartitionPolicyTable(); + original.reBalanceDataPartitionPolicy(groups(6)); + Map previous = activate(original, 6); + DataPartitionPolicyTable recovered = new DataPartitionPolicyTable(); + recovered.reBalanceDataPartitionPolicy(groups(6)); + recovered.setDataAllotMap(previous); + Assert.assertEquals(previous, activate(recovered, 6)); + } + + @Test + public void testAutoAndCustomKeepExistingSparseAssignments() { + for (RegionGroupExtensionPolicy policy : + Arrays.asList(RegionGroupExtensionPolicy.AUTO, RegionGroupExtensionPolicy.CUSTOM)) { + CONF.setDataRegionGroupExtensionPolicy(policy); + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + table.reBalanceDataPartitionPolicy(groups(2)); + Map previous = activate(table, 6); + table.reBalanceDataPartitionPolicy(groups(6)); + Assert.assertEquals(previous, activate(table, 6)); + + DataPartitionPolicyTable recovered = new DataPartitionPolicyTable(); + recovered.reBalanceDataPartitionPolicy(groups(6)); + recovered.setDataAllotMap(previous); + Assert.assertEquals(previous, activate(recovered, 6)); + } + } + + @Test + public void testEmptyRecoveryCanActivateFirstSlot() { + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + table.reBalanceDataPartitionPolicy(Collections.emptyList()); + table.setDataAllotMap(Collections.emptyMap()); + table.reBalanceDataPartitionPolicy(groups(1)); + table.setDataAllotMap(Collections.emptyMap()); + assertBalanced(groups(1), activate(table, 1)); + } + + private static List groups(int count) { + List groups = new ArrayList<>(); + for (int id = 0; id < count; id++) { + groups.add(new TConsensusGroupId(TConsensusGroupType.DataRegion, id)); + } + return groups; + } + + private static Map activate( + DataPartitionPolicyTable table, int slotCount) { + Map assignments = new HashMap<>(); + for (int id = 0; id < slotCount; id++) { + TSeriesPartitionSlot slot = new TSeriesPartitionSlot(id); + assignments.put(slot, table.getRegionGroupIdOrActivateIfNecessary(slot)); + } + return assignments; + } + + private static void assertBalanced( + List groups, Map assignments) { + Map counts = new HashMap<>(); + assignments.values().forEach(group -> counts.merge(group, 1, Integer::sum)); + Assert.assertEquals(new HashSet<>(groups), counts.keySet()); + Assert.assertTrue( + counts.toString(), + Collections.max(counts.values()) - Collections.min(counts.values()) <= 1); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/partition/PartitionManagerProactiveExtensionTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/partition/PartitionManagerProactiveExtensionTest.java new file mode 100644 index 0000000000000..1eb9c7acb203e --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/partition/PartitionManagerProactiveExtensionTest.java @@ -0,0 +1,811 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.manager.partition; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.partition.DataPartitionTable; +import org.apache.iotdb.commons.partition.SchemaPartitionTable; +import org.apache.iotdb.commons.partition.SeriesPartitionTable; +import org.apache.iotdb.confignode.conf.ConfigNodeConfig; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.partition.GetDataPartitionPlan; +import org.apache.iotdb.confignode.consensus.request.read.partition.GetOrCreateDataPartitionPlan; +import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; +import org.apache.iotdb.confignode.consensus.request.write.database.PreDeleteDatabasePlan; +import org.apache.iotdb.confignode.consensus.request.write.partition.CreateDataPartitionPlan; +import org.apache.iotdb.confignode.consensus.request.write.partition.CreateSchemaPartitionPlan; +import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; +import org.apache.iotdb.confignode.exception.NotEnoughDataNodeException; +import org.apache.iotdb.confignode.manager.IManager; +import org.apache.iotdb.confignode.manager.ProcedureManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; +import org.apache.iotdb.confignode.persistence.partition.PartitionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(Parameterized.class) +public class PartitionManagerProactiveExtensionTest { + + private static final String DATABASE = "root.proactive"; + private static final String SECOND_DATABASE = "root.other"; + private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); + + @Parameterized.Parameters(name = "{0}") + public static Collection parameters() { + return Arrays.asList( + new Object[] {TConsensusGroupType.SchemaRegion}, + new Object[] {TConsensusGroupType.DataRegion}); + } + + private final TConsensusGroupType type; + private PartitionInfo partitionInfo; + private PartitionManager partitionManager; + private ClusterSchemaManager schemaManager; + private LoadManager loadManager; + private ProcedureManager procedureManager; + private ConsensusManager consensusManager; + private RegionGroupExtensionPolicy originalSchemaPolicy; + private RegionGroupExtensionPolicy originalDataPolicy; + private int originalSeriesSlotNum; + + public PartitionManagerProactiveExtensionTest(TConsensusGroupType type) { + this.type = type; + } + + @Before + public void setUp() throws Exception { + originalSchemaPolicy = CONF.getSchemaRegionGroupExtensionPolicy(); + originalDataPolicy = CONF.getDataRegionGroupExtensionPolicy(); + originalSeriesSlotNum = CONF.getSeriesSlotNum(); + CONF.setSchemaRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.PROACTIVE); + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.PROACTIVE); + CONF.setSeriesSlotNum(100); + + partitionInfo = new PartitionInfo(); + for (String database : Arrays.asList(DATABASE, SECOND_DATABASE)) { + partitionInfo.createDatabase( + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(database))); + } + IManager manager = mock(IManager.class); + schemaManager = mock(ClusterSchemaManager.class); + loadManager = mock(LoadManager.class); + procedureManager = mock(ProcedureManager.class); + consensusManager = mock(ConsensusManager.class); + when(manager.getClusterSchemaManager()).thenReturn(schemaManager); + when(manager.getLoadManager()).thenReturn(loadManager); + when(manager.getProcedureManager()).thenReturn(procedureManager); + when(manager.getConsensusManager()).thenReturn(consensusManager); + when(schemaManager.getMinRegionGroupNum(DATABASE, type)).thenReturn(4); + when(schemaManager.getMaxRegionGroupNum(DATABASE, type)).thenReturn(5); + when(schemaManager.getMaxRegionGroupNum(SECOND_DATABASE, type)).thenReturn(3); + when(loadManager.allocateRegionGroups(anyMap(), any(TConsensusGroupType.class))) + .thenAnswer( + invocation -> { + Map allotments = invocation.getArgument(0); + TConsensusGroupType allocatedType = invocation.getArgument(1); + return regionPlan(allotments, allocatedType); + }); + when(procedureManager.createRegionGroups(any(TConsensusGroupType.class), any())) + .thenAnswer( + invocation -> { + partitionInfo.createRegionGroups(invocation.getArgument(1)); + return RpcUtils.SUCCESS_STATUS; + }); + partitionManager = new PartitionManager(manager, partitionInfo); + } + + @After + public void tearDown() { + if (partitionManager != null) { + partitionManager.stopRegionCleaner(); + } + CONF.setSchemaRegionGroupExtensionPolicy(originalSchemaPolicy); + CONF.setDataRegionGroupExtensionPolicy(originalDataPolicy); + CONF.setSeriesSlotNum(originalSeriesSlotNum); + } + + @Test + public void testEachNewSlotCreatesOneRegionUntilMaximum() throws Exception { + for (int slot = 0; slot < 8; slot++) { + assertSuccess(extend(DATABASE, slots(slot))); + // A configured minimum of four is reached gradually, as with AUTO. + assertEquals(Math.min(slot + 1, 5), partitionInfo.getRegionGroupCount(DATABASE, type)); + persistSlots(DATABASE, type, 0, slot); + } + verify(loadManager, never()).allocateRegionGroups(anyMap(), eq(otherType())); + } + + @Test + public void testDuplicatePendingSlotsDoNotInflateMinimumGrowth() throws Exception { + List pending = slots(0, 0); + if (type == TConsensusGroupType.SchemaRegion) { + pending = + partitionInfo + .filterUnassignedSchemaPartitionSlots(Collections.singletonMap(DATABASE, pending)) + .get(DATABASE); + } + assertSuccess(extend(DATABASE, pending)); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + persistSlots(DATABASE, type, 0, 0); + assertSuccess(extend(DATABASE, slots(1, 1))); + assertEquals(2, partitionInfo.getRegionGroupCount(DATABASE, type)); + } + + @Test + public void testMixedDataRequestOnlyCountsSlotsWithMissingTimePartitions() throws Exception { + assumeTrue(type == TConsensusGroupType.DataRegion); + assertSuccess(extend(DATABASE, slots(0, 1))); + persistSlots(DATABASE, type, 0, 0, 1); + configureDataPartitionConsensusAndAllocation(); + clearInvocations(loadManager); + + GetOrCreateDataPartitionPlan request = + new GetOrCreateDataPartitionPlan( + Collections.singletonMap(DATABASE, dataRequest(0, 0, 1, 2))); + assertSuccess(partitionManager.getOrCreateDataPartition(request).getStatus()); + assertEquals(3, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(DATABASE, 1), type); + + clearInvocations(loadManager); + assertSuccess(partitionManager.getOrCreateDataPartition(request).getStatus()); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testSatisfiedDatabaseDoesNotGrowForAnotherDatabasesPendingSlots() throws Exception { + assumeTrue(type == TConsensusGroupType.DataRegion); + assertSuccess(extend(DATABASE, slots(0))); + persistSlots(DATABASE, type, 0, 0); + configureDataPartitionConsensusAndAllocation(); + clearInvocations(loadManager); + + Map> request = new HashMap<>(); + request.put(DATABASE, dataRequest(0, 0)); + request.put(SECOND_DATABASE, dataRequest(0, 1)); + assertSuccess( + partitionManager + .getOrCreateDataPartition(new GetOrCreateDataPartitionPlan(request)) + .getStatus()); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + assertEquals(1, partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(SECOND_DATABASE, 1), type); + } + + @Test + public void testNewTimePartitionsStillGrowSingleSlotToMinimum() throws Exception { + assumeTrue(type == TConsensusGroupType.DataRegion); + configureDataPartitionConsensusAndAllocation(); + for (int time = 0; time < 6; time++) { + assertSuccess( + partitionManager + .getOrCreateDataPartition( + new GetOrCreateDataPartitionPlan( + Collections.singletonMap(DATABASE, dataRequest(time, 0)))) + .getStatus()); + assertEquals(Math.min(4, time + 1), partitionInfo.getRegionGroupCount(DATABASE, type)); + } + } + + private Map dataRequest(long time, int... ids) { + Map request = new HashMap<>(); + for (TSeriesPartitionSlot slot : slots(ids)) { + request.put( + slot, + new TTimeSlotList(Collections.singletonList(new TTimePartitionSlot(time)), false, false)); + } + return request; + } + + private void configureDataPartitionConsensusAndAllocation() throws Exception { + when(consensusManager.read(any(GetDataPartitionPlan.class))) + .thenAnswer(invocation -> partitionInfo.getDataPartition(invocation.getArgument(0))); + when(consensusManager.confirmLeader()).thenReturn(RpcUtils.SUCCESS_STATUS); + when(consensusManager.write(any(CreateDataPartitionPlan.class))) + .thenAnswer(invocation -> partitionInfo.createDataPartition(invocation.getArgument(0))); + when(loadManager.allocateDataPartition(anyMap())) + .thenAnswer( + invocation -> { + Map> pending = + invocation.getArgument(0); + Map assigned = new HashMap<>(); + for (Map.Entry> database : + pending.entrySet()) { + DataPartitionTable table = new DataPartitionTable(); + TConsensusGroupId region = + partitionInfo.getAllRegionGroupIds(database.getKey(), type).get(0); + database + .getValue() + .forEach( + (slot, times) -> { + SeriesPartitionTable series = new SeriesPartitionTable(); + times + .getTimePartitionSlots() + .forEach(time -> series.putDataPartition(time, region)); + table.getDataPartitionMap().put(slot, series); + }); + assigned.put(database.getKey(), table); + } + return assigned; + }); + } + + @Test + public void testBatchCountsDistinctPersistedAndIncomingSlots() throws Exception { + when(schemaManager.getMinRegionGroupNum(DATABASE, type)).thenReturn(1); + assertSuccess(extend(DATABASE, slots(1))); + persistSlots(DATABASE, type, 0, 1); + clearInvocations(loadManager); + + assertSuccess(extend(DATABASE, slots(1, 2, 2, 3, 3))); + assertEquals(3, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(DATABASE, 2), type); + persistSlots(DATABASE, type, 0, 2, 3); + assertEquals(3, partitionInfo.getSeriesPartitionSlotsCount(DATABASE, type, slots(1, 1))); + + clearInvocations(loadManager); + for (int time = 1; time <= 3; time++) { + assertSuccess(extend(DATABASE, slots(1, 2))); + persistSlots(DATABASE, type, time, 1, 2); + } + assertEquals(3, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testSchemaAndDataActivationAreIndependent() throws Exception { + partitionInfo.createRegionGroups( + regionPlan(Collections.singletonMap(DATABASE, 1), otherType())); + persistSlots(DATABASE, otherType(), 0, 10, 11, 12, 13); + + assertSuccess(extend(DATABASE, slots(10))); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + persistSlots(DATABASE, type, 0, 10); + assertEquals(1, partitionInfo.getSeriesPartitionSlotsCount(DATABASE, type, slots(10))); + assertEquals(4, partitionInfo.getSeriesPartitionSlotsCount(DATABASE, otherType(), slots(10))); + + assertSuccess(extend(DATABASE, slots(11))); + assertEquals(2, partitionInfo.getRegionGroupCount(DATABASE, type)); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, otherType())); + } + + @Test + public void testBatchUsesEachDatabaseMaximum() throws Exception { + Map> requested = new HashMap<>(); + requested.put(DATABASE, slots(0, 1, 2, 3)); + requested.put(SECOND_DATABASE, slots(0, 1, 2, 3, 4, 5)); + assertSuccess(partitionManager.extendRegionGroupIfNecessary(requested, type)); + assertEquals(4, partitionInfo.getRegionGroupCount(DATABASE, type)); + assertEquals(3, partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + Map expected = new HashMap<>(); + expected.put(DATABASE, 4); + expected.put(SECOND_DATABASE, 3); + verify(loadManager).allocateRegionGroups(expected, type); + } + + @Test + public void testNoPreallocationWithoutActivatedSlots() throws Exception { + assertSuccess(partitionManager.extendRegionGroupIfNecessary(Collections.emptyMap(), type)); + assertSuccess(extend(DATABASE, Collections.emptyList())); + assertEquals(0, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testSingleSlotTimePartitionsReachDefaultDataMinimumLikeAuto() throws Exception { + assumeTrue(type == TConsensusGroupType.DataRegion); + checkSingleSlotTimePartitionsReachMinimumLikeAuto(2); + } + + @Test + public void testSingleSlotTimePartitionsReachConfiguredDataMinimumLikeAuto() throws Exception { + assumeTrue(type == TConsensusGroupType.DataRegion); + checkSingleSlotTimePartitionsReachMinimumLikeAuto(4); + } + + @Test + public void testEachDatabaseReachesItsOwnMinimum() throws Exception { + assumeTrue(type == TConsensusGroupType.DataRegion); + when(schemaManager.getMinRegionGroupNum(SECOND_DATABASE, type)).thenReturn(2); + + for (int time = 0; time < 6; time++) { + Map> pending = new HashMap<>(); + for (String database : Arrays.asList(DATABASE, SECOND_DATABASE)) { + pending.put(database, pendingDataSlots(database, time)); + } + assertSuccess(partitionManager.extendRegionGroupIfNecessary(pending, type)); + assertEquals(Math.min(time + 1, 4), partitionInfo.getRegionGroupCount(DATABASE, type)); + assertEquals(Math.min(time + 1, 2), partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + persistSlots(DATABASE, type, time, 0); + persistSlots(SECOND_DATABASE, type, time, 0); + } + } + + @Test + public void testNoMinimumGrowthWithoutPendingSlots() throws Exception { + assertSuccess(extend(DATABASE, slots(0))); + persistSlots(DATABASE, type, 0, 0); + clearInvocations(loadManager); + + assertSuccess(partitionManager.extendRegionGroupIfNecessary(Collections.emptyMap(), type)); + assertSuccess(extend(DATABASE, Collections.emptyList())); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testExistingRegionsAreNotRemovedOrExceeded() throws Exception { + partitionInfo.createRegionGroups(regionPlan(Collections.singletonMap(DATABASE, 4), type)); + when(schemaManager.getMaxRegionGroupNum(DATABASE, type)).thenReturn(3); + assertSuccess(extend(DATABASE, slots(0))); + assertSuccess(extend(DATABASE, slots(0, 1, 2, 3, 4))); + assertEquals(4, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testAllDisabledRegionsCreateOneReplacement() throws Exception { + when(schemaManager.getMinRegionGroupNum(DATABASE, type)).thenReturn(1); + assertSuccess(extend(DATABASE, slots(0))); + persistSlots(DATABASE, type, 0, 0); + TConsensusGroupId disabledGroup = partitionInfo.getAllRegionGroupIds(DATABASE, type).get(0); + when(loadManager.getRegionGroupStatus(disabledGroup)).thenReturn(RegionGroupStatus.Disabled); + clearInvocations(loadManager); + + assertSuccess(extend(DATABASE, slots(0))); + assertEquals(2, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(DATABASE, 1), type); + for (TConsensusGroupId group : partitionInfo.getAllRegionGroupIds(DATABASE, type)) { + if (!disabledGroup.equals(group)) { + when(loadManager.getRegionGroupStatus(group)).thenReturn(RegionGroupStatus.Running); + } + } + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, slots(0))); + assertSuccess(extend(DATABASE, slots(1))); + assertEquals(2, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testAllDisabledRegionsRespectMaximum() throws Exception { + assertSuccess(extend(DATABASE, slots(0, 1, 2, 3, 4))); + persistSlots(DATABASE, type, 0, 0, 1, 2, 3, 4); + for (TConsensusGroupId group : partitionInfo.getAllRegionGroupIds(DATABASE, type)) { + when(loadManager.getRegionGroupStatus(group)).thenReturn(RegionGroupStatus.Disabled); + } + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, slots(0))); + assertEquals(5, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testDisabledStatusOfOtherTypeDoesNotAffectReplacement() throws Exception { + when(schemaManager.getMinRegionGroupNum(DATABASE, type)).thenReturn(1); + assertSuccess(extend(DATABASE, slots(0))); + persistSlots(DATABASE, type, 0, 0); + partitionInfo.createRegionGroups( + regionPlan(Collections.singletonMap(DATABASE, 1), otherType())); + TConsensusGroupId currentGroup = partitionInfo.getAllRegionGroupIds(DATABASE, type).get(0); + TConsensusGroupId otherGroup = partitionInfo.getAllRegionGroupIds(DATABASE, otherType()).get(0); + when(loadManager.getRegionGroupStatus(currentGroup)).thenReturn(RegionGroupStatus.Running); + when(loadManager.getRegionGroupStatus(otherGroup)).thenReturn(RegionGroupStatus.Disabled); + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, slots(0))); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + + when(loadManager.getRegionGroupStatus(currentGroup)).thenReturn(RegionGroupStatus.Disabled); + when(loadManager.getRegionGroupStatus(otherGroup)).thenReturn(RegionGroupStatus.Running); + assertSuccess(extend(DATABASE, slots(0))); + assertEquals(2, partitionInfo.getRegionGroupCount(DATABASE, type)); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, otherType())); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(DATABASE, 1), type); + } + + @Test + public void testNoReplacementWithoutPendingSlots() throws Exception { + assertSuccess(extend(DATABASE, slots(0))); + persistSlots(DATABASE, type, 0, 0); + TConsensusGroupId group = partitionInfo.getAllRegionGroupIds(DATABASE, type).get(0); + when(loadManager.getRegionGroupStatus(group)).thenReturn(RegionGroupStatus.Disabled); + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, Collections.emptyList())); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testCustomStillAllocatesMaximumImmediately() throws Exception { + setPolicy(RegionGroupExtensionPolicy.CUSTOM); + assertSuccess(extend(DATABASE, slots(0))); + assertEquals(5, partitionInfo.getRegionGroupCount(DATABASE, type)); + setPolicy(RegionGroupExtensionPolicy.PROACTIVE); + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, slots(1))); + assertEquals(5, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testAutoKeepsSlowerGrowthAndSwitchCatchesUp() throws Exception { + setPolicy(RegionGroupExtensionPolicy.AUTO); + when(schemaManager.getMinRegionGroupNum(DATABASE, type)).thenReturn(1); + for (int slot = 0; slot < 3; slot++) { + assertSuccess(extend(DATABASE, slots(slot))); + persistSlots(DATABASE, type, 0, slot); + } + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + + setPolicy(RegionGroupExtensionPolicy.PROACTIVE); + assertSuccess(extend(DATABASE, slots(3))); + assertEquals(4, partitionInfo.getRegionGroupCount(DATABASE, type)); + } + + @Test + public void testAutoAndProactiveReachSameMaximum() throws Exception { + setPolicy(RegionGroupExtensionPolicy.AUTO); + List allSlots = + IntStream.range(0, CONF.getSeriesSlotNum()) + .mapToObj(TSeriesPartitionSlot::new) + .collect(Collectors.toList()); + assertSuccess(extend(DATABASE, allSlots)); + assertEquals(5, partitionInfo.getRegionGroupCount(DATABASE, type)); + + setPolicy(RegionGroupExtensionPolicy.PROACTIVE); + when(schemaManager.getMaxRegionGroupNum(SECOND_DATABASE, type)).thenReturn(5); + assertSuccess(extend(SECOND_DATABASE, allSlots)); + assertEquals(5, partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + } + + @Test + public void testSixSlotsGrowFasterThanAutoWithOneThousandSlots() throws Exception { + configureSparseSlotComparison(); + int minimum = type == TConsensusGroupType.SchemaRegion ? 1 : 2; + setPolicy(RegionGroupExtensionPolicy.AUTO); + for (int slot = 0; slot < 6; slot++) { + assertSuccess(extend(DATABASE, slots(slot))); + persistSlots(DATABASE, type, 0, slot); + assertEquals(Math.min(slot + 1, minimum), partitionInfo.getRegionGroupCount(DATABASE, type)); + } + + setPolicy(RegionGroupExtensionPolicy.PROACTIVE); + for (int slot = 0; slot < 6; slot++) { + assertSuccess(extend(SECOND_DATABASE, slots(slot))); + persistSlots(SECOND_DATABASE, type, 0, slot); + assertEquals(slot + 1, partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + } + // Six active slots are still below the nine-group resource limit. + assertEquals(6, partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + clearInvocations(loadManager); + assertSuccess(extend(SECOND_DATABASE, slots(0, 1, 2, 3, 4, 5))); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + + for (int slot = 6; slot < 11; slot++) { + assertSuccess(extend(SECOND_DATABASE, slots(slot))); + persistSlots(SECOND_DATABASE, type, 0, slot); + assertEquals(Math.min(slot + 1, 9), partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + } + } + + @Test + public void testSixSlotBatchGrowsFasterThanAutoWithOneThousandSlots() throws Exception { + configureSparseSlotComparison(); + setPolicy(RegionGroupExtensionPolicy.AUTO); + assertSuccess(extend(DATABASE, slots(0, 1, 2, 3, 4, 5))); + assertEquals( + type == TConsensusGroupType.SchemaRegion ? 1 : 2, + partitionInfo.getRegionGroupCount(DATABASE, type)); + + setPolicy(RegionGroupExtensionPolicy.PROACTIVE); + assertSuccess(extend(SECOND_DATABASE, slots(0, 1, 2, 3, 4, 5))); + assertEquals(6, partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(SECOND_DATABASE, 6), type); + } + + @Test + public void testFilteredRequestCountsEachSeriesSlotOnce() throws Exception { + when(schemaManager.getMinRegionGroupNum(DATABASE, type)).thenReturn(1); + assertSuccess(extend(DATABASE, slots(0))); + persistSlots(DATABASE, type, 0, 0); + + Collection pendingSlots; + if (type == TConsensusGroupType.SchemaRegion) { + pendingSlots = + partitionInfo + .filterUnassignedSchemaPartitionSlots( + Collections.singletonMap(DATABASE, slots(0, 1, 2))) + .get(DATABASE); + assertEquals(slots(1, 2), pendingSlots); + } else { + Map request = new HashMap<>(); + for (TSeriesPartitionSlot slot : slots(0, 1, 2)) { + request.put( + slot, + new TTimeSlotList(Collections.singletonList(new TTimePartitionSlot(1)), false, false)); + } + pendingSlots = + partitionInfo + .filterUnassignedDataPartitionSlots(Collections.singletonMap(DATABASE, request)) + .get(DATABASE) + .keySet(); + // Filtering inserted empty entries for slots 1 and 2, but only slot 0 is assigned. + assertEquals( + 1, partitionInfo.getSeriesPartitionSlotsCount(DATABASE, type, Collections.emptyList())); + assertEquals(2, partitionInfo.getSeriesPartitionSlotsCount(DATABASE, type, slots(3, 3))); + } + assertSuccess(extend(DATABASE, pendingSlots)); + assertEquals(3, partitionInfo.getRegionGroupCount(DATABASE, type)); + } + + @Test + public void testPreDeletedDatabaseIsReported() throws Exception { + partitionInfo.preDeleteDatabase( + new PreDeleteDatabasePlan(DATABASE, PreDeleteDatabasePlan.PreDeleteType.EXECUTE)); + assertEquals( + TSStatusCode.DATABASE_NOT_EXIST.getStatusCode(), extend(DATABASE, slots(0)).getCode()); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testHigherResourceMaximumAllowsFurtherGrowth() throws Exception { + when(schemaManager.getMaxRegionGroupNum(DATABASE, type)).thenReturn(3); + assertSuccess(extend(DATABASE, slots(0, 1, 2, 3, 4, 5))); + persistSlots(DATABASE, type, 0, 0, 1, 2, 3, 4, 5); + assertEquals(3, partitionInfo.getRegionGroupCount(DATABASE, type)); + clearInvocations(loadManager); + + when(schemaManager.getMaxRegionGroupNum(DATABASE, type)).thenReturn(5); + assertSuccess(extend(DATABASE, slots(6))); + assertEquals(5, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(DATABASE, 2), type); + } + + @Test + public void testAllocationFailureIsReturnedWithoutPersistingRegions() throws Exception { + doThrow(new NotEnoughDataNodeException(Collections.emptyList(), 3)) + .when(loadManager) + .allocateRegionGroups(anyMap(), eq(type)); + assertEquals( + TSStatusCode.NO_ENOUGH_DATANODE.getStatusCode(), extend(DATABASE, slots(0)).getCode()); + assertEquals(0, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(procedureManager, never()).createRegionGroups(any(), any()); + } + + @Test + public void testProcedureFailureIsReturned() throws Exception { + TSStatus failure = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + doReturn(failure).when(procedureManager).createRegionGroups(eq(type), any()); + assertEquals(failure, extend(DATABASE, slots(0))); + assertEquals(0, partitionInfo.getRegionGroupCount(DATABASE, type)); + } + + @Test + public void testRetryOnlyCreatesRemainingRegionsAfterPartialSuccess() throws Exception { + TSStatus failure = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + doAnswer( + invocation -> { + CreateRegionGroupsPlan requested = invocation.getArgument(1); + CreateRegionGroupsPlan partial = new CreateRegionGroupsPlan(); + partial.addRegionGroup(DATABASE, requested.getRegionGroupMap().get(DATABASE).get(0)); + partitionInfo.createRegionGroups(partial); + return failure; + }) + .doAnswer( + invocation -> { + partitionInfo.createRegionGroups(invocation.getArgument(1)); + return RpcUtils.SUCCESS_STATUS; + }) + .when(procedureManager) + .createRegionGroups(eq(type), any()); + + assertEquals(failure, extend(DATABASE, slots(0, 1, 2))); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, slots(0, 1, 2))); + // The three pending slots can now complete the configured minimum of four. + assertEquals(4, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(DATABASE, 3), type); + persistSlots(DATABASE, type, 0, 0, 1, 2); + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, slots(0, 1, 2))); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testMissingDatabaseIsReported() throws Exception { + assertEquals( + TSStatusCode.DATABASE_NOT_EXIST.getStatusCode(), + extend("root.missing", slots(0)).getCode()); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + private TSStatus extend(String database, Collection requested) { + return partitionManager.extendRegionGroupIfNecessary( + Collections.singletonMap(database, requested), type); + } + + private void checkSingleSlotTimePartitionsReachMinimumLikeAuto(int minimum) throws Exception { + for (String database : Arrays.asList(DATABASE, SECOND_DATABASE)) { + when(schemaManager.getMinRegionGroupNum(database, type)).thenReturn(minimum); + when(schemaManager.getMaxRegionGroupNum(database, type)).thenReturn(5); + setPolicy( + database.equals(DATABASE) + ? RegionGroupExtensionPolicy.AUTO + : RegionGroupExtensionPolicy.PROACTIVE); + + for (int request = 0; request < minimum + 2; request++) { + // Several new time partitions of one existing series slot still add at most one group. + int firstTime = request * 3; + Collection pending = + pendingDataSlots(database, firstTime, firstTime + 1, firstTime + 2); + assertEquals(slots(0), pending); + if (request >= minimum) { + clearInvocations(loadManager); + } + assertSuccess(extend(database, pending)); + assertEquals( + Math.min(request + 1, minimum), partitionInfo.getRegionGroupCount(database, type)); + if (request >= minimum) { + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + for (int time = firstTime; time < firstTime + 3; time++) { + persistSlots(database, type, time, 0); + } + assertEquals( + 1, partitionInfo.getSeriesPartitionSlotsCount(database, type, Collections.emptyList())); + assertEquals( + Collections.emptyList(), + unassignedDataSlots(database, firstTime) + .get(new TSeriesPartitionSlot(0)) + .getTimePartitionSlots()); + } + } + } + + private List pendingDataSlots(String database, int... times) { + return unassignedDataSlots(database, times).keySet().stream().collect(Collectors.toList()); + } + + private Map unassignedDataSlots( + String database, int... times) { + TTimeSlotList timeSlots = + new TTimeSlotList( + Arrays.stream(times).mapToObj(TTimePartitionSlot::new).collect(Collectors.toList()), + false, + false); + return partitionInfo + .filterUnassignedDataPartitionSlots( + Collections.singletonMap( + database, Collections.singletonMap(new TSeriesPartitionSlot(0), timeSlots))) + .get(database); + } + + private void configureSparseSlotComparison() { + CONF.setSeriesSlotNum(1000); + int minimum = type == TConsensusGroupType.SchemaRegion ? 1 : 2; + for (String database : Arrays.asList(DATABASE, SECOND_DATABASE)) { + when(schemaManager.getMinRegionGroupNum(database, type)).thenReturn(minimum); + when(schemaManager.getMaxRegionGroupNum(database, type)).thenReturn(9); + } + } + + private void setPolicy(RegionGroupExtensionPolicy policy) { + if (type == TConsensusGroupType.SchemaRegion) { + CONF.setSchemaRegionGroupExtensionPolicy(policy); + } else { + CONF.setDataRegionGroupExtensionPolicy(policy); + } + } + + private TConsensusGroupType otherType() { + return type == TConsensusGroupType.SchemaRegion + ? TConsensusGroupType.DataRegion + : TConsensusGroupType.SchemaRegion; + } + + private CreateRegionGroupsPlan regionPlan( + Map allotments, TConsensusGroupType allocatedType) { + CreateRegionGroupsPlan plan = new CreateRegionGroupsPlan(); + allotments.forEach( + (database, count) -> { + for (int i = 0; i < count; i++) { + plan.addRegionGroup( + database, + new TRegionReplicaSet( + new TConsensusGroupId(allocatedType, partitionInfo.generateNextRegionGroupId()), + Collections.singletonList(new TDataNodeLocation().setDataNodeId(0)))); + } + }); + return plan; + } + + private void persistSlots( + String database, TConsensusGroupType persistedType, long time, int... slotIds) + throws Exception { + TConsensusGroupId regionId = partitionInfo.getAllRegionGroupIds(database, persistedType).get(0); + if (persistedType == TConsensusGroupType.SchemaRegion) { + SchemaPartitionTable table = new SchemaPartitionTable(); + for (TSeriesPartitionSlot slot : slots(slotIds)) { + table.getSchemaPartitionMap().put(slot, regionId); + } + CreateSchemaPartitionPlan plan = new CreateSchemaPartitionPlan(); + plan.setAssignedSchemaPartition(Collections.singletonMap(database, table)); + partitionInfo.createSchemaPartition(plan); + } else { + DataPartitionTable table = new DataPartitionTable(); + for (TSeriesPartitionSlot slot : slots(slotIds)) { + SeriesPartitionTable seriesTable = new SeriesPartitionTable(); + seriesTable.putDataPartition(new TTimePartitionSlot(time), regionId); + table.getDataPartitionMap().put(slot, seriesTable); + } + CreateDataPartitionPlan plan = new CreateDataPartitionPlan(); + plan.setAssignedDataPartition(Collections.singletonMap(database, table)); + partitionInfo.createDataPartition(plan); + } + } + + private static List slots(int... ids) { + return Arrays.stream(ids).mapToObj(TSeriesPartitionSlot::new).collect(Collectors.toList()); + } + + private static void assertSuccess(TSStatus status) { + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } +} diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 2c6186f6b4dd8..ce806f57541ec 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -678,51 +678,86 @@ series_slot_num=1000 # Datatype: String series_partition_executor_class=org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor -# The policy of extension SchemaRegionGroup for each Database. +# The policy of extending SchemaRegionGroups for each Database. # These policies are currently supported: -# 1. CUSTOM(Each Database will allocate schema_region_group_per_database RegionGroups as soon as created) -# 2. AUTO(Each Database will automatically extend SchemaRegionGroups based on the data it has) +# 1. CUSTOM: For known workloads that need manual resource allocation; creates the configured target +# number of SchemaRegionGroups per Database when schema partitions are first allocated. +# 2. AUTO: For most routine workloads (roughly 80% as an approximate planning guideline); gradually +# meets the configured minimum and expands with slot occupancy up to the resource-based maximum, +# balancing parallelism and RegionGroup management overhead. +# 3. PROACTIVE: Complements AUTO where it does not expand eagerly enough, such as very few devices with +# many measurements and a high load; creates groups earlier for more parallelism at the cost of +# additional RegionGroup management overhead. +# AUTO and PROACTIVE use the same per-Database maximum calculation, including resource sharing across +# Databases. PROACTIVE expands more eagerly within this limit; both have the same group count once +# they reach the same maximum. +# PROACTIVE targets one SchemaRegionGroup per distinct schema series slot, not per measurement, +# and grows toward the configured minimum incrementally like AUTO, without exceeding the maximum. +# If all SchemaRegionGroups are disabled, PROACTIVE may add one group above this target, +# without exceeding the maximum, to allow partition allocation to continue. +# Switching policies does not remove existing RegionGroups. # effectiveMode: hot_reload # Datatype: String -schema_region_group_extension_policy=AUTO +schema_region_group_extension_policy=PROACTIVE # When set schema_region_group_extension_policy=CUSTOM, # this parameter is the default number of SchemaRegionGroups for each Database. -# When set schema_region_group_extension_policy=AUTO, -# this parameter is the default minimal number of SchemaRegionGroups for each Database. +# When set schema_region_group_extension_policy=AUTO or PROACTIVE, +# this parameter is the default minimum number of SchemaRegionGroups and a lower bound for the per-Database maximum. +# Both policies grow toward this minimum incrementally, adding at most the number of pending series slots +# to satisfy the minimum. PROACTIVE also grows with the number of active series slots. # effectiveMode: hot_reload # Datatype: Integer default_schema_region_group_num_per_database=1 -# Only take effect when set schema_region_group_extension_policy=AUTO. -# This parameter is the maximum number of SchemaRegions expected to be managed by each DataNode. -# Notice: Since each Database requires at least one SchemaRegionGroup to manage its schema, -# this parameter doesn't limit the upper bound of cluster SchemaRegions when there are too many Databases. +# Only takes effect when schema_region_group_extension_policy=AUTO or PROACTIVE. +# The expected number of SchemaRegions per DataNode, used to calculate the same per-Database maximum +# for AUTO and PROACTIVE. This is not a hard limit on node or cluster totals: per-Database minimums, +# rounding, and existing groups can raise the total above this resource estimate. # effectiveMode: hot_reload # Datatype: Integer schema_region_per_data_node=1 -# The policy of extension DataRegionGroup for each Database. +# The policy of extending DataRegionGroups for each Database. # These policies are currently supported: -# 1. CUSTOM(Each Database will allocate data_region_group_per_database DataRegionGroups as soon as created) -# 2. AUTO(Each Database will automatically extend DataRegionGroups based on the data it has) +# 1. CUSTOM: For known workloads that need manual resource allocation; creates the configured target +# number of DataRegionGroups per Database when data partitions are first allocated. +# 2. AUTO: For most routine workloads (roughly 80% as an approximate planning guideline); gradually +# meets the configured minimum and expands with slot occupancy up to the resource-based maximum, +# balancing parallelism and RegionGroup management overhead. +# 3. PROACTIVE: Complements AUTO where it does not expand eagerly enough, such as very few devices with +# many measurements and a high load; creates groups earlier for more parallelism at the cost of +# additional RegionGroup management overhead. +# AUTO and PROACTIVE use the same per-Database maximum calculation, including resource sharing across +# Databases. PROACTIVE expands more eagerly within this limit; both have the same group count once +# they reach the same maximum. +# PROACTIVE targets one DataRegionGroup per distinct data series slot, not per measurement, +# and grows toward the configured minimum incrementally like AUTO, without exceeding the maximum. +# New time partitions in an existing slot can also trigger this incremental growth toward the minimum. +# If all DataRegionGroups are disabled, PROACTIVE may add one group above this target, +# without exceeding the maximum, to allow partition allocation to continue. +# PROACTIVE balances active slots in its allocation policy table so new time partitions can use new groups. +# Existing time-partition assignments are retained. +# Switching policies does not remove existing RegionGroups. # effectiveMode: hot_reload # Datatype: String -data_region_group_extension_policy=AUTO +data_region_group_extension_policy=PROACTIVE # When set data_region_group_extension_policy=CUSTOM, # this parameter is the default number of DataRegionGroups for each Database. -# When set data_region_group_extension_policy=AUTO, -# this parameter is the default minimal number of DataRegionGroups for each Database. +# When set data_region_group_extension_policy=AUTO or PROACTIVE, +# this parameter is the default minimum number of DataRegionGroups and a lower bound for the per-Database maximum. +# Both policies grow toward this minimum incrementally, adding at most the number of pending series slots +# to satisfy the minimum. PROACTIVE also grows with the number of active series slots. # effectiveMode: hot_reload # Datatype: Integer default_data_region_group_num_per_database=2 -# Only take effect when set data_region_group_extension_policy=AUTO. -# This parameter is the maximum number of DataRegions expected to be managed by each DataNode. -# Set to 0 means that each node automatically has the number of CPU cores / 2 regions -# Notice: Since each Database requires at least two DataRegionGroups to manage its data, -# this parameter doesn't limit the upper bound of cluster DataRegions when there are too many Databases. +# Only takes effect when data_region_group_extension_policy=AUTO or PROACTIVE. +# The expected number of DataRegions per DataNode, used to calculate the same per-Database maximum +# for AUTO and PROACTIVE. Set to 0 to use half the CPU core count as the expected number for each DataNode. +# This is not a hard limit on node or cluster totals: per-Database minimums, rounding, and existing +# groups can raise the total above this resource estimate. # effectiveMode: hot_reload # Datatype: Integer data_region_per_data_node=0 From 1637378c4fdcf85457b7c8b9f6a75236b4c28f60 Mon Sep 17 00:00:00 2001 From: Di Wang Date: Wed, 23 Sep 2026 12:24:50 +0800 Subject: [PATCH 2/2] Fix quota accounting and integration test ordering Fix setSpaceQuotaTest0 by counting each SchemaRegion once so valid quota increases are accepted. Fix setSpaceQuotaTest5 by summing device and timeseries usage without double-counting prior regions. Fix testShowDevicesWithLimit by checking two distinct valid devices without assuming their order. Fix testPartitionByAndOrderByWithNulls by ordering output by device, value, and time while preserving window semantics. Fix maxByMinByExtremeTest by asserting unique extrema exactly and accepting only valid tied candidates. Fix IoTDBTableViewQueryIT.test by sorting inputs before LINEAR and PREVIOUS FILL to compare identical row sequences. --- .../org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java | 12 +++++++++--- .../db/it/IoTDBMultiTAGsWithAttributesTableIT.java | 10 +++++++++- .../relational/it/db/it/IoTDBWindowFunctionIT.java | 2 +- .../it/query/view/recent/IoTDBTableViewQueryIT.java | 9 +++++---- .../confignode/manager/ClusterQuotaManager.java | 5 ++--- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java index 8c816f61fcda7..a9b0fbea52d9e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java @@ -1047,7 +1047,13 @@ public void testShowDevicesWithLimit() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { try { - List exps = Arrays.asList("root.sg1.d0,false", "root.sg1.d1,false"); + List exps = + new ArrayList<>( + Arrays.asList( + "root.sg1.d0,false", + "root.sg1.d1,false", + "root.sg1.d2,false", + "root.sg1.d3,false")); statement.execute("INSERT INTO root.sg1.d0(timestamp, s1) VALUES (5, 5)"); statement.execute("INSERT INTO root.sg1.d1(timestamp, s2) VALUES (5, 5)"); @@ -1057,8 +1063,8 @@ public void testShowDevicesWithLimit() throws SQLException { int count = 0; try (ResultSet resultSet = statement.executeQuery("show devices limit 2")) { while (resultSet.next()) { - Assert.assertEquals( - exps.get(count), resultSet.getString(1) + "," + resultSet.getString(2)); + String device = resultSet.getString(1) + "," + resultSet.getString(2); + Assert.assertTrue(device, exps.remove(device)); ++count; } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java index 7243ca5736657..1b293f75865c3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java @@ -1380,14 +1380,22 @@ public void maxByMinByExtremeTest() { tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); expectedHeader = buildHeaders(30); + // Exclude l4 so both blob extrema are unique; tied rows may arrive in either order. sql = - "select max_by(time,blob),max_by(device,blob),max_by(level,blob),max_by(attr1,blob),max_by(attr2,blob),max_by(num,blob),max_by(bignum,blob),max_by(floatnum,blob),max_by(str,blob),max_by(bool,blob),max_by(date,blob),max_by(blob,blob),max_by(ts,blob),max_by(stringv,blob),max_by(doubleNum,blob),min_by(time,blob),min_by(device,blob),min_by(level,blob),min_by(attr1,blob),min_by(attr2,blob),min_by(num,blob),min_by(bignum,blob),min_by(floatnum,blob),min_by(str,blob),min_by(bool,blob),min_by(date,blob),min_by(blob,blob),min_by(ts,blob),min_by(stringv,blob),min_by(doubleNum,blob) from table0"; + "select max_by(time,blob),max_by(device,blob),max_by(level,blob),max_by(attr1,blob),max_by(attr2,blob),max_by(num,blob),max_by(bignum,blob),max_by(floatnum,blob),max_by(str,blob),max_by(bool,blob),max_by(date,blob),max_by(blob,blob),max_by(ts,blob),max_by(stringv,blob),max_by(doubleNum,blob),min_by(time,blob),min_by(device,blob),min_by(level,blob),min_by(attr1,blob),min_by(attr2,blob),min_by(num,blob),min_by(bignum,blob),min_by(floatnum,blob),min_by(str,blob),min_by(bool,blob),min_by(date,blob),min_by(blob,blob),min_by(ts,blob),min_by(stringv,blob),min_by(doubleNum,blob) from table0 where level != 'l4'"; retArray = new String[] { "1971-01-01T00:00:10.000Z,d1,l5,null,null,7,2147983648,213.112,lemon,true,null,0x108dcd63,2024-09-25T06:15:35.000Z,null,null,1970-01-01T00:00:00.020Z,d1,l2,yy,zz,2,2147483648,434.12,pineapple,true,null,0x108dcd62,2024-09-24T06:15:35.000Z,null,6666.8,", }; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + // Retain the unfiltered ties: either associated value is valid, but the extrema are fixed. + expectedHeader = buildHeaders(4); + sql = + "select max_by(num,blob) in (7,9),min_by(num,blob) in (2,5),max_by(blob,blob),min_by(blob,blob) from table0"; + retArray = new String[] {"true,true,0x108dcd63,0x108dcd62,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = buildHeaders(3); sql = "select extreme(num),extreme(bignum),extreme(floatnum) from table0"; retArray = new String[] {"15,3147483648,4654.231,"}; diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java index ccc267e1ccd5f..d3dc0323ee819 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java @@ -259,7 +259,7 @@ public void testPartitionByAndOrderByWithNulls() { "2021-01-01T09:20:00.000Z,null,null,2,", }; tableResultSetEqualTest( - "SELECT *, count(value) OVER (PARTITION BY device ORDER BY value) AS cnt FROM demo2 ORDER BY device", + "SELECT *, count(value) OVER (PARTITION BY device ORDER BY value) AS cnt FROM demo2 ORDER BY device,value,time", expectedHeader, retArray, DATABASE_NAME); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java index 3dc7386a71ae1..1d7933b30292c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java @@ -175,15 +175,16 @@ public void test() throws Exception { "select current from view1 where time > 1", "select current from table1 where time > 1" + (aligned ? "" : " and current is not null"), true); + // FILL runs before an outer ORDER BY, so give both sources the same ordered input. compareQueryResults( session, - "select * from view1 fill method linear", - "select * from table1 fill method linear", + "select * from (select * from view1 order by time, battery) fill method linear", + "select * from (select * from table1 order by time, battery) fill method linear", true); compareQueryResults( session, - "select * from view1 fill method previous", - "select * from table1 fill method previous", + "select * from (select * from view1 order by time, battery) fill method previous", + "select * from (select * from table1 order by time, battery) fill method previous", true); compareQueryResults( session, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterQuotaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterQuotaManager.java index d53890f75c341..7af050dacd4f8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterQuotaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterQuotaManager.java @@ -272,11 +272,10 @@ public void updateSpaceQuotaUsage() { .forEach( schemaRegionId -> { if (deviceNum.containsKey(schemaRegionId)) { - deviceCount.addAndGet(deviceCount.get() + deviceNum.get(schemaRegionId)); + deviceCount.addAndGet(deviceNum.get(schemaRegionId)); } if (timeSeriesNum.containsKey(schemaRegionId)) { - timeSeriesCount.addAndGet( - timeSeriesCount.get() + timeSeriesNum.get(schemaRegionId)); + timeSeriesCount.addAndGet(timeSeriesNum.get(schemaRegionId)); } }); quotaInfo.getSpaceQuotaUsage().get(entry.getKey()).setDeviceNum(deviceCount.get());