From c777b85bc8a8cb928dba405a4b684eebd819e8b9 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 16 May 2026 19:53:24 +0800 Subject: [PATCH] [fix](fe) Fix stale timestamp in CatalogRecycleBin erase daemon --- .../doris/catalog/CatalogRecycleBin.java | 31 +- .../doris/catalog/CatalogRecycleBinTest.java | 507 +++++++++++++++++- 2 files changed, 508 insertions(+), 30 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java index 9ad1594bc4c0a2..8fbef7bd814dab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java @@ -34,6 +34,7 @@ import org.apache.doris.qe.GlobalVariable; import org.apache.doris.thrift.TStorageMedium; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -56,6 +57,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.LongSupplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -67,6 +69,14 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable { private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + // Injectable clock source, defaults to system clock + private LongSupplier clock = System::currentTimeMillis; + + @VisibleForTesting + public void setClock(LongSupplier clock) { + this.clock = clock; + } + private void readLock() { lock.readLock().lock(); } @@ -189,7 +199,7 @@ public boolean recycleDatabase(Database db, Set tableNames, Set ta // The 'force drop' database should be recycle immediately. recycleTime = 0; } else if (!isReplay || replayRecycleTime == 0) { - recycleTime = System.currentTimeMillis(); + recycleTime = clock.getAsLong(); } else { recycleTime = replayRecycleTime; } @@ -218,7 +228,7 @@ public boolean recycleTable(long dbId, Table table, boolean isReplay, // The 'force drop' table should be recycle immediately. recycleTime = 0; } else if (!isReplay || replayRecycleTime == 0) { - recycleTime = System.currentTimeMillis(); + recycleTime = clock.getAsLong(); } else { recycleTime = replayRecycleTime; } @@ -247,7 +257,7 @@ public boolean recyclePartition(long dbId, long tableId, String tableName, Parti // recycle partition RecyclePartitionInfo partitionInfo = new RecyclePartitionInfo(dbId, tableId, partition, range, listPartitionItem, dataProperty, replicaAlloc, isInMemory, isMutable); - idToRecycleTime.put(partition.getId(), System.currentTimeMillis()); + idToRecycleTime.put(partition.getId(), clock.getAsLong()); idToPartition.put(partition.getId(), partitionInfo); dbTblIdPartitionNameToIds.computeIfAbsent(Pair.of(dbId, tableId), k -> new ConcurrentHashMap<>()) .computeIfAbsent(partition.getName(), k -> ConcurrentHashMap.newKeySet()).add(partition.getId()); @@ -291,7 +301,7 @@ private boolean isExpire(long id, long currentTimeMs) { && latency > Config.catalog_trash_expire_second * 1000L; } - private void eraseDatabase(long currentTimeMs, int keepNum) { + protected void eraseDatabase(long currentTimeMs, int keepNum) { int eraseNum = 0; StopWatch watch = StopWatch.createStarted(); try { @@ -442,7 +452,7 @@ public void replayEraseDatabase(long dbId) { } } - private void eraseTable(long currentTimeMs, int keepNum) { + protected void eraseTable(long currentTimeMs, int keepNum) { int eraseNum = 0; StopWatch watch = StopWatch.createStarted(); try { @@ -574,7 +584,7 @@ public void replayEraseTable(long tableId) { } } - private void erasePartition(long currentTimeMs, int keepNum) { + protected void erasePartition(long currentTimeMs, int keepNum) { int eraseNum = 0; StopWatch watch = StopWatch.createStarted(); try { @@ -1393,13 +1403,14 @@ public void addTabletToInvertedIndex() { @Override protected void runAfterCatalogReady() { - long currentTimeMs = System.currentTimeMillis(); // should follow the partition/table/db order // in case of partition(table) is still in recycle bin but table(db) is missing + // Each erase method gets its own currentTimeMs to avoid using a stale timestamp, + // since previous erase operations may take significant time. int keepNum = Config.max_same_name_catalog_trash_num; - erasePartition(currentTimeMs, keepNum); - eraseTable(currentTimeMs, keepNum); - eraseDatabase(currentTimeMs, keepNum); + erasePartition(clock.getAsLong(), keepNum); + eraseTable(clock.getAsLong(), keepNum); + eraseDatabase(clock.getAsLong(), keepNum); } public List> getInfo() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogRecycleBinTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogRecycleBinTest.java index 99831c65578064..3c018eb5af01e9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogRecycleBinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogRecycleBinTest.java @@ -28,6 +28,7 @@ import org.apache.doris.common.util.URI; import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder; import org.apache.doris.thrift.TStorageMedium; +import org.apache.doris.thrift.TStorageType; import org.apache.doris.utframe.UtFrameUtils; import com.google.common.collect.Lists; @@ -56,6 +57,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.LongSupplier; import java.util.stream.Collectors; public class CatalogRecycleBinTest { @@ -93,7 +95,7 @@ public void testRecycleNotEmptyDatabase() { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Set tableNames = Sets.newHashSet(CatalogTestUtil.testTable1); Set tableIds = Sets.newHashSet(CatalogTestUtil.testTableId1); @@ -189,7 +191,8 @@ public void testReadRecycleBinDatabaseDoesNotRegisterNereidsFunctions() throws E URI.create("file:///tmp/recycled_py_udf.py"), "evaluate", null, - null); + null + ); function.setRuntimeVersion("3.8"); function.setFunctionCode("def evaluate(x):\n return x + 1\n"); function.setNullableMode(NullableMode.ALWAYS_NULLABLE); @@ -225,7 +228,7 @@ public void testRecycleTable() { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Optional table = db.getTable(CatalogTestUtil.testTableId1); Assert.assertTrue(table.isPresent()); @@ -250,7 +253,7 @@ public void testForceDropTable() { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Optional
table = db.getTable(CatalogTestUtil.testTableId1); Assert.assertTrue(table.isPresent()); @@ -282,7 +285,7 @@ public void testRecyclePartition() { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Optional
table = db.getTable(CatalogTestUtil.testTableId1); Assert.assertTrue(table.isPresent()); @@ -302,7 +305,7 @@ public void testRecyclePartition() { new ReplicaAllocation((short) 3), false, false - ); + ); Assert.assertTrue(result); Assert.assertTrue(recycleBin.isRecyclePartition(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1, CatalogTestUtil.testPartitionId1)); @@ -317,7 +320,7 @@ public void testRecyclePartition() { new ReplicaAllocation((short) 3), false, false - ); + ); // test recycling same partition again should fail Assert.assertFalse(result); } @@ -370,7 +373,7 @@ public void testRecycleSameNamePartition() { new ReplicaAllocation((short) 3), false, false - ); + ); Assert.assertTrue(result); Assert.assertTrue(recycleBin.isRecyclePartition(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1, recyclePartitionNum)); @@ -416,7 +419,7 @@ public void testRecoverDatabaseWithTable() throws Exception { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Set tableNames = db.getTableNames(); Set tableIds = Sets.newHashSet(db.getTableIds()); @@ -447,7 +450,7 @@ public void testRecoverTable() throws Exception { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Optional
table = db.getTable(CatalogTestUtil.testTableId1); Assert.assertTrue(table.isPresent()); @@ -471,7 +474,7 @@ public void testRecoverPartition() throws Exception { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Optional
table = db.getTable(CatalogTestUtil.testTableId1); Assert.assertTrue(table.isPresent()); @@ -509,7 +512,7 @@ public void testGetRecycleIds() { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Optional
table = db.getTable(CatalogTestUtil.testTableId1); Assert.assertTrue(table.isPresent()); @@ -555,7 +558,7 @@ public void testAllTabletsInRecycledStatus() { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Optional
table = db.getTable(CatalogTestUtil.testTableId1); Assert.assertTrue(table.isPresent()); @@ -604,7 +607,7 @@ public void testEraseTableInstantly() throws Exception { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Optional
table = db.getTable(CatalogTestUtil.testTableId1); Assert.assertTrue(table.isPresent()); @@ -629,7 +632,7 @@ public void testErasePartitionInstantly() throws Exception { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Optional
table = db.getTable(CatalogTestUtil.testTableId1); Assert.assertTrue(table.isPresent()); @@ -668,7 +671,7 @@ public void testReplayOperations() { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Set tableNames = Sets.newHashSet(db.getTableNames()); Set tableIds = Sets.newHashSet(db.getTableIds()); @@ -720,7 +723,7 @@ public void testGetInfo() { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Set tableNames = Sets.newHashSet(db.getTableNames()); Set tableIds = Sets.newHashSet(db.getTableIds()); @@ -769,7 +772,7 @@ public void testGetDbToRecycleSize() { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Set tableNames = Sets.newHashSet(db.getTableNames()); Set tableIds = Sets.newHashSet(db.getTableIds()); @@ -821,7 +824,7 @@ public void testRecoverNonExistentPartition() throws Exception { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Optional
table = db.getTable(CatalogTestUtil.testTableId1); Assert.assertTrue(table.isPresent()); @@ -843,7 +846,7 @@ public void testAddTabletToInvertedIndex() { CatalogTestUtil.testIndexId1, CatalogTestUtil.testTabletId1, CatalogTestUtil.testStartVersion - ); + ); Optional
table = db.getTable(CatalogTestUtil.testTableId1); Assert.assertTrue(table.isPresent()); @@ -1058,4 +1061,468 @@ public void testMicrobatchEraseReleasesLockBetweenItems() throws Exception { CatalogTestUtil.testTableId1, 9000)); } } + + private static class ControllableClock implements LongSupplier { + private long currentTime; + private final long initialTime; + + public ControllableClock(long initialTime) { + this.currentTime = initialTime; + this.initialTime = initialTime; + } + + @Override + public long getAsLong() { + return currentTime; + } + + public void advance(long millis) { + this.currentTime += millis; + } + + public void backoff(long millis) { + this.currentTime -= millis; + } + + public void reset() { + this.currentTime = this.initialTime; + } + } + + /** + * Test old code bug: Partition timeout causes table/db cleanup delay. + * Partition: expired (recycled long ago) + * Table/Db: NOT expired at startTime, but become expired after erasePartition takes time + * Old code uses stale startTime, so table/db are NOT cleaned. + */ + @Test + public void testOldCodePartitionTimeoutCausesTableDbDelay() { + long origExpireSecond = Config.catalog_trash_expire_second; + boolean origIgnoreMinErase = Config.catalog_trash_ignore_min_erase_latency; + try { + Config.catalog_trash_expire_second = 1; + Config.catalog_trash_ignore_min_erase_latency = true; + + final long baseId = System.nanoTime(); + final long testDbId = baseId + 1; + final long testTableId = baseId + 2; + final long testPartitionId = baseId + 3; + final long testIndexId = baseId + 4; + final long testTabletId = baseId + 5; + + long startTime = System.currentTimeMillis(); + long expireMs = Config.catalog_trash_expire_second * 1000L; // 1000ms + long slowOperationMs = expireMs + 100; // 1100ms + + ControllableClock clock = new ControllableClock(startTime); + + class BuggyRecycleBin extends CatalogRecycleBin { + BuggyRecycleBin() { + setClock(clock); + } + + @Override + public void runAfterCatalogReady() { + long sharedTime = clock.getAsLong(); + int keepNum = Config.max_same_name_catalog_trash_num; + erasePartition(sharedTime, keepNum); + eraseTable(sharedTime, keepNum); + eraseDatabase(sharedTime, keepNum); + } + + @Override + protected void erasePartition(long currentTimeMs, int keepNum) { + clock.advance(slowOperationMs); + super.erasePartition(currentTimeMs, keepNum); + } + } + + BuggyRecycleBin recycleBin = new BuggyRecycleBin(); + + Database db = createSimpleTestDatabase( + testDbId, testTableId, testPartitionId, + testIndexId, testTabletId, + CatalogTestUtil.testStartVersion); + + OlapTable table = (OlapTable) db.getTable(testTableId).get(); + Partition partition = table.getPartition(testPartitionId); + + Set tableNames = Sets.newHashSet(); + Set tableIds = Sets.newHashSet(); + for (Table tbl : db.getTables()) { + tableNames.add(tbl.getName()); + tableIds.add(tbl.getId()); + } + + // Partition: recycleTime = startTime - 1100 (expired at startTime) + clock.backoff(expireMs + 100); + recycleBin.recyclePartition(testDbId, testTableId, table.getName(), partition, null, null, + new DataProperty(TStorageMedium.HDD), new ReplicaAllocation((short) 3), false, false); + clock.reset(); + + // Table: recycleTime = startTime - 200 (NOT expired at startTime) + clock.backoff(200); + for (Table tbl : db.getTables()) { + db.unregisterTable(tbl.getId()); + recycleBin.recycleTable(testDbId, tbl, false, false, 0); + } + clock.reset(); + + // Db: recycleTime = startTime - 100 (NOT expired at startTime) + clock.backoff(100); + recycleBin.recycleDatabase(db, tableNames, tableIds, false, false, 0); + clock.reset(); + + // Run cleanup + // Timeline: + // 1. erasePartition starts: sharedTime = startTime + // Partition: expired (latency = 1100 > 1000) -> cleaned + // During erasePartition, clock advances to startTime+1100 + // 2. eraseTable uses sharedTime = startTime (stale!) + // Table recycleTime = startTime-200, latency = 200 < 1000 -> NOT expired (bug!) + // 3. eraseDatabase similarly NOT cleaned + recycleBin.runAfterCatalogReady(); + + // Verify bug: only partition cleaned, table and db remain + Assert.assertNull(recycleBin.getRecycleTimeById(testPartitionId)); + Assert.assertNotNull(recycleBin.getRecycleTimeById(testTableId)); + Assert.assertNotNull(recycleBin.getRecycleTimeById(testDbId)); + + } finally { + Config.catalog_trash_expire_second = origExpireSecond; + Config.catalog_trash_ignore_min_erase_latency = origIgnoreMinErase; + } + } + + /** + * Test old code bug: Partition and table timeout cause db cleanup delay. + * Partition/Table: expired at startTime + * Db: NOT expired at startTime, but become expired after erasePartition+eraseTable take time + * Old code uses stale startTime, so db is NOT cleaned. + */ + @Test + public void testOldCodePartitionAndTableTimeoutCauseDbDelay() { + long origExpireSecond = Config.catalog_trash_expire_second; + boolean origIgnoreMinErase = Config.catalog_trash_ignore_min_erase_latency; + try { + Config.catalog_trash_expire_second = 1; + Config.catalog_trash_ignore_min_erase_latency = true; + + final long baseId = System.nanoTime(); + final long testDbId = baseId + 1; + final long testTableId = baseId + 2; + final long testPartitionId = baseId + 3; + final long testIndexId = baseId + 4; + final long testTabletId = baseId + 5; + + long startTime = System.currentTimeMillis(); + long expireMs = Config.catalog_trash_expire_second * 1000L; // 1000ms + long slowOperationMs = expireMs + 100; // 1100ms + + ControllableClock clock = new ControllableClock(startTime); + + class BuggyRecycleBin extends CatalogRecycleBin { + BuggyRecycleBin() { + setClock(clock); + } + + @Override + public void runAfterCatalogReady() { + long sharedTime = clock.getAsLong(); + int keepNum = Config.max_same_name_catalog_trash_num; + erasePartition(sharedTime, keepNum); + eraseTable(sharedTime, keepNum); + eraseDatabase(sharedTime, keepNum); + } + + @Override + protected void erasePartition(long currentTimeMs, int keepNum) { + clock.advance(slowOperationMs / 2); + super.erasePartition(currentTimeMs, keepNum); + } + + @Override + protected void eraseTable(long currentTimeMs, int keepNum) { + clock.advance(slowOperationMs / 2); + super.eraseTable(currentTimeMs, keepNum); + } + } + + BuggyRecycleBin recycleBin = new BuggyRecycleBin(); + + Database db = createSimpleTestDatabase( + testDbId, testTableId, testPartitionId, + testIndexId, testTabletId, + CatalogTestUtil.testStartVersion); + + OlapTable table = (OlapTable) db.getTable(testTableId).get(); + Partition partition = table.getPartition(testPartitionId); + + Set tableNames = Sets.newHashSet(); + Set tableIds = Sets.newHashSet(); + for (Table tbl : db.getTables()) { + tableNames.add(tbl.getName()); + tableIds.add(tbl.getId()); + } + + // Partition: recycleTime = startTime - 1100 (expired at startTime) + clock.backoff(expireMs + 100); + recycleBin.recyclePartition(testDbId, testTableId, table.getName(), partition, null, null, + new DataProperty(TStorageMedium.HDD), new ReplicaAllocation((short) 3), false, false); + clock.reset(); + + // Table: recycleTime = startTime - 1050 (expired at startTime) + clock.backoff(expireMs + 50); + for (Table tbl : db.getTables()) { + db.unregisterTable(tbl.getId()); + recycleBin.recycleTable(testDbId, tbl, false, false, 0); + } + clock.reset(); + + // Db: recycleTime = startTime - 200 (NOT expired at startTime) + clock.backoff(200); + recycleBin.recycleDatabase(db, tableNames, tableIds, false, false, 0); + clock.reset(); + + // Run cleanup + // Timeline: + // 1. erasePartition: sharedTime=startTime, clock advances to startTime+550 + // 2. eraseTable: sharedTime=startTime (stale!), clock advances to startTime+1100 + // 3. eraseDatabase: sharedTime=startTime (stale!) + // Db recycleTime=startTime-200, latency=200<1000 -> NOT cleaned (bug!) + recycleBin.runAfterCatalogReady(); + + // Verify bug: partition and table cleaned, db remains + Assert.assertNull(recycleBin.getRecycleTimeById(testPartitionId)); + Assert.assertNull(recycleBin.getRecycleTimeById(testTableId)); + Assert.assertNotNull(recycleBin.getRecycleTimeById(testDbId)); + + } finally { + Config.catalog_trash_expire_second = origExpireSecond; + Config.catalog_trash_ignore_min_erase_latency = origIgnoreMinErase; + } + } + + /** + * Test new code fix: All objects (partition, table, db) are expired. + * Even with erasePartition taking long time, new code correctly cleans all expired objects. + * Expected: All objects are cleaned up (null). + */ + @Test + public void testNewCodePartitionTimeoutTableDbWork() { + long origExpireSecond = Config.catalog_trash_expire_second; + boolean origIgnoreMinErase = Config.catalog_trash_ignore_min_erase_latency; + try { + Config.catalog_trash_expire_second = 1; + Config.catalog_trash_ignore_min_erase_latency = true; + + final long baseId = System.nanoTime(); + final long testDbId = baseId + 1; + final long testTableId = baseId + 2; + final long testPartitionId = baseId + 3; + final long testIndexId = baseId + 4; + final long testTabletId = baseId + 5; + + long currentTimeMillis = System.currentTimeMillis(); + long expireMs = Config.catalog_trash_expire_second * 1000L; + long offset = expireMs + 100; + long advanceOffset = expireMs + 100; + + ControllableClock clock = new ControllableClock(currentTimeMillis); + + // Fixed: independent timestamps, partition advances to simulate processing delay + class FixedRecycleBin extends CatalogRecycleBin { + FixedRecycleBin() { + setClock(clock); + } + + @Override + protected void erasePartition(long currentTimeMs, int keepNum) { + clock.advance(advanceOffset); + super.erasePartition(currentTimeMs, keepNum); + } + } + + FixedRecycleBin recycleBin = new FixedRecycleBin(); + + Database db = createSimpleTestDatabase( + testDbId, testTableId, testPartitionId, + testIndexId, testTabletId, + CatalogTestUtil.testStartVersion); + + OlapTable table = (OlapTable) db.getTable(testTableId).get(); + Partition partition = table.getPartition(testPartitionId); + + Set tableNames = Sets.newHashSet(); + Set tableIds = Sets.newHashSet(); + for (Table tbl : db.getTables()) { + tableNames.add(tbl.getName()); + tableIds.add(tbl.getId()); + } + + // Partition dropped first: recycleTime = currentTimeMillis - offset + clock.backoff(offset); + recycleBin.recyclePartition(testDbId, testTableId, table.getName(), partition, null, null, + new DataProperty(TStorageMedium.HDD), new ReplicaAllocation((short) 3), false, false); + + // Table and db dropped later: recycleTime = currentTimeMillis + clock.advance(offset); + for (Table tbl : db.getTables()) { + db.unregisterTable(tbl.getId()); + recycleBin.recycleTable(testDbId, tbl, false, false, 0); + } + recycleBin.recycleDatabase(db, tableNames, tableIds, false, false, 0); + + // erasePartition gets currentTimeMillis, then advances clock + // eraseTable gets currentTimeMillis + advanceOffset + // eraseDatabase gets currentTimeMillis + advanceOffset + recycleBin.runAfterCatalogReady(); + + Assert.assertNull(recycleBin.getRecycleTimeById(testPartitionId)); + Assert.assertNull(recycleBin.getRecycleTimeById(testTableId)); + Assert.assertNull(recycleBin.getRecycleTimeById(testDbId)); + + } finally { + Config.catalog_trash_expire_second = origExpireSecond; + Config.catalog_trash_ignore_min_erase_latency = origIgnoreMinErase; + } + } + + /** + * Test new code fix: All objects (partition, table, db) are expired. + * Even with erasePartition and eraseTable taking long time, new code correctly cleans all expired objects. + * Expected: All objects are cleaned up (null). + */ + @Test + public void testNewCodePartitionAndTableTimeoutDbWork() { + long origExpireSecond = Config.catalog_trash_expire_second; + boolean origIgnoreMinErase = Config.catalog_trash_ignore_min_erase_latency; + try { + Config.catalog_trash_expire_second = 1; + Config.catalog_trash_ignore_min_erase_latency = true; + + final long baseId = System.nanoTime(); + final long testDbId = baseId + 1; + final long testTableId = baseId + 2; + final long testPartitionId = baseId + 3; + final long testIndexId = baseId + 4; + final long testTabletId = baseId + 5; + + long currentTimeMillis = System.currentTimeMillis(); + long expireMs = Config.catalog_trash_expire_second * 1000L; + long offset = expireMs + 100; + long advanceOffset = expireMs + 100; + + ControllableClock clock = new ControllableClock(currentTimeMillis); + + // Fixed: independent timestamps, both partition and table advance + class FixedRecycleBin extends CatalogRecycleBin { + FixedRecycleBin() { + setClock(clock); + } + + @Override + protected void erasePartition(long currentTimeMs, int keepNum) { + clock.advance(advanceOffset / 2); + super.erasePartition(currentTimeMs, keepNum); + } + + @Override + protected void eraseTable(long currentTimeMs, int keepNum) { + clock.advance(advanceOffset / 2); + super.eraseTable(currentTimeMs, keepNum); + } + } + + FixedRecycleBin recycleBin = new FixedRecycleBin(); + + Database db = createSimpleTestDatabase( + testDbId, testTableId, testPartitionId, + testIndexId, testTabletId, + CatalogTestUtil.testStartVersion); + + OlapTable table = (OlapTable) db.getTable(testTableId).get(); + Partition partition = table.getPartition(testPartitionId); + + Set tableNames = Sets.newHashSet(); + Set tableIds = Sets.newHashSet(); + for (Table tbl : db.getTables()) { + tableNames.add(tbl.getName()); + tableIds.add(tbl.getId()); + } + + // Partition and table dropped first: recycleTime = currentTimeMillis - offset + clock.backoff(offset); + recycleBin.recyclePartition(testDbId, testTableId, table.getName(), partition, null, null, + new DataProperty(TStorageMedium.HDD), new ReplicaAllocation((short) 3), false, false); + for (Table tbl : db.getTables()) { + db.unregisterTable(tbl.getId()); + recycleBin.recycleTable(testDbId, tbl, false, false, 0); + } + + // Db dropped later: recycleTime = currentTimeMillis + clock.advance(offset); + recycleBin.recycleDatabase(db, tableNames, tableIds, false, false, 0); + + recycleBin.runAfterCatalogReady(); + + Assert.assertNull(recycleBin.getRecycleTimeById(testPartitionId)); + Assert.assertNull(recycleBin.getRecycleTimeById(testTableId)); + Assert.assertNull(recycleBin.getRecycleTimeById(testDbId)); + + } finally { + Config.catalog_trash_expire_second = origExpireSecond; + Config.catalog_trash_ignore_min_erase_latency = origIgnoreMinErase; + } + } + + private Database createSimpleTestDatabase(long dbId, long tableId, long partitionId, + long indexId, long tabletId, long startVersion) { + List columns = new ArrayList<>(); + Column k1 = new Column("k1", PrimitiveType.INT); + k1.setIsKey(true); + columns.add(k1); + Column k2 = new Column("k2", PrimitiveType.INT); + k2.setIsKey(true); + columns.add(k2); + Column v = new Column("v", ScalarType.createType(PrimitiveType.DOUBLE), false, AggregateType.SUM, "0", ""); + columns.add(v); + + List keysColumn = new ArrayList<>(); + keysColumn.add(new Column("k1", PrimitiveType.INT)); + keysColumn.add(new Column("k2", PrimitiveType.INT)); + HashDistributionInfo distributionInfo = new HashDistributionInfo(10, keysColumn); + + Tablet tablet = new LocalTablet(tabletId); + for (int i = 0; i < 3; i++) { + long replicaId = tabletId * 100 + i; + Replica replica = new LocalReplica(replicaId, 100 + i, startVersion, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + tablet.addReplica(replica, true); + } + + MaterializedIndex index = new MaterializedIndex(indexId, IndexState.NORMAL); + TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, 0, TStorageMedium.HDD); + index.addTablet(tablet, tabletMeta); + + Partition partition = new Partition(partitionId, "p_" + partitionId, index, distributionInfo); + partition.updateVisibleVersion(startVersion); + partition.setNextVersion(startVersion + 1); + + PartitionInfo partitionInfo = new SinglePartitionInfo(); + partitionInfo.setDataProperty(partitionId, new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM)); + partitionInfo.setReplicaAllocation(partitionId, new ReplicaAllocation((short) 3)); + + OlapTable table = new OlapTable(tableId, "t_" + tableId, columns, KeysType.AGG_KEYS, + partitionInfo, distributionInfo); + table.addPartition(partition); + table.setIndexMeta(indexId, "idx_" + indexId, columns, 0, 0, (short) 1, + TStorageType.COLUMN, KeysType.AGG_KEYS); + table.setBaseIndexId(indexId); + + Database db = new Database(dbId, "db_" + dbId); + db.registerTable(table); + + return db; + } }