diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/CompactionCleaner.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/CompactionCleaner.java index fd49bbb90f13..5e13f45bcf26 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/CompactionCleaner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/CompactionCleaner.java @@ -198,6 +198,7 @@ private void cleanUsingLocation(CompactionInfo ci, String path, boolean requires deleted = fsRemover.clean(getCleaningRequestBasedOnLocation(ci, path)); } if (!deleted.isEmpty()) { + ci.setSoftDelete(true); txnHandler.markCleaned(ci); } else { txnHandler.clearCleanerStart(ci); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfo.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfo.java index e1c1f492bab8..5c392477c6a0 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfo.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfo.java @@ -87,6 +87,7 @@ public class CompactionInfo implements Comparable { private String fullPartitionName = null; private String fullTableName = null; private StringableMap propertiesMap; + private boolean softDelete; public CompactionInfo(String dbname, String tableName, String partName, CompactionType type) { this.dbname = dbname; @@ -190,6 +191,7 @@ public String toString() { .append("numberOfBuckets", numberOfBuckets) .append("orderByClause", orderByClause) .append("minOpenWriteTxnId", minOpenWriteTxnId) + .append("softDelete", softDelete) .build(); } @@ -368,4 +370,12 @@ public void setWriteIds(boolean hasUncompactedAborts, Set writeIds) { public boolean isAbortedTxnCleanup() { return type == CompactionType.ABORT_TXN_CLEANUP; } + + public void setSoftDelete(boolean softDelete) { + this.softDelete = softDelete; + } + + public boolean isSoftDelete() { + return softDelete; + } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/MarkCleanedFunction.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/MarkCleanedFunction.java index 2bd96f53a1da..27b67d1feaeb 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/MarkCleanedFunction.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/MarkCleanedFunction.java @@ -50,6 +50,12 @@ public MarkCleanedFunction(CompactionInfo info) { public Void execute(MultiDataSourceJdbcResource jdbcResource) throws MetaException { NamedParameterJdbcTemplate jdbcTemplate = jdbcResource.getJdbcTemplate(); MapSqlParameterSource param; + if (info.isSoftDelete()) { + // Remove compaction queue record and return + removeCompactionAndAbortRetryEntries(info, jdbcTemplate); + return null; + } + if (!info.isAbortedTxnCleanup()) { param = new MapSqlParameterSource() .addValue("id", info.id)