From 688554a23bf1fec44e09447f1e1198c867771517 Mon Sep 17 00:00:00 2001 From: Matt Byrd Date: Thu, 10 Aug 2023 16:25:52 +0100 Subject: [PATCH] Uncompressed size is being used for compressed tables in maintenance operations use compression ratio to scale the amount of remaining disk usage CompactionInfo::estimatedRemainingWriteBytes is uncompressed, causing compaction to be rejected due to size when it has space to complete patch by Matt Byrd; reviewed by Dmitry Konstantinov, Marcus Eriksson for CASSANDRA-21245 Co-authored-by: Jon Meredith --- .../cassandra/cache/AutoSavingCache.java | 4 +- .../db/compaction/ActiveCompactions.java | 5 ++- .../db/compaction/CompactionInfo.java | 41 ++++++++++++------- .../db/compaction/CompactionIterator.java | 7 ++++ .../db/compaction/CompactionTask.java | 2 +- .../cassandra/db/view/ViewBuilderTask.java | 4 +- .../db/virtual/SSTableTasksTable.java | 4 ++ .../internal/CollatedViewIndexBuilder.java | 4 +- .../sai/StorageAttachedIndexBuilder.java | 1 + .../index/sasi/SASIIndexBuilder.java | 1 + .../sstable/format/SortedTableScrubber.java | 1 + .../sstable/format/SortedTableVerifier.java | 1 + .../IndexSummaryRedistribution.java | 2 +- .../cassandra/metrics/CompactionMetrics.java | 3 ++ .../cassandra/streaming/StreamSession.java | 2 +- .../org/apache/cassandra/tools/NodeProbe.java | 1 + .../tools/nodetool/CompactionStats.java | 25 +++++++++-- .../test/CompactionDiskSpaceTest.java | 4 +- .../test/SecondaryIndexCompactionTest.java | 6 +-- .../test/StreamsDiskSpaceTest.java | 4 +- .../db/compaction/CompactionInfoTest.java | 4 +- .../db/compaction/CompactionsCQLTest.java | 31 +++++++++++--- .../db/repair/PendingAntiCompactionTest.java | 6 +-- .../db/virtual/SSTableTasksTableTest.java | 5 ++- .../indexsummary/IndexSummaryManagerTest.java | 2 +- .../tools/nodetool/CompactionStatsTest.java | 39 ++++++++++-------- 26 files changed, 144 insertions(+), 65 deletions(-) diff --git a/src/java/org/apache/cassandra/cache/AutoSavingCache.java b/src/java/org/apache/cassandra/cache/AutoSavingCache.java index 3826eb412e0..f1df894d454 100644 --- a/src/java/org/apache/cassandra/cache/AutoSavingCache.java +++ b/src/java/org/apache/cassandra/cache/AutoSavingCache.java @@ -327,6 +327,7 @@ else if (cacheType == CacheService.CacheType.COUNTER_CACHE) type, 0, keysEstimate, + keysEstimate, Unit.KEYS, nextTimeUUID(), getCacheDataPath(CURRENT_VERSION).toPath().toString()); @@ -341,7 +342,8 @@ public CompactionInfo getCompactionInfo() { // keyset can change in size, thus total can too // TODO need to check for this one... was: info.forProgress(keysWritten, Math.max(keysWritten, keys.size())); - return info.forProgress(keysWritten, Math.max(keysWritten, keysEstimate)); + long totalKeys = Math.max(keysWritten, keysEstimate); + return info.forProgress(keysWritten, totalKeys, totalKeys); } public void saveCache() diff --git a/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java b/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java index 4e238ad95d4..ab5bb00e170 100644 --- a/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java +++ b/src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java @@ -49,13 +49,14 @@ public void finishCompaction(CompactionInfo.Holder ci) { compactions.remove(ci); CompactionManager.instance.getMetrics().bytesCompacted.inc(ci.getCompactionInfo().getTotal()); + CompactionManager.instance.getMetrics().compressedBytesCompacted.inc(ci.getCompactionInfo().getTotalCompressed()); CompactionManager.instance.getMetrics().totalCompactionsCompleted.mark(); } /** * Get the estimated number of bytes remaining to write per sstable directory */ - public Map estimatedRemainingWriteBytes() + public Map estimatedRemainingWriteToDiskBytes() { synchronized (compactions) { @@ -66,7 +67,7 @@ public Map estimatedRemainingWriteBytes() List directories = compactionInfo.getTargetDirectories(); if (directories == null || directories.isEmpty()) continue; - long remainingWriteBytesPerDataDir = compactionInfo.estimatedRemainingWriteBytes() / directories.size(); + long remainingWriteBytesPerDataDir = compactionInfo.estimatedRemainingWriteToDiskBytes() / directories.size(); for (File directory : directories) writeBytesPerSSTableDir.merge(directory, remainingWriteBytesPerDataDir, Long::sum); } diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java b/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java index 0bfc925a7d0..73dc06911a4 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java @@ -42,6 +42,7 @@ public final class CompactionInfo public static final String COLUMNFAMILY = "columnfamily"; public static final String COMPLETED = "completed"; public static final String TOTAL = "total"; + public static final String TOTAL_COMPRESSED = "totalCompressed"; public static final String TASK_TYPE = "taskType"; public static final String UNIT = "unit"; public static final String COMPACTION_ID = "compactionId"; @@ -52,16 +53,18 @@ public final class CompactionInfo private final OperationType tasktype; private final long completed; private final long total; + private final long totalCompressed; private final Unit unit; private final TimeUUID compactionId; private final ImmutableSet sstables; private final String targetDirectory; - public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId, Collection sstables, String targetDirectory) + public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, Unit unit, TimeUUID compactionId, Collection sstables, String targetDirectory) { this.tasktype = tasktype; this.completed = completed; this.total = total; + this.totalCompressed = totalCompressed; this.metadata = metadata; this.unit = unit; this.compactionId = compactionId; @@ -69,38 +72,38 @@ public CompactionInfo(TableMetadata metadata, OperationType tasktype, long compl this.targetDirectory = targetDirectory; } - public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, TimeUUID compactionId, Collection sstables, String targetDirectory) + public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, TimeUUID compactionId, Collection sstables, String targetDirectory) { - this(metadata, tasktype, completed, total, Unit.BYTES, compactionId, sstables, targetDirectory); + this(metadata, tasktype, completed, total, totalCompressed, Unit.BYTES, compactionId, sstables, targetDirectory); } - public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, TimeUUID compactionId, Collection sstables) + public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, TimeUUID compactionId, Collection sstables) { - this(metadata, tasktype, completed, total, Unit.BYTES, compactionId, sstables, null); + this(metadata, tasktype, completed, total, totalCompressed, Unit.BYTES, compactionId, sstables, null); } /** * Special compaction info where we always need to cancel the compaction - for example ViewBuilderTask where we don't know * the sstables at construction */ - public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId) + public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, Unit unit, TimeUUID compactionId) { - return withoutSSTables(metadata, tasktype, completed, total, unit, compactionId, null); + return withoutSSTables(metadata, tasktype, completed, total, totalCompressed, unit, compactionId, null); } /** * Special compaction info where we always need to cancel the compaction - for example AutoSavingCache where we don't know * the sstables at construction */ - public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId, String targetDirectory) + public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, Unit unit, TimeUUID compactionId, String targetDirectory) { - return new CompactionInfo(metadata, tasktype, completed, total, unit, compactionId, ImmutableSet.of(), targetDirectory); + return new CompactionInfo(metadata, tasktype, completed, total, totalCompressed, unit, compactionId, ImmutableSet.of(), targetDirectory); } /** @return A copy of this CompactionInfo with updated progress. */ - public CompactionInfo forProgress(long complete, long total) + public CompactionInfo forProgress(long complete, long total, long totalCompressed) { - return new CompactionInfo(metadata, tasktype, complete, total, unit, compactionId, sstables, targetDirectory); + return new CompactionInfo(metadata, tasktype, complete, total, totalCompressed, unit, compactionId, sstables, targetDirectory); } public Optional getKeyspace() @@ -128,6 +131,11 @@ public long getTotal() return total; } + public long getTotalCompressed() + { + return totalCompressed; + } + public OperationType getTaskType() { return tasktype; @@ -183,12 +191,16 @@ public String targetDirectory() /** * Note that this estimate is based on the amount of data we have left to read - it assumes input * size == output size for a compaction, which is not really true, but should most often provide a worst case - * remaining write size. + * remaining write size. We also scale by the effective compression ratio since total/completed are for the uncompressed size. */ - public long estimatedRemainingWriteBytes() + public long estimatedRemainingWriteToDiskBytes() { if (unit == Unit.BYTES && tasktype.writesData) - return getTotal() - getCompleted(); + { + final long total = getTotal(); + double compressionRatio = total == 0 ? 1 : ((double) totalCompressed / (double)total); + return (long)(compressionRatio * (total - getCompleted())); + } return 0; } @@ -216,6 +228,7 @@ public Map asMap() ret.put(COLUMNFAMILY, getTable().orElse(null)); ret.put(COMPLETED, Long.toString(completed)); ret.put(TOTAL, Long.toString(total)); + ret.put(TOTAL_COMPRESSED, Long.toString(totalCompressed)); ret.put(TASK_TYPE, tasktype.toString()); ret.put(UNIT, unit.toString()); ret.put(COMPACTION_ID, compactionId == null ? "" : compactionId.toString()); diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java index 00e3dee5af2..e550977d304 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java @@ -98,6 +98,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte private final long nowInSec; private final TimeUUID compactionId; private final long totalBytes; + private final long totalCompressedBytes; private long bytesRead; private long totalSourceCQLRows; @@ -135,9 +136,14 @@ public CompactionIterator(OperationType type, this.bytesRead = 0; long bytes = 0; + long compressedBytes = 0; for (ISSTableScanner scanner : scanners) + { bytes += scanner.getLengthInBytes(); + compressedBytes += scanner.getCompressedLengthInBytes(); + } this.totalBytes = bytes; + this.totalCompressedBytes = compressedBytes; this.mergeCounters = new long[scanners.size()]; // note that we leak `this` from the constructor when calling beginCompaction below, this means we have to get the sstables before // calling that to avoid a NPE. @@ -170,6 +176,7 @@ public CompactionInfo getCompactionInfo() type, bytesRead, totalBytes, + totalCompressedBytes, compactionId, sstables, targetDirectory); diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java index 9566ef84304..f57e9342d44 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@ -406,7 +406,7 @@ protected boolean buildCompactionCandidatesForAvailableDiskSpace(final Set expectedWriteSize = CompactionManager.instance.active.estimatedRemainingWriteBytes(); + Map expectedWriteSize = CompactionManager.instance.active.estimatedRemainingWriteToDiskBytes(); // todo: abort streams if they block compactions if (cfs.getDirectories().hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSize, expectedWriteSize)) diff --git a/src/java/org/apache/cassandra/db/view/ViewBuilderTask.java b/src/java/org/apache/cassandra/db/view/ViewBuilderTask.java index 9a72c1e270a..8443413726b 100644 --- a/src/java/org/apache/cassandra/db/view/ViewBuilderTask.java +++ b/src/java/org/apache/cassandra/db/view/ViewBuilderTask.java @@ -204,13 +204,13 @@ public CompactionInfo getCompactionInfo() if (range.left.getPartitioner().splitter().isPresent()) { long progress = prevToken == null ? 0 : Math.round(prevToken.getPartitioner().splitter().get().positionInRange(prevToken, range) * 1000); - return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, progress, 1000, Unit.RANGES, compactionId); + return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, progress, 1000, 1000, Unit.RANGES, compactionId); } // When there is no splitter, estimate based on number of total keys but // take the max with keysBuilt + 1 to avoid having more completed than total long keysTotal = Math.max(keysBuilt + 1, baseCfs.estimatedKeysForRange(range)); - return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, keysBuilt, keysTotal, Unit.KEYS, compactionId); + return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, keysBuilt, keysTotal, keysTotal, Unit.KEYS, compactionId); } @Override diff --git a/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java b/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java index e2f38f8e920..7fc1970cb6e 100644 --- a/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java +++ b/src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java @@ -37,6 +37,7 @@ final class SSTableTasksTable extends AbstractVirtualTable private final static String PROGRESS = "progress"; private final static String SSTABLES = "sstables"; private final static String TOTAL = "total"; + private final static String TOTAL_COMPRESSED = "total_compressed"; private final static String UNIT = "unit"; private final static String TARGET_DIRECTORY = "target_directory"; @@ -54,6 +55,7 @@ final class SSTableTasksTable extends AbstractVirtualTable .addRegularColumn(PROGRESS, LongType.instance) .addRegularColumn(SSTABLES, Int32Type.instance) .addRegularColumn(TOTAL, LongType.instance) + .addRegularColumn(TOTAL_COMPRESSED, LongType.instance) .addRegularColumn(UNIT, UTF8Type.instance) .addRegularColumn(TARGET_DIRECTORY, UTF8Type.instance) .build()); @@ -67,6 +69,7 @@ public DataSet data() { long completed = task.getCompleted(); long total = task.getTotal(); + long totalCompressed = task.getTotalCompressed(); double completionRatio = total == 0L ? 1.0 : (((double) completed) / total); @@ -79,6 +82,7 @@ public DataSet data() .column(SSTABLES, task.getSSTables().size()) .column(TOTAL, total) .column(UNIT, task.getUnit().toString().toLowerCase()) + .column(TOTAL_COMPRESSED, totalCompressed) .column(TARGET_DIRECTORY, task.targetDirectory()); } diff --git a/src/java/org/apache/cassandra/index/internal/CollatedViewIndexBuilder.java b/src/java/org/apache/cassandra/index/internal/CollatedViewIndexBuilder.java index 07bdc420ca0..92c48cc7ca4 100644 --- a/src/java/org/apache/cassandra/index/internal/CollatedViewIndexBuilder.java +++ b/src/java/org/apache/cassandra/index/internal/CollatedViewIndexBuilder.java @@ -61,8 +61,10 @@ public CompactionInfo getCompactionInfo() OperationType.INDEX_BUILD, iter.getBytesRead(), iter.getTotalBytes(), + iter.getTotalBytes(), compactionId, - sstables); + sstables + ); } public void build() diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java index 55f6381859c..fe856059f81 100644 --- a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java +++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java @@ -249,6 +249,7 @@ public CompactionInfo getCompactionInfo() OperationType.INDEX_BUILD, bytesProcessed, totalSizeInBytes, + totalSizeInBytes, compactionId, sstables.keySet()); } diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java b/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java index 555bce1b9ad..4e71538e41f 100644 --- a/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java +++ b/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java @@ -134,6 +134,7 @@ public CompactionInfo getCompactionInfo() OperationType.INDEX_BUILD, bytesProcessed, totalBytesToProcess, + totalBytesToProcess, compactionId, sstables.keySet(), targetDirectory); diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java index e8fbea22d27..a83527509fc 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java @@ -382,6 +382,7 @@ public CompactionInfo getCompactionInfo() OperationType.SCRUB, dataFile.getFilePointer(), dataFile.length(), + sstable.onDiskLength(), scrubCompactionId, ImmutableSet.of(sstable), File.getPath(sstable.getFilename()).getParent().toString()); diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java index f68e1c96845..a45dc585a90 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java @@ -493,6 +493,7 @@ public CompactionInfo getCompactionInfo() OperationType.VERIFY, dataFile.getFilePointer(), dataFile.length(), + sstable.onDiskLength(), verificationCompactionId, ImmutableSet.of(sstable)); } diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java index d27969719ab..5e2d0584e71 100644 --- a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java +++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java @@ -357,7 +357,7 @@ static > Pair, List perTableIdIncomingBytes, for (FileStore fs : allWriteableFileStores) newStreamBytesToWritePerFileStore.merge(fs, totalBytesInPerFileStore, Long::sum); } - Map totalCompactionWriteRemaining = Directories.perFileStore(CompactionManager.instance.active.estimatedRemainingWriteBytes(), + Map totalCompactionWriteRemaining = Directories.perFileStore(CompactionManager.instance.active.estimatedRemainingWriteToDiskBytes(), fileStoreMapper); long totalStreamRemaining = StreamManager.instance.getTotalRemainingOngoingBytes(); long totalBytesStreamRemainingPerFileStore = totalStreamRemaining / Math.max(1, allFileStores.size()); diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 7e7934a9605..bebc54d460a 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -2090,6 +2090,7 @@ public Object getCompactionMetric(String metricName) switch(metricName) { case "BytesCompacted": + case "CompressedBytesCompacted": case "CompactionsAborted": case "CompactionsReduced": case "SSTablesDroppedFromCompaction": diff --git a/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java b/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java index c80de91d97d..07710ebf215 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java +++ b/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java @@ -90,11 +90,17 @@ private void compactionsStats(NodeProbe probe, TableBuilder tableBuilder) tableBuilder.add("compactions completed", String.valueOf(totalCompactionsCompletedMetrics.getCount())); CassandraMetricsRegistry.JmxCounterMBean bytesCompacted = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("BytesCompacted"); + CassandraMetricsRegistry.JmxCounterMBean compressedBytesCompacted = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("CompressedBytesCompacted"); if (humanReadable) + { tableBuilder.add("data compacted", FileUtils.stringifyFileSize(Double.parseDouble(Long.toString(bytesCompacted.getCount())))); + tableBuilder.add("compressed data compacted", FileUtils.stringifyFileSize(Double.parseDouble(Long.toString(compressedBytesCompacted.getCount())))); + } else + { tableBuilder.add("data compacted", Long.toString(bytesCompacted.getCount())); - + tableBuilder.add("compressed data compacted", Long.toString(compressedBytesCompacted.getCount())); + } CassandraMetricsRegistry.JmxCounterMBean compactionsAborted = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("CompactionsAborted"); tableBuilder.add("compactions aborted", Long.toString(compactionsAborted.getCount())); @@ -129,13 +135,14 @@ public static void reportCompactionTable(List> compactions, l long remainingBytes = 0; if (vtableOutput) - table.add("keyspace", "table", "task id", "completion ratio", "kind", "progress", "sstables", "total", "unit", "target directory"); + table.add("keyspace", "table", "task id", "completion ratio", "kind", "progress", "sstables", "total", "total compressed", "unit", "target directory"); else table.add("id", "compaction type", "keyspace", "table", "completed", "total", "unit", "progress"); for (Map c : compactions) { long total = Long.parseLong(c.get(CompactionInfo.TOTAL)); + String totalCompressedValue = c.get(CompactionInfo.TOTAL_COMPRESSED); long completed = Long.parseLong(c.get(CompactionInfo.COMPLETED)); String taskType = c.get(CompactionInfo.TASK_TYPE); String keyspace = c.get(CompactionInfo.KEYSPACE); @@ -145,12 +152,22 @@ public static void reportCompactionTable(List> compactions, l String[] tables = c.get(CompactionInfo.SSTABLES).split(","); String progressStr = toFileSize ? FileUtils.stringifyFileSize(completed) : Long.toString(completed); String totalStr = toFileSize ? FileUtils.stringifyFileSize(total) : Long.toString(total); + String totalCompressedStr; + if (totalCompressedValue != null) + { + long totalCompressed = Long.parseLong(totalCompressedValue); + totalCompressedStr = toFileSize ? FileUtils.stringifyFileSize(totalCompressed) : Long.toString(totalCompressed); + } + else + { + totalCompressedStr = "n/a"; + } String percentComplete = total == 0 ? "n/a" : new DecimalFormat("0.00").format((double) completed / total * 100) + '%'; String id = c.get(CompactionInfo.COMPACTION_ID); if (vtableOutput) { String targetDirectory = c.get(CompactionInfo.TARGET_DIRECTORY); - table.add(keyspace, columnFamily, id, percentComplete, taskType, progressStr, String.valueOf(tables.length), totalStr, unit, targetDirectory); + table.add(keyspace, columnFamily, id, percentComplete, taskType, progressStr, String.valueOf(tables.length), totalStr, totalCompressedStr, unit, targetDirectory); } else table.add(id, taskType, keyspace, columnFamily, progressStr, totalStr, unit, percentComplete); @@ -169,4 +186,4 @@ public static void reportCompactionTable(List> compactions, l table.printTo(out); } -} \ No newline at end of file +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/CompactionDiskSpaceTest.java b/test/distributed/org/apache/cassandra/distributed/test/CompactionDiskSpaceTest.java index 099f87dd40b..89e43ecb264 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/CompactionDiskSpaceTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/CompactionDiskSpaceTest.java @@ -113,7 +113,7 @@ public static class BB public static void install(ClassLoader cl, Integer node) { new ByteBuddy().rebase(ActiveCompactions.class) - .method(named("estimatedRemainingWriteBytes")) + .method(named("estimatedRemainingWriteToDiskBytes")) .intercept(MethodDelegation.to(BB.class)) .make() .load(cl, ClassLoadingStrategy.Default.INJECTION); @@ -125,7 +125,7 @@ public static void install(ClassLoader cl, Integer node) .load(cl, ClassLoadingStrategy.Default.INJECTION); } - public static Map estimatedRemainingWriteBytes() + public static Map estimatedRemainingWriteToDiskBytes() { if (sstableDir != null) return ImmutableMap.of(sstableDir, estimatedRemaining.get()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexCompactionTest.java b/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexCompactionTest.java index 9d168145c55..35d47bc4bd7 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexCompactionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexCompactionTest.java @@ -60,7 +60,7 @@ public void test2iCompaction() throws IOException // emulate ongoing index compaction: CompactionInfo.Holder h = new MockHolder(i.getIndexCfs().metadata(), idxSSTables); CompactionManager.instance.active.beginCompaction(h); - CompactionManager.instance.active.estimatedRemainingWriteBytes(); + CompactionManager.instance.active.estimatedRemainingWriteToDiskBytes(); CompactionManager.instance.active.finishCompaction(h); }); } @@ -79,7 +79,7 @@ public MockHolder(TableMetadata metadata, Set sstables) @Override public CompactionInfo getCompactionInfo() { - return new CompactionInfo(metadata, OperationType.COMPACTION, 0, 1000, nextTimeUUID(), sstables); + return new CompactionInfo(metadata, OperationType.COMPACTION, 0, 1000, 300, nextTimeUUID(), sstables); } @Override @@ -88,4 +88,4 @@ public boolean isGlobal() return false; } } -} \ No newline at end of file +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/StreamsDiskSpaceTest.java b/test/distributed/org/apache/cassandra/distributed/test/StreamsDiskSpaceTest.java index 5d72660fe18..0e84ca612c2 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/StreamsDiskSpaceTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/StreamsDiskSpaceTest.java @@ -72,7 +72,7 @@ public void testAbortStreamsWhenOngoingCompactionsLeaveInsufficientSpace() throw .withConfig(config -> config.set("hinted_handoff_enabled", false) .with(GOSSIP) .with(NETWORK)) - .withInstanceInitializer((cl, id) -> BB.doInstall(cl, id, ActiveCompactions.class, "estimatedRemainingWriteBytes")) + .withInstanceInitializer((cl, id) -> BB.doInstall(cl, id, ActiveCompactions.class, "estimatedRemainingWriteToDiskBytes")) .start())) { cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int) with compaction={'class': 'SizeTieredCompactionStrategy'}"); @@ -138,7 +138,7 @@ public static long getTotalRemainingOngoingBytes() return ongoing.get(); } - public static Map estimatedRemainingWriteBytes() + public static Map estimatedRemainingWriteToDiskBytes() { Map ret = new HashMap<>(); if (datadir != null) diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionInfoTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionInfoTest.java index 753a18505de..d995941ef5a 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionInfoTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionInfoTest.java @@ -39,7 +39,7 @@ public void testCompactionInfoToStringContainsTaskId() { ColumnFamilyStore cfs = MockSchema.newCFS(); TimeUUID expectedTaskId = nextTimeUUID(); - CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, expectedTaskId, new ArrayList<>()); + CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, 1000, expectedTaskId, new ArrayList<>()); Assertions.assertThat(compactionInfo.toString()) .contains(expectedTaskId.toString()); } @@ -50,7 +50,7 @@ public void testCompactionInfoToStringFormat() UUID tableId = UUID.randomUUID(); TimeUUID taskId = nextTimeUUID(); ColumnFamilyStore cfs = MockSchema.newCFS(builder -> builder.id(TableId.fromUUID(tableId))); - CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, taskId, new ArrayList<>()); + CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, 300, taskId, new ArrayList<>()); Assertions.assertThat(compactionInfo.toString()) .isEqualTo("Compaction(%s, 0 / 1000 bytes)@%s(mockks, mockcf1)", taskId, tableId); } diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java index 364ab56792e..5a6e04251a0 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java @@ -83,7 +83,7 @@ public class CompactionsCQLTest extends CQLTester public void before() throws IOException { strategy = DatabaseDescriptor.getCorruptedTombstoneStrategy(); - + CommitLog.instance.resetUnsafe(true); } @@ -872,7 +872,9 @@ public void testNoDiskspace() throws Throwable execute("insert into %s (id, i) values (?,?)", i, i); getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); } - CompactionInfo.Holder holder = holder(OperationType.COMPACTION); + // When we have an existing compaction with sstables of total size more than double the available space, + // we should not be able to then run a major compaction + CompactionInfo.Holder holder = holder(OperationType.COMPACTION, 2); CompactionManager.instance.active.beginCompaction(holder); try { @@ -888,7 +890,19 @@ public void testNoDiskspace() throws Throwable CompactionManager.instance.active.finishCompaction(holder); } // don't block compactions if there is a huge validation - holder = holder(OperationType.VALIDATION); + holder = holder(OperationType.VALIDATION, 2); + CompactionManager.instance.active.beginCompaction(holder); + try + { + getCurrentColumnFamilyStore().forceMajorCompaction(); + } + finally + { + CompactionManager.instance.active.finishCompaction(holder); + } + + // Should be able to run when the sstables in question are 90% of the total available space + holder = holder(OperationType.COMPACTION, 0.9); CompactionManager.instance.active.beginCompaction(holder); try { @@ -900,7 +914,7 @@ public void testNoDiskspace() throws Throwable } } - private CompactionInfo.Holder holder(OperationType opType) + private CompactionInfo.Holder holder(OperationType opType, double availableSpaceMultiplier) { CompactionInfo.Holder holder = new CompactionInfo.Holder() { @@ -910,12 +924,17 @@ public CompactionInfo getCompactionInfo() for (File f : getCurrentColumnFamilyStore().getDirectories().getCFDirectories()) availableSpace += PathUtils.tryGetSpace(f.toPath(), FileStore::getUsableSpace); + Set liveSSTables = getCurrentColumnFamilyStore().getLiveSSTables(); + long totalDiskUsage = (long)(availableSpace * availableSpaceMultiplier); + // Arbitrary compression ratio of 3.4 + long totalUncompressedSize = (long) ((double) totalDiskUsage * 3.4); return new CompactionInfo(getCurrentColumnFamilyStore().metadata(), opType, +0, - +availableSpace * 2, + totalUncompressedSize, + totalDiskUsage, nextTimeUUID(), - getCurrentColumnFamilyStore().getLiveSSTables()); + liveSSTables); } public boolean isGlobal() diff --git a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java index 4960bfd1888..b837b0ea2ae 100644 --- a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java @@ -609,7 +609,7 @@ private void tryPredicate(ColumnFamilyStore cfs, List compacting, { public CompactionInfo getCompactionInfo() { - return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 1000, nextTimeUUID(), compacting); + return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 1000, 1000, nextTimeUUID(), compacting); } public boolean isGlobal() @@ -650,7 +650,7 @@ public void testRetries() throws InterruptedException, ExecutionException { public CompactionInfo getCompactionInfo() { - return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, nextTimeUUID(), cfs.getLiveSSTables()); + return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, 0, nextTimeUUID(), cfs.getLiveSSTables()); } public boolean isGlobal() @@ -703,7 +703,7 @@ public void testRetriesTimeout() throws InterruptedException, ExecutionException { public CompactionInfo getCompactionInfo() { - return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, nextTimeUUID(), cfs.getLiveSSTables()); + return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, 0, nextTimeUUID(), cfs.getLiveSSTables()); } public boolean isGlobal() diff --git a/test/unit/org/apache/cassandra/db/virtual/SSTableTasksTableTest.java b/test/unit/org/apache/cassandra/db/virtual/SSTableTasksTableTest.java index 6e8a13612b3..365e6420ee3 100644 --- a/test/unit/org/apache/cassandra/db/virtual/SSTableTasksTableTest.java +++ b/test/unit/org/apache/cassandra/db/virtual/SSTableTasksTableTest.java @@ -69,6 +69,7 @@ public void testSelectAll() throws Throwable long bytesCompacted = 123; long bytesTotal = 123456; + long totalCompressedBytes = 112233; TimeUUID compactionId = nextTimeUUID(); List sstables = IntStream.range(0, 10) .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs)) @@ -80,7 +81,7 @@ public void testSelectAll() throws Throwable { public CompactionInfo getCompactionInfo() { - return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, directory); + return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables, directory); } public boolean isGlobal() @@ -93,7 +94,7 @@ public boolean isGlobal() UntypedResultSet result = execute("SELECT * FROM vts.sstable_tasks"); assertRows(result, row(CQLTester.KEYSPACE, currentTable(), compactionId, 1.0 * bytesCompacted / bytesTotal, OperationType.COMPACTION.toString().toLowerCase(), bytesCompacted, sstables.size(), - directory, bytesTotal, CompactionInfo.Unit.BYTES.toString())); + directory, bytesTotal, totalCompressedBytes, CompactionInfo.Unit.BYTES.toString())); CompactionManager.instance.active.finishCompaction(compactionHolder); result = execute("SELECT * FROM vts.sstable_tasks"); diff --git a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java index 40d04a48401..aa3eb75811d 100644 --- a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java @@ -654,7 +654,7 @@ public void testCancelIndexHelper(Consumer cancelFunction) th { public CompactionInfo getCompactionInfo() { - return new CompactionInfo(cfs.metadata(), OperationType.UNKNOWN, 0, 0, nextTimeUUID(), compacting); + return new CompactionInfo(cfs.metadata(), OperationType.UNKNOWN, 0, 0, 0, nextTimeUUID(), compacting); } public boolean isGlobal() diff --git a/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java b/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java index 8758c3f8fd8..8ccf9c54a2d 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java @@ -104,6 +104,7 @@ public void testCompactionStats() long bytesCompacted = 123; long bytesTotal = 123456; + long totalCompressedBytes = 112233; TimeUUID compactionId = nextTimeUUID(); List sstables = IntStream.range(0, 10) .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs)) @@ -112,7 +113,7 @@ public void testCompactionStats() { public CompactionInfo getCompactionInfo() { - return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables); + return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables); } public boolean isGlobal() @@ -134,6 +135,7 @@ compactionId, OperationType.COMPACTION, CQLTester.KEYSPACE, currentTable(), byte assertThat(stdout).containsPattern("pending tasks\\s+[0-9]*"); assertThat(stdout).containsPattern("compactions completed\\s+[0-9]*"); assertThat(stdout).containsPattern("data compacted\\s+[0-9]*"); + assertThat(stdout).containsPattern("compressed data compacted\\s+[0-9]*"); assertThat(stdout).containsPattern("compactions aborted\\s+[0-9]*"); assertThat(stdout).containsPattern("compactions reduced\\s+[0-9]*"); assertThat(stdout).containsPattern("sstables dropped from compaction\\s+[0-9]*"); @@ -153,6 +155,7 @@ public void testCompactionStatsVtable() long bytesCompacted = 123; long bytesTotal = 123456; + long totalCompressedBytes = 112233; TimeUUID compactionId = nextTimeUUID(); List sstables = IntStream.range(0, 10) .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs)) @@ -162,7 +165,7 @@ public void testCompactionStatsVtable() { public CompactionInfo getCompactionInfo() { - return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, targetDirectory); + return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables, targetDirectory); } public boolean isGlobal() @@ -175,7 +178,7 @@ public boolean isGlobal() { public CompactionInfo getCompactionInfo() { - return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, compactionId, sstables); + return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables); } public boolean isGlobal() @@ -187,16 +190,16 @@ public boolean isGlobal() CompactionManager.instance.active.beginCompaction(compactionHolder); CompactionManager.instance.active.beginCompaction(nonCompactionHolder); String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "-V"); - assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit\\s+target directory"); - String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s", + assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+total compressed\\s+unit\\s+target directory"); + String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s", CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100, - OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, CompactionInfo.Unit.BYTES, + OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, totalCompressedBytes, CompactionInfo.Unit.BYTES, targetDirectory); assertThat(stdout).containsPattern(expectedStatsPattern); - String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s", + String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s", CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100, - OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, CompactionInfo.Unit.BYTES); + OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, totalCompressedBytes, CompactionInfo.Unit.BYTES); assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction); CompactionManager.instance.active.finishCompaction(compactionHolder); @@ -212,6 +215,7 @@ public void testCompactionStatsHumanReadable() long bytesCompacted = 123; long bytesTotal = 123456; + long totalCompressedBytes = 112233; TimeUUID compactionId = nextTimeUUID(); List sstables = IntStream.range(0, 10) .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs)) @@ -220,7 +224,7 @@ public void testCompactionStatsHumanReadable() { public CompactionInfo getCompactionInfo() { - return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables); + return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables); } public boolean isGlobal() @@ -249,6 +253,7 @@ public void testCompactionStatsVtableHumanReadable() long bytesCompacted = 123; long bytesTotal = 123456; + long totalCompressedBytes = 112233; TimeUUID compactionId = nextTimeUUID(); List sstables = IntStream.range(0, 10) .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs)) @@ -258,7 +263,7 @@ public void testCompactionStatsVtableHumanReadable() { public CompactionInfo getCompactionInfo() { - return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, targetDirectory); + return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables, targetDirectory); } public boolean isGlobal() @@ -271,7 +276,7 @@ public boolean isGlobal() { public CompactionInfo getCompactionInfo() { - return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, compactionId, sstables); + return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables); } public boolean isGlobal() @@ -283,15 +288,15 @@ public boolean isGlobal() CompactionManager.instance.active.beginCompaction(compactionHolder); CompactionManager.instance.active.beginCompaction(nonCompactionHolder); String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "--vtable", "--human-readable"); - assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit\\s+target directory"); - String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s", + assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+total compressed\\s+unit\\s+target directory"); + String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s", CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100, - OperationType.COMPACTION, "123 bytes", sstables.size(), "120.56 KiB", CompactionInfo.Unit.BYTES, + OperationType.COMPACTION, "123 bytes", sstables.size(), "120.56 KiB", "109.6 KiB", CompactionInfo.Unit.BYTES, targetDirectory); assertThat(stdout).containsPattern(expectedStatsPattern); - String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s", + String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s", CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100, - OperationType.CLEANUP, "123 bytes", sstables.size(), "120.56 KiB", CompactionInfo.Unit.BYTES); + OperationType.CLEANUP, "123 bytes", sstables.size(), "120.56 KiB", "109.6 KiB", CompactionInfo.Unit.BYTES); assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction); CompactionManager.instance.active.finishCompaction(compactionHolder); @@ -321,4 +326,4 @@ private String waitForNumberOfPendingTasks(int pendingTasksToWaitFor, String... return stdout.get(); } -} \ No newline at end of file +}