From fa2412e3dfa2e23c7936c78dca60a631e7fb585a Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Mon, 31 Aug 2026 17:58:10 +0300 Subject: [PATCH 01/44] raw --- .../SnapshotStatusReproducerTest.java | 167 ++++++++++++++++++ .../snapshot/SnapshotStatusTask.java | 45 ++++- .../snapshot/SnapshotRestoreProcess.java | 2 +- 3 files changed, 204 insertions(+), 10 deletions(-) create mode 100644 modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java new file mode 100644 index 0000000000000..1462cdeb9b0bf --- /dev/null +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java @@ -0,0 +1,167 @@ +package org.apache.ignite.internal.commandline.snapshot; + +import java.util.Collection; +import java.util.concurrent.CountDownLatch; +import java.util.stream.IntStream; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotPartitionsVerifyResult; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.util.GridCommandHandlerAbstractTest; +import org.jetbrains.annotations.Nullable; +import org.junit.Test; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.MINUTES; +import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED; +import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; +import static org.apache.ignite.testframework.GridTestUtils.assertContains; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** + * + */ +public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest { + /** Snapshot check latch. */ + private static CountDownLatch snapshotCheckLatch; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setPluginProviders(new BlockingCheckSnapshotPluginProvider()) + .setIncludeEventTypes(EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + autoConfirmation = false; + + cleanPersistenceDir(); + + snapshotCheckLatch = new CountDownLatch(1); + + startGrids(3); + + grid(0).cluster().state(ClusterState.ACTIVE); + + IgniteCache cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME); + + IntStream.range(0, 2048).forEach(i -> cache.put(i, i)); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(true); + + cleanPersistenceDir(); + } + + /** + * + */ + @Test + public void test() throws Exception { + injectTestSystemOut(); + + IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); + + String snapshotName = "test_snapshot"; + + snapshotMgr.createSnapshot(snapshotName).get(getTestTimeout()); + + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + CountDownLatch restoreStarterdLatch = new CountDownLatch(1); + + grid(0).events().localListen( + e -> { + restoreStarterdLatch.countDown(); + + return false; + }, + EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED + ); + + IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot(snapshotName, null, null, 0, true); + + try { + restoreStarterdLatch.await(getTestTimeout(), MILLISECONDS); + assertFalse("Snapshot future has finished", restoreFut.isDone()); + + int code = execute("--snapshot", "restore", snapshotName, "--status"); + + assertEquals("Unexpected exit code", EXIT_CODE_OK, code); + + assertFalse("Snapshot future has finished", restoreFut.isDone()); + + code = execute("--snapshot", "status"); + + assertEquals("Unexpected exit code", EXIT_CODE_OK, code); + assertContains(log, testOut.toString(), "Restore snapshot operation is in progress."); + } + finally { + snapshotCheckLatch.countDown(); + + // Wait for future to finish in order to avoid excessive message about task cancellation. + restoreFut.get(); + } + } + + /** */ + private static class BlockingCheckSnapshotPluginProvider extends AbstractTestPluginProvider { + /** {@inheritDoc} */ + @Override public String name() { + return "BlockingCheckSnapshotPluginProvider"; + } + + /** {@inheritDoc} */ + @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { + if (IgniteSnapshotManager.class.equals(cls)) + return (T)new BlockingCheckSnapshotManager(((IgniteEx)ctx.grid()).context()); + + return null; + } + } + + /** Blocks restore proccess. */ + protected static class BlockingCheckSnapshotManager extends IgniteSnapshotManager { + /** */ + public BlockingCheckSnapshotManager(GridKernalContext ctx) { + super(ctx); + } + + /** {@inheritDoc} */ + @Override public IgniteInternalFuture checkSnapshot( + String name, + @Nullable String snpPath, + @Nullable Collection grps, + boolean includeCustomHandlers, + int incIdx, boolean check + ) { + return super.checkSnapshot(name, snpPath, grps, includeCustomHandlers, incIdx, check) + .chain(fut -> { + try { + if(check) + ;// snapshotCheckLatch.await(5, MINUTES); + } + catch (Throwable e) { + log.error(X.getFullStackTrace(e)); + } + + return fut.result(); + }); + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 01476d309e71f..f60fd461d2ce5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -84,14 +84,14 @@ public class SnapshotStatusTask extends VisorMultiNodeTask s.requestId.equals(s0.requestId)); + res = F.view(res, s -> s.reqId.equals(s0.reqId)); // Merge nodes progress. Map> progress = new HashMap<>(); res.forEach(s -> progress.putAll(s.progress)); - return new SnapshotStatus(s0.op, s0.name, s0.incIdx, s0.requestId, s0.startTime, progress); + return new SnapshotStatus(s0.op, s0.name, s0.incIdx, s0.reqId, s0.startTime, progress); } /** */ @@ -164,6 +164,30 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { ); } + mreg = ignite.context().metric().registry(SNAPSHOT_); + + long startTime = mreg.findMetric("startTime").value(); + + if (startTime > mreg.findMetric("endTime").value()) { + return new SnapshotStatus( + SnapshotOperation.RESTORE, + mreg.findMetric("snapshotName").getAsString(), + mreg.findMetric("incrementIndex").value(), + mreg.findMetric("requestId").getAsString(), + mreg.findMetric("startTime").value(), + F.asMap( + ignite.localNode().id(), + new T5<>( + (long)mreg.findMetric("processedPartitions").value(), + (long)mreg.findMetric("totalPartitions").value(), + (long)mreg.findMetric("processedWalSegments").value(), + (long)mreg.findMetric("totalWalSegments").value(), + mreg.findMetric("processedWalEntries").value() + ) + ) + ); + } + return null; } } @@ -183,7 +207,7 @@ public static class SnapshotStatus implements Serializable { private final int incIdx; /** Request ID. */ - private final String requestId; + private final String reqId; /** Start time. */ private final long startTime; @@ -196,14 +220,14 @@ public SnapshotStatus( SnapshotOperation op, String name, int incIdx, - String requestId, + String reqId, long startTime, Map> progress ) { this.op = op; this.name = name; this.incIdx = incIdx; - this.requestId = requestId; + this.reqId = reqId; this.startTime = startTime; this.progress = Collections.unmodifiableMap(progress); } @@ -225,7 +249,7 @@ public int incrementIndex() { /** @return Request ID. */ public String requestId() { - return requestId; + return reqId; } /** @return Start time. */ @@ -241,10 +265,13 @@ public Map> progress() { /** Snapshot operation type. */ public enum SnapshotOperation { - /** Create snapshot. */ + /** Snapshot creation. */ CREATE, - /** Restore snapshot. */ - RESTORE + /** Snapshot restoration. */ + RESTORE, + + /** Snapshot checking before restoration. */ + CHECK_RESTORE } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 77f56f4d4cc7f..50c589407d154 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -351,7 +351,7 @@ public IgniteFutureImpl start( .stream() .findFirst(); - if (!firstMeta.isPresent()) { + if (firstMeta.isEmpty()) { finishProcess( fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "No snapshot metadata read") From 19dde26c9ccd7decac6c0c775235736d75402ebf Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Tue, 1 Sep 2026 01:50:27 +0300 Subject: [PATCH 02/44] in-progress --- .../SnapshotStatusReproducerTest.java | 11 +- .../management/api/ComputeCommand.java | 2 +- .../snapshot/SnapshotStatusCommand.java | 36 ++- .../snapshot/SnapshotStatusTask.java | 263 ++++++++++++++---- .../snapshot/SnapshotCheckProcess.java | 7 + .../snapshot/SnapshotRestoreStatusTask.java | 2 +- .../feature/SupportedFeatureRegistry.java | 3 + .../resources/META-INF/classnames.properties | 2 + .../TestIgniteReleaseFeatures_2_19_0.java | 3 + 9 files changed, 261 insertions(+), 68 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java index 1462cdeb9b0bf..1de6947a6720b 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java @@ -24,11 +24,8 @@ import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; import static org.apache.ignite.testframework.GridTestUtils.assertContains; -import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; -/** - * - */ +/** */ public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest { /** Snapshot check latch. */ private static CountDownLatch snapshotCheckLatch; @@ -68,9 +65,7 @@ public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest cleanPersistenceDir(); } - /** - * - */ + /** */ @Test public void test() throws Exception { injectTestSystemOut(); @@ -154,7 +149,7 @@ public BlockingCheckSnapshotManager(GridKernalContext ctx) { .chain(fut -> { try { if(check) - ;// snapshotCheckLatch.await(5, MINUTES); + snapshotCheckLatch.await(5, MINUTES); } catch (Throwable e) { log.error(X.getFullStackTrace(e)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java index 10cdcf343c57e..6551e4ac48883 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java @@ -28,7 +28,7 @@ * Command that executed with some compute task. */ public interface ComputeCommand extends Command { - /** @return Task class. */ + /** @return Default task class. */ public Class> taskClass(); /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 18f4fc5b690e6..66212036030e7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -52,7 +52,7 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand taskClass() { + @Override public Class taskClass() { return SnapshotStatusTask.class; } @@ -64,15 +64,18 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand 0; GridStringBuilder s = new GridStringBuilder(); if (isCreating) s.a("Create snapshot operation is in progress.").nl(); - else + else if(isRestoring) s.a("Restore snapshot operation is in progress.").nl(); + else + s.a("Check snapshot operation is in progress.").nl(); s.a("Snapshot name: ").a(status.name()).nl(); s.a("Incremental: ").a(isIncremental).nl(); @@ -94,10 +97,12 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand> rows = status.progress().entrySet().stream().sorted(Map.Entry.comparingByKey()) .map(e -> desc.buildRow(e.getKey(), e.getValue())) @@ -246,4 +251,25 @@ private static class RestoreIncrementalSnapshotTaskProgressDesc extends Snapshot return result; } } + + /** */ + private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { + /** */ + CheckSnapshotTaskProgressDesc(boolean incremental) { + super(F.asList("Node ID", "Processed, bytes", "Total, bytes", "Percent")); + } + + /** {@inheritDoc} */ + @Override public List buildRow(UUID nodeId, T5 progress) { + long processed = progress.get1(); + long total = progress.get2(); + + if (total <= 0) + return F.asList(nodeId, "unknown", "unknown", "unknown"); + + String percent = (int)(processed * 100 / total) + "%"; + + return F.asList(nodeId, U.humanReadableByteCount(processed), U.humanReadableByteCount(total), percent); + } + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index f60fd461d2ce5..116e0d04b17d2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -18,25 +18,35 @@ package org.apache.ignite.internal.management.snapshot; import java.io.Serializable; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.management.api.NoArg; +import org.apache.ignite.internal.managers.discovery.IgniteClusterNode; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperationRequest; +import org.apache.ignite.internal.processors.metric.impl.MetricUtils; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; +import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.util.lang.GridFunc; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T5; import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorMultiNodeTask; import org.apache.ignite.internal.visor.VisorTaskArgument; import org.apache.ignite.metric.MetricRegistry; +import org.apache.ignite.resources.LoggerResource; +import org.apache.ignite.spi.metric.BooleanMetric; import org.apache.ignite.spi.metric.IntMetric; import org.apache.ignite.spi.metric.LongMetric; import org.jetbrains.annotations.Nullable; @@ -54,9 +64,68 @@ public class SnapshotStatusTask extends VisorMultiNodeTask job(NoArg arg) { - return new SnapshotStatusJob(arg, debug); + if (checkStatusSupported == null) + resolveCheckStatusSupported(); + + assert checkStatusSupported != null; + + return checkStatusSupported ? new SnapshotStatusJobV2(arg, debug) : new SnapshotStatusJob(arg, debug); + } + + /** */ + private void resolveCheckStatusSupported() { + var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); + + if (!ignite.context().rollingUpgrade().features().isActive(feature)) { + if (log.isInfoEnabled()) { + log.info("The snapshot-check-aware status feature isn't enabled. The status is available only for " + + "snapshot creation and restoration."); + } + + checkStatusSupported = false; + + return; + } + + for (var n : ignite.cluster().nodes()) { + if (!(n instanceof IgniteClusterNode)) { + if (log.isInfoEnabled()) { + log.info(String.format( + "Cannot extract features of node %s. The status is available only for snapshot creation and restoration.", + n.id() + )); + } + + checkStatusSupported = false; + + return; + } + + if (!((IgniteClusterNode)n).features().contains(feature)) { + if (log.isInfoEnabled()) { + log.info(String.format( + "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + + "for snapshot creation and restoration.", + n.id() + )); + } + + checkStatusSupported = false; + + return; + } + } + + checkStatusSupported = true; } /** {@inheritDoc} */ @@ -65,7 +134,7 @@ public class SnapshotStatusTask extends VisorMultiNodeTask results) { + @Override protected @Nullable SnapshotStatus reduce0(List results) { if (results.isEmpty()) throw new IgniteException("Failed to get the snapshot status. Topology is empty."); @@ -75,23 +144,41 @@ public class SnapshotStatusTask extends VisorMultiNodeTask res = F.viewReadOnly(results, ComputeJobResult::getData, r -> r.getData() != null); + Collection res0 = F.viewReadOnly(results, ComputeJobResult::getData, r -> r.getData() != null); // There is no snapshot operation. - if (res.isEmpty()) + if (res0.isEmpty()) return null; - SnapshotStatus s0 = F.first(res); + SnapshotStatus firstRes = F.first(res0); // Filter out differing requests due to concurrent updates on nodes. - res = F.view(res, s -> s.reqId.equals(s0.reqId)); + Collection sameRqRes = F.view(res0, s -> s.reqId.equals(firstRes.reqId)); + + if (firstRes instanceof SnapshotStatusTask.SnapshotStatusV2) { + var statusV2 = (SnapshotStatusTask.SnapshotStatusV2)firstRes; + + assert !F.isEmpty(statusV2.allCheckStatuses); + + Map> mergedAllCheckStatuses = U.newHashMap(sameRqRes.size()); + + sameRqRes.forEach(s -> { + assert s instanceof SnapshotStatusTask.SnapshotStatusV2; + + mergedAllCheckStatuses.putAll(((SnapshotStatusTask.SnapshotStatusV2)s).allCheckStatuses); + }); + + statusV2.allCheckStatuses = mergedAllCheckStatuses; + + return statusV2; + } // Merge nodes progress. - Map> progress = new HashMap<>(); + Map> mergedProgress = U.newHashMap(sameRqRes.size()); - res.forEach(s -> progress.putAll(s.progress)); + sameRqRes.forEach(s -> mergedProgress.putAll(s.progress)); - return new SnapshotStatus(s0.op, s0.name, s0.incIdx, s0.reqId, s0.startTime, progress); + return new SnapshotStatus(firstRes.op, firstRes.name, firstRes.incIdx, firstRes.reqId, firstRes.startTime, mergedProgress); } /** */ @@ -103,12 +190,12 @@ private static class SnapshotStatusJob extends SnapshotJob( mreg.findMetric("CurrentSnapshotProcessedSize").value(), mreg.findMetric("CurrentSnapshotTotalSize").value(), - -1L, -1L, -1L); + -1L, + -1L, + -1L + ); } return new SnapshotStatus( @@ -164,41 +254,17 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { ); } - mreg = ignite.context().metric().registry(SNAPSHOT_); - - long startTime = mreg.findMetric("startTime").value(); - - if (startTime > mreg.findMetric("endTime").value()) { - return new SnapshotStatus( - SnapshotOperation.RESTORE, - mreg.findMetric("snapshotName").getAsString(), - mreg.findMetric("incrementIndex").value(), - mreg.findMetric("requestId").getAsString(), - mreg.findMetric("startTime").value(), - F.asMap( - ignite.localNode().id(), - new T5<>( - (long)mreg.findMetric("processedPartitions").value(), - (long)mreg.findMetric("totalPartitions").value(), - (long)mreg.findMetric("processedWalSegments").value(), - (long)mreg.findMetric("totalWalSegments").value(), - mreg.findMetric("processedWalEntries").value() - ) - ) - ); - } - return null; } } /** Snapshot operation status. */ - public static class SnapshotStatus implements Serializable { + static class SnapshotStatus implements Serializable { /** */ private static final long serialVersionUID = 0L; - /** Operation type. */ - private final SnapshotOperation op; + /** Operation type. {@code Null} for other operation types. */ + private final @Nullable SnapshotOperation op; /** Snapshot name. */ private final String name; @@ -216,8 +282,8 @@ public static class SnapshotStatus implements Serializable { private final Map> progress; /** */ - public SnapshotStatus( - SnapshotOperation op, + private SnapshotStatus( + @Nullable SnapshotOperation op, String name, int incIdx, String reqId, @@ -232,46 +298,137 @@ public SnapshotStatus( this.progress = Collections.unmodifiableMap(progress); } - /** @return Operation type. */ - public SnapshotOperation operation() { + /** @return Operation type. {@code Null} for other operation types. */ + @Nullable SnapshotOperation operation() { return op; } /** @return Snapshot name. */ - public String name() { + String name() { return name; } /** @return Incremental snapshot index. */ - public int incrementIndex() { + int incrementIndex() { return incIdx; } /** @return Request ID. */ - public String requestId() { + String requestId() { return reqId; } /** @return Start time. */ - public long startTime() { + long startTime() { return startTime; } /** @return Progress of operation on nodes. */ - public Map> progress() { + Map> progress() { return progress; } } /** Snapshot operation type. */ - public enum SnapshotOperation { + enum SnapshotOperation { /** Snapshot creation. */ CREATE, /** Snapshot restoration. */ - RESTORE, + RESTORE + } + + /** */ + private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected @Nullable SnapshotStatusV2 run(@Nullable NoArg arg) throws IgniteException { + var res1 = super.run(arg); + + if (res1 != null) + return new SnapshotStatusV2(res1); + + List checkStatuses = null; + + for (var snpCheckMReg : ignite.context().metric()) { + if (!snpCheckMReg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) + continue; + + if (checkStatuses == null) + checkStatuses = new ArrayList<>(); - /** Snapshot checking before restoration. */ - CHECK_RESTORE + int incIdx = snpCheckMReg.findMetric("incrementIndex") == null + ? 0 + : ((IntMetric)snpCheckMReg.findMetric("incrementIndex")).value(); + + T5 metrics; + + if (incIdx > 0) { + metrics = new T5<>( + (long)snpCheckMReg.findMetric("processedWalSegments").value(), + (long)snpCheckMReg.findMetric("totalWalSegments").value(), + -1L, + -1L, + -1L + ); + } + else { + metrics = new T5<>( + snpCheckMReg.findMetric("checkPartitions").value() ? 1L : 0L, + (long)snpCheckMReg.findMetric("processedPartitions").value(), + (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), + (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), + (long)snpCheckMReg.findMetric("snapshotPartsToProcess").value() + ); + } + + checkStatuses.add(new SnapshotStatus( + null, + MetricUtils.fromFullName(snpCheckMReg.name()).get2(), + incIdx, + snpCheckMReg.findMetric("requestId").getAsString(), + ((LongMetric)snpCheckMReg.findMetric("startTime")).value(), + GridFunc.asMap(ignite.localNode().id(), metrics) + )); + } + + return checkStatuses == null ? null : new SnapshotStatusV2(Collections.singletonMap(ignite.localNode().id(), checkStatuses)); + } + } + + /** Supports snapsho status. */ + private static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { + /** */ + private static final long serialVersionUID = 0L; + + /** Statuses of snapshot check operations per nodeID. */ + private @Nullable Map> allCheckStatuses; + + /** */ + private SnapshotStatusV2(SnapshotStatus s1) { + super(s1.op, s1.name, s1.incIdx, s1.reqId, s1.startTime, s1.progress); + } + + /** */ + private SnapshotStatusV2(Map> allCheckStatuses) { + // Single, V1 status holds first found check status. + super( + null, + F.first(allCheckStatuses.values()).get(0).name, + F.first(allCheckStatuses.values()).get(0).incIdx, + F.first(allCheckStatuses.values()).get(0).reqId, + F.first(allCheckStatuses.values()).get(0).startTime, + F.first(allCheckStatuses.values()).get(0).progress + ); + + this.allCheckStatuses = allCheckStatuses; + } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index 2f19878617a15..f069919248f91 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -732,6 +732,13 @@ private void registerMetrics(SnapshotCheckContext ctx) { mreg.register("startTime", U::currentTimeMillis, "The system time of the start of the cluster snapshot check operation on current node."); + mreg.register( + "requestId", + () -> ctx.req.requestId().toString(), + String.class, + "The request ID of the last running cluster snapshot restore operation on this node." + ); + if (ctx.req.incrementalIndex() > 0) { mreg.register("incrementIndex", ctx.req::incrementalIndex, "The index of incremental snapshot of the snapshot check operation."); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java index 14a395779c9d4..6df4081be8c29 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java @@ -40,7 +40,7 @@ */ @GridInternal @Deprecated -class SnapshotRestoreStatusTask extends ComputeTaskAdapter { +public class SnapshotRestoreStatusTask extends ComputeTaskAdapter { /** Serial version uid. */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java index 7b3e55b85d3c4..fc91f1b838065 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java @@ -93,4 +93,7 @@ public class SupportedFeatureRegistry { /** */ public static final IgniteFeature ROLLING_UPGRADE_FEATURE = new IgniteCoreFeature(0); + + /** */ + public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = new IgniteCoreFeature(1); } diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index 65f0244da47b7..11594924fe6fd 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -658,7 +658,9 @@ org.apache.ignite.internal.management.snapshot.SnapshotRestoreTask$SnapshotStart org.apache.ignite.internal.management.snapshot.SnapshotStatusTask org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatus +org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusV2 org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusJob +org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusJobV2 org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationCommandArg org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationResetAllCommandArg org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationResetCommandArg diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java index 3cb497ee64701..7fb49a2744440 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java @@ -21,4 +21,7 @@ public class TestIgniteReleaseFeatures_2_19_0 { /** */ public static final IgniteFeature ROLLING_UPGRADE_FEATURE = new IgniteCoreFeature(0); + + /** */ + public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = new IgniteCoreFeature(1); } From f18d2fc3fa170b5489f79b5afb90a78bf7cc366e Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 1 Sep 2026 15:40:03 +0300 Subject: [PATCH 03/44] raw --- .../SnapshotStatusReproducerTest.java | 144 +++++++++--------- .../management/api/ComputeCommand.java | 2 +- .../snapshot/SnapshotStatusCommand.java | 46 +++++- .../snapshot/SnapshotStatusTask.java | 2 +- .../snapshot/SnapshotCheckProcess.java | 2 + .../util/distributed/SingleNodeMessage.java | 6 + 6 files changed, 116 insertions(+), 86 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java index 1de6947a6720b..115441ea33b37 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java @@ -1,40 +1,38 @@ package org.apache.ignite.internal.commandline.snapshot; -import java.util.Collection; +import java.util.Arrays; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import java.util.stream.IntStream; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.cluster.ClusterState; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.GridKernalContext; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; -import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotPartitionsVerifyResult; +import org.apache.ignite.internal.util.distributed.SingleNodeMessage; import org.apache.ignite.internal.util.future.IgniteFutureImpl; -import org.apache.ignite.internal.util.typedef.X; -import org.apache.ignite.plugin.AbstractTestPluginProvider; -import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.util.GridCommandHandlerAbstractTest; import org.jetbrains.annotations.Nullable; import org.junit.Test; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.MINUTES; -import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_METAS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; import static org.apache.ignite.testframework.GridTestUtils.assertContains; /** */ public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest { - /** Snapshot check latch. */ - private static CountDownLatch snapshotCheckLatch; - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - return super.getConfiguration(igniteInstanceName) - .setPluginProviders(new BlockingCheckSnapshotPluginProvider()) - .setIncludeEventTypes(EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED); + return super.getConfiguration(igniteInstanceName).setCommunicationSpi(new TestCommunicationSpi()); } /** {@inheritDoc} */ @@ -45,8 +43,6 @@ public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest cleanPersistenceDir(); - snapshotCheckLatch = new CountDownLatch(1); - startGrids(3); grid(0).cluster().state(ClusterState.ACTIVE); @@ -78,85 +74,81 @@ public void test() throws Exception { grid(0).destroyCache(DEFAULT_CACHE_NAME); - CountDownLatch restoreStarterdLatch = new CountDownLatch(1); + awaitPartitionMapExchange(); - grid(0).events().localListen( - e -> { - restoreStarterdLatch.countDown(); + CountDownLatch checkSingleResultsReceivedLatch = new CountDownLatch(2); + CountDownLatch restoreSingleResultsReceivedLatch = new CountDownLatch(2); + CountDownLatch proceedCheckLatch = new CountDownLatch(1); - return false; - }, - EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED - ); + for(var ig : Arrays.asList(grid(1), grid(2))) { + ((TestCommunicationSpi)ig.configuration().getCommunicationSpi()).msgCsmr = msg -> { + if(!(msg instanceof GridIoMessage ioMsg)) + return; - IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot(snapshotName, null, null, 0, true); + if(!(ioMsg.message() instanceof SingleNodeMessage sm)) + return; - try { - restoreStarterdLatch.await(getTestTimeout(), MILLISECONDS); - assertFalse("Snapshot future has finished", restoreFut.isDone()); + if (sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) { + restoreSingleResultsReceivedLatch.countDown(); + } else if (sm.type() == CHECK_SNAPSHOT_METAS.ordinal()) { + checkSingleResultsReceivedLatch.countDown(); - int code = execute("--snapshot", "restore", snapshotName, "--status"); + try { + assertTrue(proceedCheckLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }; + } - assertEquals("Unexpected exit code", EXIT_CODE_OK, code); + IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot(snapshotName, null, null, 0, true); - assertFalse("Snapshot future has finished", restoreFut.isDone()); + assertTrue(checkSingleResultsReceivedLatch.await(getTestTimeout(), MILLISECONDS)); - code = execute("--snapshot", "status"); + // Make sure no restoration started or finished. + assertTrue(restoreSingleResultsReceivedLatch.getCount() > 0); + assertFalse("Snapshot future has finished", restoreFut.isDone()); - assertEquals("Unexpected exit code", EXIT_CODE_OK, code); - assertContains(log, testOut.toString(), "Restore snapshot operation is in progress."); - } - finally { - snapshotCheckLatch.countDown(); + int code = execute("--snapshot", "status"); - // Wait for future to finish in order to avoid excessive message about task cancellation. - restoreFut.get(); - } + assertEquals("Unexpected exit code", EXIT_CODE_OK, code); + assertContains(log, testOut.toString(), "Restore snapshot operation is in progress."); + + assertTrue(proceedCheckLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + + // Wait for future to finish in order to avoid excessive message about task cancellation. + restoreFut.get(); } /** */ - private static class BlockingCheckSnapshotPluginProvider extends AbstractTestPluginProvider { - /** {@inheritDoc} */ - @Override public String name() { - return "BlockingCheckSnapshotPluginProvider"; - } + private class TestCommunicationSpi extends TcpCommunicationSpi { + /** */ + private volatile @Nullable Consumer msgCsmr; /** {@inheritDoc} */ - @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { - if (IgniteSnapshotManager.class.equals(cls)) - return (T)new BlockingCheckSnapshotManager(((IgniteEx)ctx.grid()).context()); + @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException { + var msgCsmr = this.msgCsmr; - return null; - } - } + if (msgCsmr != null) + msgCsmr.accept(msg); - /** Blocks restore proccess. */ - protected static class BlockingCheckSnapshotManager extends IgniteSnapshotManager { - /** */ - public BlockingCheckSnapshotManager(GridKernalContext ctx) { - super(ctx); + super.sendMessage(node, msg); } /** {@inheritDoc} */ - @Override public IgniteInternalFuture checkSnapshot( - String name, - @Nullable String snpPath, - @Nullable Collection grps, - boolean includeCustomHandlers, - int incIdx, boolean check - ) { - return super.checkSnapshot(name, snpPath, grps, includeCustomHandlers, incIdx, check) - .chain(fut -> { - try { - if(check) - snapshotCheckLatch.await(5, MINUTES); - } - catch (Throwable e) { - log.error(X.getFullStackTrace(e)); - } + @Override public void sendMessage( + ClusterNode node, + Message msg, + IgniteInClosure ackC + ) throws IgniteSpiException { + var msgCsmr = this.msgCsmr; + + if (msgCsmr != null) + msgCsmr.accept(msg); - return fut.result(); - }); + super.sendMessage(node, msg, ackC); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java index 6551e4ac48883..10cdcf343c57e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java @@ -28,7 +28,7 @@ * Command that executed with some compute task. */ public interface ComputeCommand extends Command { - /** @return Default task class. */ + /** @return Task class. */ public Class> taskClass(); /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 66212036030e7..9ff10c7121926 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -72,7 +72,7 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand buildRow(UUID nodeId, T5 progress) { - long processed = progress.get1(); - long total = progress.get2(); + if (inc) { + long processed = progress.get1(); + long total = progress.get2(); - if (total <= 0) - return F.asList(nodeId, "unknown", "unknown", "unknown"); + if (total <= 0) + return F.asList(nodeId, "unknown", "unknown", "unknown"); - String percent = (int)(processed * 100 / total) + "%"; + String percent = (int)(processed * 100 / total) + "%"; - return F.asList(nodeId, U.humanReadableByteCount(processed), U.humanReadableByteCount(total), percent); + return F.asList(nodeId, U.humanReadableByteCount(processed), U.humanReadableByteCount(total), percent); + } + + long partitionsToCheck = progress.get3(); + long partsToCheck = progress.get5(); + + if (partitionsToCheck <= 0 || partsToCheck <= 0) + return F.asList(nodeId, "unknown", "unknown", "unknown", "unknown", "unknown", "unknown"); + + // Checked partitions in current part ratio * shapshot parts ratio. + double totalRatio = ((double)progress.get2() / partitionsToCheck) * ((double)progress.get4() / partsToCheck); + + return F.asList( + nodeId, + progress.get1() == 0L ? "false" : "true", // Full check flag; + U.humanReadableByteCount(progress.get2()), // Checked partitions in current snapshot part; + U.humanReadableByteCount(partitionsToCheck), // Total partitions in current snapshot part; + U.humanReadableByteCount(progress.get4()), // Checked shapshot parts; + U.humanReadableByteCount(partsToCheck), // Total snapshot parts to check. + ((int)(totalRatio * 100.0d)) + '%' + ); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 116e0d04b17d2..bab1af847cb23 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -383,7 +383,7 @@ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { metrics = new T5<>( snpCheckMReg.findMetric("checkPartitions").value() ? 1L : 0L, (long)snpCheckMReg.findMetric("processedPartitions").value(), - (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), + (long)snpCheckMReg.findMetric("totalPartitions").value(), (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), (long)snpCheckMReg.findMetric("snapshotPartsToProcess").value() ); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index f069919248f91..6ee811b8858c8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -724,6 +724,8 @@ private boolean baseline(UUID nodeId) { /** */ private void registerMetrics(SnapshotCheckContext ctx) { + log.error("TEST | registerMetrics"); + MetricRegistryImpl mreg = kctx.metric().registry(MetricUtils.metricName(SNAPSHOT_CHECK_METRIC, ctx.req.snapshotName())); assert !mreg.iterator().hasNext(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java index fc6d5943bbee1..5c9d9cb276ef8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java @@ -22,6 +22,7 @@ import org.apache.ignite.internal.Order; import org.apache.ignite.internal.util.ErrorMessage; import org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType; +import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageFactory; import org.jetbrains.annotations.Nullable; @@ -99,4 +100,9 @@ public boolean hasError() { @Nullable public Throwable error() { return ErrorMessage.error(errMsg); } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(SingleNodeMessage.class, this); + } } From dfe63eec11070e844bd2384e106e27af49de04b8 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 1 Sep 2026 15:40:03 +0300 Subject: [PATCH 04/44] raw --- .../SnapshotStatusReproducerTest.java | 158 +++++++------- .../management/api/ComputeCommand.java | 2 +- .../snapshot/SnapshotStatusCommand.java | 136 ++++++++---- .../snapshot/SnapshotStatusTask.java | 206 ++++++++++++------ .../util/distributed/SingleNodeMessage.java | 6 + 5 files changed, 323 insertions(+), 185 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java index 1de6947a6720b..f70a56da143aa 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java @@ -1,40 +1,39 @@ package org.apache.ignite.internal.commandline.snapshot; -import java.util.Collection; +import java.util.Arrays; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.stream.IntStream; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.cluster.ClusterState; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.GridKernalContext; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; -import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotPartitionsVerifyResult; +import org.apache.ignite.internal.util.distributed.SingleNodeMessage; import org.apache.ignite.internal.util.future.IgniteFutureImpl; -import org.apache.ignite.internal.util.typedef.X; -import org.apache.ignite.plugin.AbstractTestPluginProvider; -import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.util.GridCommandHandlerAbstractTest; import org.jetbrains.annotations.Nullable; import org.junit.Test; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.MINUTES; -import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_PARTS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; import static org.apache.ignite.testframework.GridTestUtils.assertContains; /** */ public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest { - /** Snapshot check latch. */ - private static CountDownLatch snapshotCheckLatch; - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - return super.getConfiguration(igniteInstanceName) - .setPluginProviders(new BlockingCheckSnapshotPluginProvider()) - .setIncludeEventTypes(EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED); + return super.getConfiguration(igniteInstanceName).setCommunicationSpi(new TestCommunicationSpi()); } /** {@inheritDoc} */ @@ -45,8 +44,6 @@ public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest cleanPersistenceDir(); - snapshotCheckLatch = new CountDownLatch(1); - startGrids(3); grid(0).cluster().state(ClusterState.ACTIVE); @@ -72,91 +69,94 @@ public void test() throws Exception { IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); - String snapshotName = "test_snapshot"; - - snapshotMgr.createSnapshot(snapshotName).get(getTestTimeout()); + snapshotMgr.createSnapshot("test_snapshot").get(getTestTimeout()); grid(0).destroyCache(DEFAULT_CACHE_NAME); - CountDownLatch restoreStarterdLatch = new CountDownLatch(1); + awaitPartitionMapExchange(); + + var checkSingleResultsReceivedLatch = new CountDownLatch(2); + var restoreSingleResultsReceivedLatch = new AtomicInteger(2); + var proceedCheckLatch = new CountDownLatch(1); + + for (var ig : Arrays.asList(grid(1), grid(2))) { + ((TestCommunicationSpi)ig.configuration().getCommunicationSpi()).msgCsmr = msg -> { + if (!(msg instanceof GridIoMessage ioMsg)) + return; + + if (!(ioMsg.message() instanceof SingleNodeMessage sm)) + return; - grid(0).events().localListen( - e -> { - restoreStarterdLatch.countDown(); + if (sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) + restoreSingleResultsReceivedLatch.decrementAndGet(); + else if (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()) { + checkSingleResultsReceivedLatch.countDown(); - return false; - }, - EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED - ); + try { + assertTrue(proceedCheckLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }; + } - IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot(snapshotName, null, null, 0, true); + IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot("test_snapshot", null, null, 0, true); - try { - restoreStarterdLatch.await(getTestTimeout(), MILLISECONDS); - assertFalse("Snapshot future has finished", restoreFut.isDone()); + assertTrue(checkSingleResultsReceivedLatch.await(getTestTimeout(), MILLISECONDS)); - int code = execute("--snapshot", "restore", snapshotName, "--status"); + // Make sure no restoration started or finished. + assertTrue(restoreSingleResultsReceivedLatch.get() > 0); + assertFalse("Snapshot future has finished", restoreFut.isDone()); - assertEquals("Unexpected exit code", EXIT_CODE_OK, code); + int code = execute("--snapshot", "status"); - assertFalse("Snapshot future has finished", restoreFut.isDone()); + // Ensures that there is a status despite unstarted restore process. + assertEquals("Unexpected exit code", EXIT_CODE_OK, code); - code = execute("--snapshot", "status"); + var out = testOut.toString(); - assertEquals("Unexpected exit code", EXIT_CODE_OK, code); - assertContains(log, testOut.toString(), "Restore snapshot operation is in progress."); - } - finally { - snapshotCheckLatch.countDown(); + assertContains(log, out, "Check snapshot operation is in progress"); + assertContains(log, out, "Snapshot name: test_snapshot"); + assertContains(log, out, "Incremental: false"); + assertContains(log, out, "Estimated operation progress:"); - // Wait for future to finish in order to avoid excessive message about task cancellation. - restoreFut.get(); - } + proceedCheckLatch.countDown(); + + // Wait for future to finish in order to avoid excessive message about task cancellation. + restoreFut.get(); + + assertTrue(restoreSingleResultsReceivedLatch.get() == 0); } /** */ - private static class BlockingCheckSnapshotPluginProvider extends AbstractTestPluginProvider { - /** {@inheritDoc} */ - @Override public String name() { - return "BlockingCheckSnapshotPluginProvider"; - } + private static class TestCommunicationSpi extends TcpCommunicationSpi { + /** */ + private volatile @Nullable Consumer msgCsmr; /** {@inheritDoc} */ - @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { - if (IgniteSnapshotManager.class.equals(cls)) - return (T)new BlockingCheckSnapshotManager(((IgniteEx)ctx.grid()).context()); + @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException { + var msgCsmr = this.msgCsmr; - return null; - } - } + if (msgCsmr != null) + msgCsmr.accept(msg); - /** Blocks restore proccess. */ - protected static class BlockingCheckSnapshotManager extends IgniteSnapshotManager { - /** */ - public BlockingCheckSnapshotManager(GridKernalContext ctx) { - super(ctx); + super.sendMessage(node, msg); } /** {@inheritDoc} */ - @Override public IgniteInternalFuture checkSnapshot( - String name, - @Nullable String snpPath, - @Nullable Collection grps, - boolean includeCustomHandlers, - int incIdx, boolean check - ) { - return super.checkSnapshot(name, snpPath, grps, includeCustomHandlers, incIdx, check) - .chain(fut -> { - try { - if(check) - snapshotCheckLatch.await(5, MINUTES); - } - catch (Throwable e) { - log.error(X.getFullStackTrace(e)); - } + @Override public void sendMessage( + ClusterNode node, + Message msg, + IgniteInClosure ackC + ) throws IgniteSpiException { + var msgCsmr = this.msgCsmr; + + if (msgCsmr != null) + msgCsmr.accept(msg); - return fut.result(); - }); + super.sendMessage(node, msg, ackC); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java index 6551e4ac48883..10cdcf343c57e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/api/ComputeCommand.java @@ -28,7 +28,7 @@ * Command that executed with some compute task. */ public interface ComputeCommand extends Command { - /** @return Default task class. */ + /** @return Task class. */ public Class> taskClass(); /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 66212036030e7..1fd77f760c4fe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.stream.Collectors; import org.apache.ignite.internal.management.SystemViewCommand; @@ -64,53 +65,72 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand 0; + boolean creating = SnapshotStatusTask.SnapshotOperation.CREATE == status.operation(); + boolean restoring = SnapshotStatusTask.SnapshotOperation.RESTORE == status.operation(); + boolean inc = status.incrementIndex() > 0; - GridStringBuilder s = new GridStringBuilder(); + assert (status instanceof SnapshotStatusTask.SnapshotStatusV2) == !(creating || restoring); - if (isCreating) - s.a("Create snapshot operation is in progress.").nl(); - else if(isRestoring) - s.a("Restore snapshot operation is in progress.").nl(); - else - s.a("Check snapshot operation is in progress.").nl(); + // The check oeration can be run in parallel for different snapshots. + List multipleOpsView = creating || restoring + ? Collections.singletonList(status) + : ((SnapshotStatusTask.SnapshotStatusV2)status).allCheckStatuses; - s.a("Snapshot name: ").a(status.name()).nl(); - s.a("Incremental: ").a(isIncremental).nl(); + // Flag of additional line delimiter. + AtomicBoolean oneOp = new AtomicBoolean(true); - if (isIncremental) - s.a("Increment index: ").a(status.incrementIndex()).nl(); + multipleOpsView.forEach(s0 -> { + GridStringBuilder s = new GridStringBuilder(); - s.a("Operation request ID: ").a(status.requestId()).nl(); - s.a("Started at: ").a(DateFormat.getDateTimeInstance().format(new Date(status.startTime()))).nl(); - s.a("Duration: ").a(X.timeSpan2DHMSM(System.currentTimeMillis() - status.startTime())).nl() - .nl(); - s.a("Estimated operation progress:").nl(); + if (!oneOp.get()) + printer.accept(U.nl()); - printer.accept(s.toString()); + if (creating) { + assert multipleOpsView.size() == 1; - SnapshotTaskProgressDesc desc; + s.a("Create snapshot operation is in progress.").nl(); + } + else if (restoring) { + assert multipleOpsView.size() == 1; + + s.a("Restore snapshot operation is in progress.").nl(); + } + else + s.a("Check snapshot operation" + (multipleOpsView.size() < 2 ? " is " : "s are ") + "in progress.").nl(); - if (isCreating && isIncremental) - desc = new CreateIncrementalSnapshotTaskProgressDesc(); - else if (isCreating) - desc = new CreateFullSnapshotTaskProgressDesc(); - else if (isRestoring && isIncremental) - desc = new RestoreIncrementalSnapshotTaskProgressDesc(); - else if (isRestoring) - desc = new RestoreFullSnapshotTaskProgressDesc(); - else - desc = new CheckSnapshotTaskProgressDesc(isIncremental); + s.a("Snapshot name: ").a(s0.name()).nl(); + s.a("Incremental: ").a(inc).nl(); - List> rows = status.progress().entrySet().stream().sorted(Map.Entry.comparingByKey()) - .map(e -> desc.buildRow(e.getKey(), e.getValue())) - .collect(Collectors.toList()); + if (inc) + s.a("Increment index: ").a(s0.incrementIndex()).nl(); - SystemViewCommand.printTable(desc.titles(), desc.types(), rows, printer); + s.a("Operation request ID: ").a(s0.requestId()).nl(); + s.a("Started at: ").a(DateFormat.getDateTimeInstance().format(new Date(s0.startTime()))).nl(); + s.a("Duration: ").a(X.timeSpan2DHMSM(System.currentTimeMillis() - s0.startTime())).nl() + .nl(); + s.a("Estimated operation progress:").nl(); - printer.accept(U.nl()); + printer.accept(s.toString()); + + SnapshotTaskProgressDesc desc; + + if (creating) + desc = inc ? new CreateIncrementalSnapshotTaskProgressDesc() : new CreateFullSnapshotTaskProgressDesc(); + else if (restoring) + desc = inc ? new RestoreIncrementalSnapshotTaskProgressDesc() : new RestoreFullSnapshotTaskProgressDesc(); + else + desc = new CheckSnapshotTaskProgressDesc(inc); + + List> rows = s0.progress().entrySet().stream().sorted(Map.Entry.comparingByKey()) + .map(e -> desc.buildRow(e.getKey(), e.getValue())) + .collect(Collectors.toList()); + + SystemViewCommand.printTable(desc.titles(), desc.types(), rows, printer); + + printer.accept(U.nl()); + + oneOp.set(false); + }); } /** Describes progress of a snapshot task. */ @@ -254,22 +274,52 @@ private static class RestoreIncrementalSnapshotTaskProgressDesc extends Snapshot /** */ private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { + /** */ + private final boolean inc; + /** */ CheckSnapshotTaskProgressDesc(boolean incremental) { - super(F.asList("Node ID", "Processed, bytes", "Total, bytes", "Percent")); + super(incremental + ? F.asList("Node ID", "processedWalSegments", "totalWalSegments", "percent") + : F.asList("Node ID", "fullCheck", "processedPartitions", "totalPartitions", + "processedSnapshotParts", "snapshotPartsToProcess", "percent") + ); + + inc = incremental; } /** {@inheritDoc} */ @Override public List buildRow(UUID nodeId, T5 progress) { - long processed = progress.get1(); - long total = progress.get2(); + if (inc) { + long processed = progress.get1(); + long total = progress.get2(); - if (total <= 0) - return F.asList(nodeId, "unknown", "unknown", "unknown"); + if (total <= 0) + return F.asList(nodeId, "unknown", "unknown", "unknown"); - String percent = (int)(processed * 100 / total) + "%"; + String percent = (int)(processed * 100 / total) + "%"; - return F.asList(nodeId, U.humanReadableByteCount(processed), U.humanReadableByteCount(total), percent); + return F.asList(nodeId, processed, total, percent); + } + + long partitionsToCheck = progress.get3(); + long partsToCheck = progress.get5(); + + if (partitionsToCheck <= 0 || partsToCheck <= 0) + return F.asList(nodeId, "unknown", "unknown", "unknown", "unknown", "unknown", "unknown"); + + // Ration of checked partitions in current snapshot part * total parts ratio. + double totalRatio = ((double)progress.get2() / partitionsToCheck) * ((double)progress.get4() / partsToCheck); + + return F.asList( + nodeId, + progress.get1() == 0L ? "false" : "true", // Full check flag; + progress.get2(), // Checked partitions in current snapshot part; + partitionsToCheck, // Total partitions in current snapshot part; + progress.get4(), // Checked shapshot parts; + partsToCheck, // Total snapshot parts to check. + ((int)(totalRatio * 100.0d)) + '%' + ); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 116e0d04b17d2..02775544c1b9e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -20,7 +20,6 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; @@ -61,17 +60,25 @@ */ @GridInternal public class SnapshotStatusTask extends VisorMultiNodeTask { - /** */ + /** + * + */ private static final long serialVersionUID = 0L; - /** */ + /** + * + */ @LoggerResource private transient IgniteLogger log; - /** */ + /** + * + */ private transient @Nullable Boolean checkStatusSupported; - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override protected VisorJob job(NoArg arg) { if (checkStatusSupported == null) resolveCheckStatusSupported(); @@ -81,7 +88,9 @@ public class SnapshotStatusTask extends VisorMultiNodeTask jobNodes(VisorTaskArgument arg) { return nodeIds(ignite.cluster().forServers().nodes()); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override protected @Nullable SnapshotStatus reduce0(List results) { if (results.isEmpty()) throw new IgniteException("Failed to get the snapshot status. Topology is empty."); @@ -155,22 +168,29 @@ private void resolveCheckStatusSupported() { // Filter out differing requests due to concurrent updates on nodes. Collection sameRqRes = F.view(res0, s -> s.reqId.equals(firstRes.reqId)); - if (firstRes instanceof SnapshotStatusTask.SnapshotStatusV2) { - var statusV2 = (SnapshotStatusTask.SnapshotStatusV2)firstRes; + if (firstRes instanceof SnapshotStatusV2 firstResV2) { + assert !F.isEmpty(firstResV2.allCheckStatuses); - assert !F.isEmpty(statusV2.allCheckStatuses); - - Map> mergedAllCheckStatuses = U.newHashMap(sameRqRes.size()); + // Check status: snpName, per node collection. + Map statusesMap = U.newHashMap(sameRqRes.size()); sameRqRes.forEach(s -> { assert s instanceof SnapshotStatusTask.SnapshotStatusV2; - mergedAllCheckStatuses.putAll(((SnapshotStatusTask.SnapshotStatusV2)s).allCheckStatuses); - }); + for (SnapshotStatus s0 : ((SnapshotStatusTask.SnapshotStatusV2)s).allCheckStatuses) { + var prev = statusesMap.putIfAbsent(s0.name, s0); + + if (prev == null) + continue; + + // Merge nodes progress. + prev.progress().putAll(s0.progress()); + } - statusV2.allCheckStatuses = mergedAllCheckStatuses; + firstResV2.allCheckStatuses = new ArrayList<>(statusesMap.values()); + }); - return statusV2; + return firstResV2; } // Merge nodes progress. @@ -181,20 +201,26 @@ private void resolveCheckStatusSupported() { return new SnapshotStatus(firstRes.op, firstRes.name, firstRes.incIdx, firstRes.reqId, firstRes.startTime, mergedProgress); } - /** */ + /** + * + */ private static class SnapshotStatusJob extends SnapshotJob { - /** */ + /** + * + */ private static final long serialVersionUID = 0L; /** - * @param arg Job argument. + * @param arg Job argument. * @param debug Flag indicating whether debug information should be printed into node log. */ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { super(arg, debug); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { if (!CU.isPersistenceEnabled(ignite.context().config())) return null; @@ -258,30 +284,48 @@ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { } } - /** Snapshot operation status. */ - static class SnapshotStatus implements Serializable { - /** */ + /** + * Snapshot operation status. + */ + public static class SnapshotStatus implements Serializable { + /** + * + */ private static final long serialVersionUID = 0L; - /** Operation type. {@code Null} for other operation types. */ + /** + * Operation type. {@code Null} for other operation types. + */ private final @Nullable SnapshotOperation op; - /** Snapshot name. */ + /** + * Snapshot name. + */ private final String name; - /** Incremental snapshot index. */ + /** + * Incremental snapshot index. + */ private final int incIdx; - /** Request ID. */ + /** + * Request ID. + */ private final String reqId; - /** Start time. */ + /** + * Start time. + */ private final long startTime; - /** Progress of operation on nodes. */ + /** + * Progress of operation on nodes. + */ private final Map> progress; - /** */ + /** + * + */ private SnapshotStatus( @Nullable SnapshotOperation op, String name, @@ -295,60 +339,86 @@ private SnapshotStatus( this.incIdx = incIdx; this.reqId = reqId; this.startTime = startTime; - this.progress = Collections.unmodifiableMap(progress); + this.progress = progress; } - /** @return Operation type. {@code Null} for other operation types. */ + /** + * @return Operation type. {@code Null} for other operation types. + */ @Nullable SnapshotOperation operation() { return op; } - /** @return Snapshot name. */ + /** + * @return Snapshot name. + */ String name() { return name; } - /** @return Incremental snapshot index. */ + /** + * @return Incremental snapshot index. + */ int incrementIndex() { return incIdx; } - /** @return Request ID. */ + /** + * @return Request ID. + */ String requestId() { return reqId; } - /** @return Start time. */ + /** + * @return Start time. + */ long startTime() { return startTime; } - /** @return Progress of operation on nodes. */ + /** + * @return Progress of operation on nodes. + */ Map> progress() { return progress; } } - /** Snapshot operation type. */ + /** + * Snapshot operation type. + */ enum SnapshotOperation { - /** Snapshot creation. */ + /** + * Snapshot creation. + */ CREATE, - /** Snapshot restoration. */ + /** + * Snapshot restoration. + */ RESTORE } - /** */ + /** + * + */ private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { - /** */ + /** + * + */ private static final long serialVersionUID = 0L; - /** */ + /** + * + */ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { super(arg, debug); } - /** {@inheritDoc} */ + /** + * {@inheritDoc} + */ @Override protected @Nullable SnapshotStatusV2 run(@Nullable NoArg arg) throws IgniteException { var res1 = super.run(arg); @@ -383,49 +453,61 @@ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { metrics = new T5<>( snpCheckMReg.findMetric("checkPartitions").value() ? 1L : 0L, (long)snpCheckMReg.findMetric("processedPartitions").value(), - (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), + (long)snpCheckMReg.findMetric("totalPartitions").value(), (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), (long)snpCheckMReg.findMetric("snapshotPartsToProcess").value() ); } - checkStatuses.add(new SnapshotStatus( + var status = new SnapshotStatus( null, MetricUtils.fromFullName(snpCheckMReg.name()).get2(), incIdx, snpCheckMReg.findMetric("requestId").getAsString(), ((LongMetric)snpCheckMReg.findMetric("startTime")).value(), GridFunc.asMap(ignite.localNode().id(), metrics) - )); + ); + + checkStatuses.add(status); } - return checkStatuses == null ? null : new SnapshotStatusV2(Collections.singletonMap(ignite.localNode().id(), checkStatuses)); + return checkStatuses == null ? null : new SnapshotStatusV2(checkStatuses); } } - /** Supports snapsho status. */ - private static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { - /** */ + /** + * Supports snapsho status. + */ + public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { + /** + * + */ private static final long serialVersionUID = 0L; - /** Statuses of snapshot check operations per nodeID. */ - private @Nullable Map> allCheckStatuses; + /** + * Nodes' statuses of all snapshot check operations + */ + @Nullable List allCheckStatuses; - /** */ + /** + * + */ private SnapshotStatusV2(SnapshotStatus s1) { super(s1.op, s1.name, s1.incIdx, s1.reqId, s1.startTime, s1.progress); } - /** */ - private SnapshotStatusV2(Map> allCheckStatuses) { + /** + * + */ + private SnapshotStatusV2(List allCheckStatuses) { // Single, V1 status holds first found check status. super( null, - F.first(allCheckStatuses.values()).get(0).name, - F.first(allCheckStatuses.values()).get(0).incIdx, - F.first(allCheckStatuses.values()).get(0).reqId, - F.first(allCheckStatuses.values()).get(0).startTime, - F.first(allCheckStatuses.values()).get(0).progress + allCheckStatuses.get(0).name, + allCheckStatuses.get(0).incIdx, + allCheckStatuses.get(0).reqId, + allCheckStatuses.get(0).startTime, + allCheckStatuses.get(0).progress ); this.allCheckStatuses = allCheckStatuses; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java index fc6d5943bbee1..5c9d9cb276ef8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java @@ -22,6 +22,7 @@ import org.apache.ignite.internal.Order; import org.apache.ignite.internal.util.ErrorMessage; import org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType; +import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageFactory; import org.jetbrains.annotations.Nullable; @@ -99,4 +100,9 @@ public boolean hasError() { @Nullable public Throwable error() { return ErrorMessage.error(errMsg); } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(SingleNodeMessage.class, this); + } } From 4007c1567107d93e03ee72e40c010de973f331e6 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 1 Sep 2026 18:42:32 +0300 Subject: [PATCH 05/44] test fixes --- .../SnapshotStatusReproducerTest.java | 162 ------------------ .../ignite/util/GridCommandHandlerTest.java | 118 ++++++++++++- .../snapshot/SnapshotCheckProcess.java | 2 - 3 files changed, 117 insertions(+), 165 deletions(-) delete mode 100644 modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java deleted file mode 100644 index f70a56da143aa..0000000000000 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/snapshot/SnapshotStatusReproducerTest.java +++ /dev/null @@ -1,162 +0,0 @@ -package org.apache.ignite.internal.commandline.snapshot; - -import java.util.Arrays; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; -import java.util.stream.IntStream; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.IgniteException; -import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.cluster.ClusterState; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.managers.communication.GridIoMessage; -import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; -import org.apache.ignite.internal.util.distributed.SingleNodeMessage; -import org.apache.ignite.internal.util.future.IgniteFutureImpl; -import org.apache.ignite.lang.IgniteInClosure; -import org.apache.ignite.plugin.extensions.communication.Message; -import org.apache.ignite.spi.IgniteSpiException; -import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; -import org.apache.ignite.util.GridCommandHandlerAbstractTest; -import org.jetbrains.annotations.Nullable; -import org.junit.Test; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; -import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_PARTS; -import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; -import static org.apache.ignite.testframework.GridTestUtils.assertContains; - -/** */ -public class SnapshotStatusReproducerTest extends GridCommandHandlerAbstractTest { - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - return super.getConfiguration(igniteInstanceName).setCommunicationSpi(new TestCommunicationSpi()); - } - - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); - - autoConfirmation = false; - - cleanPersistenceDir(); - - startGrids(3); - - grid(0).cluster().state(ClusterState.ACTIVE); - - IgniteCache cache = grid(0).getOrCreateCache(DEFAULT_CACHE_NAME); - - IntStream.range(0, 2048).forEach(i -> cache.put(i, i)); - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - super.afterTest(); - - stopAllGrids(true); - - cleanPersistenceDir(); - } - - /** */ - @Test - public void test() throws Exception { - injectTestSystemOut(); - - IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); - - snapshotMgr.createSnapshot("test_snapshot").get(getTestTimeout()); - - grid(0).destroyCache(DEFAULT_CACHE_NAME); - - awaitPartitionMapExchange(); - - var checkSingleResultsReceivedLatch = new CountDownLatch(2); - var restoreSingleResultsReceivedLatch = new AtomicInteger(2); - var proceedCheckLatch = new CountDownLatch(1); - - for (var ig : Arrays.asList(grid(1), grid(2))) { - ((TestCommunicationSpi)ig.configuration().getCommunicationSpi()).msgCsmr = msg -> { - if (!(msg instanceof GridIoMessage ioMsg)) - return; - - if (!(ioMsg.message() instanceof SingleNodeMessage sm)) - return; - - if (sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) - restoreSingleResultsReceivedLatch.decrementAndGet(); - else if (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()) { - checkSingleResultsReceivedLatch.countDown(); - - try { - assertTrue(proceedCheckLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - }; - } - - IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot("test_snapshot", null, null, 0, true); - - assertTrue(checkSingleResultsReceivedLatch.await(getTestTimeout(), MILLISECONDS)); - - // Make sure no restoration started or finished. - assertTrue(restoreSingleResultsReceivedLatch.get() > 0); - assertFalse("Snapshot future has finished", restoreFut.isDone()); - - int code = execute("--snapshot", "status"); - - // Ensures that there is a status despite unstarted restore process. - assertEquals("Unexpected exit code", EXIT_CODE_OK, code); - - var out = testOut.toString(); - - assertContains(log, out, "Check snapshot operation is in progress"); - assertContains(log, out, "Snapshot name: test_snapshot"); - assertContains(log, out, "Incremental: false"); - assertContains(log, out, "Estimated operation progress:"); - - proceedCheckLatch.countDown(); - - // Wait for future to finish in order to avoid excessive message about task cancellation. - restoreFut.get(); - - assertTrue(restoreSingleResultsReceivedLatch.get() == 0); - } - - /** */ - private static class TestCommunicationSpi extends TcpCommunicationSpi { - /** */ - private volatile @Nullable Consumer msgCsmr; - - /** {@inheritDoc} */ - @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException { - var msgCsmr = this.msgCsmr; - - if (msgCsmr != null) - msgCsmr.accept(msg); - - super.sendMessage(node, msg); - } - - /** {@inheritDoc} */ - @Override public void sendMessage( - ClusterNode node, - Message msg, - IgniteInClosure ackC - ) throws IgniteSpiException { - var msgCsmr = this.msgCsmr; - - if (msgCsmr != null) - msgCsmr.accept(msg); - - super.sendMessage(node, msg, ackC); - } - } -} diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index b5f2659b1fa0b..70496ce45b127 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -48,6 +48,7 @@ import java.util.function.BooleanSupplier; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -121,6 +122,7 @@ import org.apache.ignite.internal.util.distributed.DistributedProcess; import org.apache.ignite.internal.util.distributed.SingleNodeMessage; import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.lang.GridFunc; import org.apache.ignite.internal.util.typedef.F; @@ -135,6 +137,7 @@ import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.metric.MetricRegistry; import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.spi.metric.Metric; @@ -148,10 +151,12 @@ import org.apache.ignite.transactions.TransactionRollbackException; import org.apache.ignite.transactions.TransactionTimeoutException; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.junit.Assume; import org.junit.Test; import static java.io.File.separatorChar; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.ignite.IgniteSystemProperties.IGNITE_CLUSTER_NAME; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -182,6 +187,7 @@ import static org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor.DEFAULT_TARGET_FOLDER; import static org.apache.ignite.internal.processors.job.GridJobProcessor.JOBS_VIEW; import static org.apache.ignite.internal.processors.task.GridTaskProcessor.TASKS_VIEW; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_PARTS; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; import static org.apache.ignite.testframework.GridTestUtils.assertContains; import static org.apache.ignite.testframework.GridTestUtils.assertNotContains; @@ -233,13 +239,16 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb /** */ protected ListeningTestLogger listeningLog; + /** */ + protected @Nullable Supplier communicationSpiSupp; + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); initDiagnosticDir(); - cleanDiagnosticDir(); + cleanPersistenceDir(); } /** {@inheritDoc} */ @@ -263,6 +272,9 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb if (listeningLog != null) cfg.setGridLogger(listeningLog); + if (communicationSpiSupp != null) + cfg.setCommunicationSpi(communicationSpiSupp.get()); + return cfg; } @@ -3658,6 +3670,80 @@ public void testSnapshotRestoreCancelAndStatus() throws Exception { assertNull(ig.cache(DEFAULT_CACHE_NAME)); } + /** Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. */ + @Test + public void testRestoreSnapshotMetricsAtStart() throws Exception { + communicationSpiSupp = TestCommunicationSpi::new; + + startGrids(3).cluster().state(ClusterState.ACTIVE); + + createCacheAndPreload(grid(1), 8192); + + IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); + + snapshotMgr.createSnapshot("test_snapshot").get(getTestTimeout()); + + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + + var checkSingleResultsReceivedLatch = new CountDownLatch(2); + var restoreSingleResultsReceivedLatch = new AtomicInteger(2); + var proceedCheckLatch = new CountDownLatch(1); + + for (var ig : Arrays.asList(grid(1), grid(2))) { + ((TestCommunicationSpi)ig.configuration().getCommunicationSpi()).msgCsmr = msg -> { + if (!(msg instanceof GridIoMessage ioMsg)) + return; + + if (!(ioMsg.message() instanceof SingleNodeMessage sm)) + return; + + if (sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) + restoreSingleResultsReceivedLatch.decrementAndGet(); + else if (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()) { + checkSingleResultsReceivedLatch.countDown(); + + try { + assertTrue(proceedCheckLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }; + } + + IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot("test_snapshot", null, null, 0, true); + + assertTrue(checkSingleResultsReceivedLatch.await(getTestTimeout(), MILLISECONDS)); + + // Make sure no restoration started or finished. + assertTrue(restoreSingleResultsReceivedLatch.get() == 2); + assertFalse("Snapshot future has finished", restoreFut.isDone()); + + injectTestSystemOut(); + + int code = execute("--snapshot", "status"); + + // Ensures that there is a status despite unstarted restore process. + assertEquals("Unexpected exit code", EXIT_CODE_OK, code); + + var out = testOut.toString(); + + assertContains(log, out, "Check snapshot operation is in progress"); + assertContains(log, out, "Snapshot name: test_snapshot"); + assertContains(log, out, "Incremental: false"); + assertContains(log, out, "Estimated operation progress:"); + + proceedCheckLatch.countDown(); + + // Wait for future to finish in order to avoid excessive message about task cancellation. + restoreFut.get(); + + assertTrue(restoreSingleResultsReceivedLatch.get() == 0); + } + /** @throws Exception If fails. */ @Test public void testSnapshotStatusInMemory() throws Exception { @@ -4066,4 +4152,34 @@ public void input(String input) { this.input = input; } } + + /** */ + private static class TestCommunicationSpi extends TcpCommunicationSpi { + /** */ + private volatile @Nullable Consumer msgCsmr; + + /** {@inheritDoc} */ + @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException { + var msgCsmr = this.msgCsmr; + + if (msgCsmr != null) + msgCsmr.accept(msg); + + super.sendMessage(node, msg); + } + + /** {@inheritDoc} */ + @Override public void sendMessage( + ClusterNode node, + Message msg, + IgniteInClosure ackC + ) throws IgniteSpiException { + var msgCsmr = this.msgCsmr; + + if (msgCsmr != null) + msgCsmr.accept(msg); + + super.sendMessage(node, msg, ackC); + } + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index 6ee811b8858c8..f069919248f91 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -724,8 +724,6 @@ private boolean baseline(UUID nodeId) { /** */ private void registerMetrics(SnapshotCheckContext ctx) { - log.error("TEST | registerMetrics"); - MetricRegistryImpl mreg = kctx.metric().registry(MetricUtils.metricName(SNAPSHOT_CHECK_METRIC, ctx.req.snapshotName())); assert !mreg.iterator().hasNext(); From e8d79ea619188d4364e73ce7f2655dd4f6ad5540 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 1 Sep 2026 19:04:13 +0300 Subject: [PATCH 06/44] test fixes --- .../ignite/util/GridCommandHandlerTest.java | 2 +- .../snapshot/SnapshotStatusTask.java | 155 +++++------------- .../snapshot/SnapshotRestoreStatusTask.java | 2 +- 3 files changed, 42 insertions(+), 117 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 70496ce45b127..f5b6de0aea469 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -248,7 +248,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb initDiagnosticDir(); - cleanPersistenceDir(); + cleanDiagnosticDir(); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 02775544c1b9e..81f89753bc705 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -60,37 +60,25 @@ */ @GridInternal public class SnapshotStatusTask extends VisorMultiNodeTask { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * - */ + /** */ @LoggerResource private transient IgniteLogger log; - /** - * - */ + /** */ private transient @Nullable Boolean checkStatusSupported; - /** - * {@inheritDoc} - */ + /** @inheritDoc} */ @Override protected VisorJob job(NoArg arg) { if (checkStatusSupported == null) resolveCheckStatusSupported(); - assert checkStatusSupported != null; - return checkStatusSupported ? new SnapshotStatusJobV2(arg, debug) : new SnapshotStatusJob(arg, debug); } - /** - * - */ + /** */ private void resolveCheckStatusSupported() { var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); @@ -137,16 +125,12 @@ private void resolveCheckStatusSupported() { checkStatusSupported = true; } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override protected Collection jobNodes(VisorTaskArgument arg) { return nodeIds(ignite.cluster().forServers().nodes()); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override protected @Nullable SnapshotStatus reduce0(List results) { if (results.isEmpty()) throw new IgniteException("Failed to get the snapshot status. Topology is empty."); @@ -201,13 +185,9 @@ private void resolveCheckStatusSupported() { return new SnapshotStatus(firstRes.op, firstRes.name, firstRes.incIdx, firstRes.reqId, firstRes.startTime, mergedProgress); } - /** - * - */ + /** */ private static class SnapshotStatusJob extends SnapshotJob { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; /** @@ -218,9 +198,7 @@ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { super(arg, debug); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { if (!CU.isPersistenceEnabled(ignite.context().config())) return null; @@ -284,48 +262,30 @@ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { } } - /** - * Snapshot operation status. - */ + /** Snapshot operation status. */ public static class SnapshotStatus implements Serializable { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * Operation type. {@code Null} for other operation types. - */ + /** Operation type. {@code Null} for other operation types. */ private final @Nullable SnapshotOperation op; - /** - * Snapshot name. - */ + /** Snapshot name. */ private final String name; - /** - * Incremental snapshot index. - */ + /** Incremental snapshot index. */ private final int incIdx; - /** - * Request ID. - */ + /** Request ID. */ private final String reqId; - /** - * Start time. - */ + /**vStart time. */ private final long startTime; - /** - * Progress of operation on nodes. - */ + /** Progress of operation on nodes. */ private final Map> progress; - /** - * - */ + /** */ private SnapshotStatus( @Nullable SnapshotOperation op, String name, @@ -342,88 +302,63 @@ private SnapshotStatus( this.progress = progress; } - /** - * @return Operation type. {@code Null} for other operation types. - */ + /** @return Operation type. {@code Null} for other operation types. */ @Nullable SnapshotOperation operation() { return op; } - /** - * @return Snapshot name. - */ + /** @return Snapshot name. */ String name() { return name; } - /** - * @return Incremental snapshot index. - */ + /** @return Incremental snapshot index. */ int incrementIndex() { return incIdx; } - /** - * @return Request ID. - */ + /** @return Request ID. */ String requestId() { return reqId; } - /** - * @return Start time. - */ + /** @return Start time. */ long startTime() { return startTime; } - /** - * @return Progress of operation on nodes. - */ + /** @return Progress of operation on nodes. */ Map> progress() { return progress; } } - /** - * Snapshot operation type. - */ + /** Snapshot operation type. */ enum SnapshotOperation { - /** - * Snapshot creation. - */ + /** Snapshot creation. */ CREATE, - /** - * Snapshot restoration. - */ + /** Snapshot restoration. */ RESTORE } - /** - * - */ + /** */ private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * - */ + /** */ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { super(arg, debug); } - /** - * {@inheritDoc} - */ - @Override protected @Nullable SnapshotStatusV2 run(@Nullable NoArg arg) throws IgniteException { + /** {@inheritDoc} */ + @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { var res1 = super.run(arg); + // Create or restore status detected. if (res1 != null) - return new SnapshotStatusV2(res1); + return res1; List checkStatuses = null; @@ -475,30 +410,20 @@ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { } } - /** - * Supports snapsho status. - */ + /** Supports snapsho status. */ public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * Nodes' statuses of all snapshot check operations - */ + /** Nodes' statuses of all snapshot check operations. */ @Nullable List allCheckStatuses; - /** - * - */ + /** */ private SnapshotStatusV2(SnapshotStatus s1) { super(s1.op, s1.name, s1.incIdx, s1.reqId, s1.startTime, s1.progress); } - /** - * - */ + /** */ private SnapshotStatusV2(List allCheckStatuses) { // Single, V1 status holds first found check status. super( diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java index 6df4081be8c29..14a395779c9d4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java @@ -40,7 +40,7 @@ */ @GridInternal @Deprecated -public class SnapshotRestoreStatusTask extends ComputeTaskAdapter { +class SnapshotRestoreStatusTask extends ComputeTaskAdapter { /** Serial version uid. */ private static final long serialVersionUID = 0L; From a57d8004f1842055cc2b2715257e2a93b145d60b Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 1 Sep 2026 19:04:13 +0300 Subject: [PATCH 07/44] test fixes --- .../ignite/util/GridCommandHandlerTest.java | 2 +- .../snapshot/SnapshotStatusCommand.java | 40 ++--- .../snapshot/SnapshotStatusTask.java | 157 +++++------------- .../snapshot/SnapshotRestoreProcess.java | 2 +- .../snapshot/SnapshotRestoreStatusTask.java | 2 +- 5 files changed, 64 insertions(+), 139 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 70496ce45b127..f5b6de0aea469 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -248,7 +248,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb initDiagnosticDir(); - cleanPersistenceDir(); + cleanDiagnosticDir(); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 1fd77f760c4fe..819aaa585d1c2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -53,7 +53,7 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand taskClass() { + @Override public Class taskClass() { return SnapshotStatusTask.class; } @@ -65,14 +65,14 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand 0; + boolean isCreating = SnapshotStatusTask.SnapshotOperation.CREATE == status.operation(); + boolean isRestoring = SnapshotStatusTask.SnapshotOperation.RESTORE == status.operation(); + boolean isIncremental = status.incrementIndex() > 0; - assert (status instanceof SnapshotStatusTask.SnapshotStatusV2) == !(creating || restoring); + assert (status instanceof SnapshotStatusTask.SnapshotStatusV2) == !(isCreating || isRestoring); - // The check oeration can be run in parallel for different snapshots. - List multipleOpsView = creating || restoring + // The check operation can be run in parallel for different snapshots. + List multipleOpsView = isCreating || isRestoring ? Collections.singletonList(status) : ((SnapshotStatusTask.SnapshotStatusV2)status).allCheckStatuses; @@ -85,12 +85,12 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand> rows = s0.progress().entrySet().stream().sorted(Map.Entry.comparingByKey()) .map(e -> desc.buildRow(e.getKey(), e.getValue())) @@ -275,7 +275,7 @@ private static class RestoreIncrementalSnapshotTaskProgressDesc extends Snapshot /** */ private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { /** */ - private final boolean inc; + private final boolean incremental; /** */ CheckSnapshotTaskProgressDesc(boolean incremental) { @@ -285,12 +285,12 @@ private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressD "processedSnapshotParts", "snapshotPartsToProcess", "percent") ); - inc = incremental; + this.incremental = incremental; } /** {@inheritDoc} */ @Override public List buildRow(UUID nodeId, T5 progress) { - if (inc) { + if (incremental) { long processed = progress.get1(); long total = progress.get2(); @@ -308,7 +308,7 @@ private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressD if (partitionsToCheck <= 0 || partsToCheck <= 0) return F.asList(nodeId, "unknown", "unknown", "unknown", "unknown", "unknown", "unknown"); - // Ration of checked partitions in current snapshot part * total parts ratio. + // Ratio of checked partitions in current snapshot part * total parts ratio. double totalRatio = ((double)progress.get2() / partitionsToCheck) * ((double)progress.get4() / partsToCheck); return F.asList( diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 02775544c1b9e..9ce1fe5613989 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -60,37 +60,25 @@ */ @GridInternal public class SnapshotStatusTask extends VisorMultiNodeTask { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * - */ + /** */ @LoggerResource private transient IgniteLogger log; - /** - * - */ + /** */ private transient @Nullable Boolean checkStatusSupported; - /** - * {@inheritDoc} - */ + /** @inheritDoc} */ @Override protected VisorJob job(NoArg arg) { if (checkStatusSupported == null) resolveCheckStatusSupported(); - assert checkStatusSupported != null; - return checkStatusSupported ? new SnapshotStatusJobV2(arg, debug) : new SnapshotStatusJob(arg, debug); } - /** - * - */ + /** */ private void resolveCheckStatusSupported() { var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); @@ -137,16 +125,12 @@ private void resolveCheckStatusSupported() { checkStatusSupported = true; } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override protected Collection jobNodes(VisorTaskArgument arg) { return nodeIds(ignite.cluster().forServers().nodes()); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override protected @Nullable SnapshotStatus reduce0(List results) { if (results.isEmpty()) throw new IgniteException("Failed to get the snapshot status. Topology is empty."); @@ -201,26 +185,20 @@ private void resolveCheckStatusSupported() { return new SnapshotStatus(firstRes.op, firstRes.name, firstRes.incIdx, firstRes.reqId, firstRes.startTime, mergedProgress); } - /** - * - */ + /** */ private static class SnapshotStatusJob extends SnapshotJob { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; /** - * @param arg Job argument. + * @param arg Job argument. * @param debug Flag indicating whether debug information should be printed into node log. */ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { super(arg, debug); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { if (!CU.isPersistenceEnabled(ignite.context().config())) return null; @@ -284,48 +262,30 @@ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { } } - /** - * Snapshot operation status. - */ + /** Snapshot operation status. */ public static class SnapshotStatus implements Serializable { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * Operation type. {@code Null} for other operation types. - */ + /** Operation type. {@code Null} for other operation types. */ private final @Nullable SnapshotOperation op; - /** - * Snapshot name. - */ + /** Snapshot name. */ private final String name; - /** - * Incremental snapshot index. - */ + /** Incremental snapshot index. */ private final int incIdx; - /** - * Request ID. - */ + /** Request ID. */ private final String reqId; - /** - * Start time. - */ + /** Start time. */ private final long startTime; - /** - * Progress of operation on nodes. - */ + /** Progress of operation on nodes. */ private final Map> progress; - /** - * - */ + /** */ private SnapshotStatus( @Nullable SnapshotOperation op, String name, @@ -342,88 +302,63 @@ private SnapshotStatus( this.progress = progress; } - /** - * @return Operation type. {@code Null} for other operation types. - */ + /** @return Operation type. {@code Null} for other operation types. */ @Nullable SnapshotOperation operation() { return op; } - /** - * @return Snapshot name. - */ + /** @return Snapshot name. */ String name() { return name; } - /** - * @return Incremental snapshot index. - */ + /** @return Incremental snapshot index. */ int incrementIndex() { return incIdx; } - /** - * @return Request ID. - */ + /** @return Request ID. */ String requestId() { return reqId; } - /** - * @return Start time. - */ + /** @return Start time. */ long startTime() { return startTime; } - /** - * @return Progress of operation on nodes. - */ + /** @return Progress of operation on nodes. */ Map> progress() { return progress; } } - /** - * Snapshot operation type. - */ + /** Snapshot operation type. */ enum SnapshotOperation { - /** - * Snapshot creation. - */ + /** Snapshot creation. */ CREATE, - /** - * Snapshot restoration. - */ + /** Snapshot restoration. */ RESTORE } - /** - * - */ + /** */ private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * - */ + /** */ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { super(arg, debug); } - /** - * {@inheritDoc} - */ - @Override protected @Nullable SnapshotStatusV2 run(@Nullable NoArg arg) throws IgniteException { + /** {@inheritDoc} */ + @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { var res1 = super.run(arg); + // Create or restore status detected. if (res1 != null) - return new SnapshotStatusV2(res1); + return res1; List checkStatuses = null; @@ -475,30 +410,20 @@ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { } } - /** - * Supports snapsho status. - */ + /** {@link SnapshotStatus} with support of snapshot check process. */ public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { - /** - * - */ + /** */ private static final long serialVersionUID = 0L; - /** - * Nodes' statuses of all snapshot check operations - */ + /** Nodes' statuses of all snapshot check operations. */ @Nullable List allCheckStatuses; - /** - * - */ + /** */ private SnapshotStatusV2(SnapshotStatus s1) { super(s1.op, s1.name, s1.incIdx, s1.reqId, s1.startTime, s1.progress); } - /** - * - */ + /** */ private SnapshotStatusV2(List allCheckStatuses) { // Single, V1 status holds first found check status. super( diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 50c589407d154..77f56f4d4cc7f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -351,7 +351,7 @@ public IgniteFutureImpl start( .stream() .findFirst(); - if (firstMeta.isEmpty()) { + if (!firstMeta.isPresent()) { finishProcess( fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "No snapshot metadata read") diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java index 6df4081be8c29..14a395779c9d4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreStatusTask.java @@ -40,7 +40,7 @@ */ @GridInternal @Deprecated -public class SnapshotRestoreStatusTask extends ComputeTaskAdapter { +class SnapshotRestoreStatusTask extends ComputeTaskAdapter { /** Serial version uid. */ private static final long serialVersionUID = 0L; From 375edb418ebc0055b80af2bd60cd0c01c7e7b034 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Wed, 2 Sep 2026 11:09:14 +0300 Subject: [PATCH 08/44] test fixes --- .../ignite/util/GridCommandHandlerTest.java | 99 +++++++------------ .../TestRecordingCommunicationSpi.java | 8 ++ 2 files changed, 42 insertions(+), 65 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index f5b6de0aea469..470388d878ae2 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -137,7 +137,6 @@ import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.metric.MetricRegistry; import org.apache.ignite.plugin.extensions.communication.Message; -import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.spi.metric.Metric; @@ -156,7 +155,6 @@ import org.junit.Test; import static java.io.File.separatorChar; -import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.ignite.IgniteSystemProperties.IGNITE_CLUSTER_NAME; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -3673,7 +3671,7 @@ public void testSnapshotRestoreCancelAndStatus() throws Exception { /** Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. */ @Test public void testRestoreSnapshotMetricsAtStart() throws Exception { - communicationSpiSupp = TestCommunicationSpi::new; + communicationSpiSupp = TestRecordingCommunicationSpi::new; startGrids(3).cluster().state(ClusterState.ACTIVE); @@ -3687,40 +3685,29 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { awaitPartitionMapExchange(); - var checkSingleResultsReceivedLatch = new CountDownLatch(2); - var restoreSingleResultsReceivedLatch = new AtomicInteger(2); - var proceedCheckLatch = new CountDownLatch(1); + TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); + TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); - for (var ig : Arrays.asList(grid(1), grid(2))) { - ((TestCommunicationSpi)ig.configuration().getCommunicationSpi()).msgCsmr = msg -> { - if (!(msg instanceof GridIoMessage ioMsg)) - return; - - if (!(ioMsg.message() instanceof SingleNodeMessage sm)) - return; - - if (sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) - restoreSingleResultsReceivedLatch.decrementAndGet(); - else if (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()) { - checkSingleResultsReceivedLatch.countDown(); - - try { - assertTrue(proceedCheckLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - }; + // Block one of the process' first messages of snapshot restoring or snapshot checking. + for (var cm : Arrays.asList(cm1, cm2)) { + cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm + && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() || sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) + ); } + // Snapshot restoration should get paused at the preceeding checking. IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot("test_snapshot", null, null, 0, true); - assertTrue(checkSingleResultsReceivedLatch.await(getTestTimeout(), MILLISECONDS)); - - // Make sure no restoration started or finished. - assertTrue(restoreSingleResultsReceivedLatch.get() == 2); - assertFalse("Snapshot future has finished", restoreFut.isDone()); + // Waiting for the nodes each to send snapshot check single result. + for (var cm : Arrays.asList(cm1, cm2)) { + assertTrue(waitForCondition( + () -> cm.blockedMessages().stream().anyMatch( + m -> m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() + ), + getTestTimeout() + )); + } injectTestSystemOut(); @@ -3736,12 +3723,24 @@ else if (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()) { assertContains(log, out, "Incremental: false"); assertContains(log, out, "Estimated operation progress:"); - proceedCheckLatch.countDown(); + // Let's suppose the restoration could start and wait for a while. + Thread.sleep(3000L); + + // Ensure that no snapshot restoration started or finished. + assertFalse("Snapshot future has finished", restoreFut.isDone()); + + for (var cm : Arrays.asList(cm1, cm2)) { + // Ensure that nthe restoration process didn't start. + assertTrue(cm.blockedMessages().stream().noneMatch( + m -> m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) + ); + + cm.stopBlock(); + } // Wait for future to finish in order to avoid excessive message about task cancellation. restoreFut.get(); - - assertTrue(restoreSingleResultsReceivedLatch.get() == 0); } /** @throws Exception If fails. */ @@ -4152,34 +4151,4 @@ public void input(String input) { this.input = input; } } - - /** */ - private static class TestCommunicationSpi extends TcpCommunicationSpi { - /** */ - private volatile @Nullable Consumer msgCsmr; - - /** {@inheritDoc} */ - @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException { - var msgCsmr = this.msgCsmr; - - if (msgCsmr != null) - msgCsmr.accept(msg); - - super.sendMessage(node, msg); - } - - /** {@inheritDoc} */ - @Override public void sendMessage( - ClusterNode node, - Message msg, - IgniteInClosure ackC - ) throws IgniteSpiException { - var msgCsmr = this.msgCsmr; - - if (msgCsmr != null) - msgCsmr.accept(msg); - - super.sendMessage(node, msg, ackC); - } - } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java index cee31a0224a8f..5d08d2d18ca0a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java @@ -32,6 +32,7 @@ import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiInClosure; @@ -248,6 +249,13 @@ public void waitForRecorded() throws InterruptedException { } } + /** */ + public List blockedMessages() { + var res = this.blockedMsgs; + + return F.isEmpty(res) ? Collections.emptyList() : Collections.unmodifiableList(res); + } + /** * @param cls Message class. * @param nodeName Node name. From 4f18f1b84b3a441d64a0f7b0ae041002809cc771 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 11:51:11 +0300 Subject: [PATCH 09/44] tets fixes --- .../ignite/util/GridCommandHandlerTest.java | 7 ++--- .../snapshot/SnapshotStatusTask.java | 28 ++++--------------- 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 470388d878ae2..8cd6c3d950b22 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3723,14 +3723,11 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { assertContains(log, out, "Incremental: false"); assertContains(log, out, "Estimated operation progress:"); - // Let's suppose the restoration could start and wait for a while. - Thread.sleep(3000L); - // Ensure that no snapshot restoration started or finished. assertFalse("Snapshot future has finished", restoreFut.isDone()); for (var cm : Arrays.asList(cm1, cm2)) { - // Ensure that nthe restoration process didn't start. + // Ensure that the restore process didn't start. assertTrue(cm.blockedMessages().stream().noneMatch( m -> m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) @@ -3740,7 +3737,7 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { } // Wait for future to finish in order to avoid excessive message about task cancellation. - restoreFut.get(); + restoreFut.get(getTestTimeout()); } /** @throws Exception If fails. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 9ce1fe5613989..34a54020b459f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -94,20 +94,7 @@ private void resolveCheckStatusSupported() { } for (var n : ignite.cluster().nodes()) { - if (!(n instanceof IgniteClusterNode)) { - if (log.isInfoEnabled()) { - log.info(String.format( - "Cannot extract features of node %s. The status is available only for snapshot creation and restoration.", - n.id() - )); - } - - checkStatusSupported = false; - - return; - } - - if (!((IgniteClusterNode)n).features().contains(feature)) { + if (!(n instanceof IgniteClusterNode cn) || !cn.features().contains(feature)) { if (log.isInfoEnabled()) { log.info(String.format( "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + @@ -141,16 +128,16 @@ private void resolveCheckStatusSupported() { if (error != null) throw new IgniteException("Failed to get the snapshot status.", error); - Collection res0 = F.viewReadOnly(results, ComputeJobResult::getData, r -> r.getData() != null); + Collection res = F.viewReadOnly(results, ComputeJobResult::getData, r -> r.getData() != null); // There is no snapshot operation. - if (res0.isEmpty()) + if (res.isEmpty()) return null; - SnapshotStatus firstRes = F.first(res0); + SnapshotStatus firstRes = F.first(res); // Filter out differing requests due to concurrent updates on nodes. - Collection sameRqRes = F.view(res0, s -> s.reqId.equals(firstRes.reqId)); + Collection sameRqRes = F.view(res, s -> s.reqId.equals(firstRes.reqId)); if (firstRes instanceof SnapshotStatusV2 firstResV2) { assert !F.isEmpty(firstResV2.allCheckStatuses); @@ -218,10 +205,7 @@ private SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { metrics = new T5<>( mreg.findMetric("CurrentSnapshotProcessedSize").value(), mreg.findMetric("CurrentSnapshotTotalSize").value(), - -1L, - -1L, - -1L - ); + -1L, -1L, -1L); } return new SnapshotStatus( From 89e50c8aa26f02783b0f667a1b46f723cf0de207 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 12:05:20 +0300 Subject: [PATCH 10/44] test fixes --- .../snapshot/SnapshotStatusTask.java | 18 +++++++----------- .../TestIgniteReleaseFeatures_2_19_0.java | 3 --- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 34a54020b459f..1093e892789c9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -83,10 +83,8 @@ private void resolveCheckStatusSupported() { var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); if (!ignite.context().rollingUpgrade().features().isActive(feature)) { - if (log.isInfoEnabled()) { - log.info("The snapshot-check-aware status feature isn't enabled. The status is available only for " + - "snapshot creation and restoration."); - } + log.warning("The snapshot-check-aware status feature isn't enabled. The status is available only for " + + "snapshot creation and restoration."); checkStatusSupported = false; @@ -95,13 +93,11 @@ private void resolveCheckStatusSupported() { for (var n : ignite.cluster().nodes()) { if (!(n instanceof IgniteClusterNode cn) || !cn.features().contains(feature)) { - if (log.isInfoEnabled()) { - log.info(String.format( - "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + - "for snapshot creation and restoration.", - n.id() - )); - } + log.warning(String.format( + "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + + "for snapshot creation and restoration.", + n.id() + )); checkStatusSupported = false; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java index 7fb49a2744440..3cb497ee64701 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_0.java @@ -21,7 +21,4 @@ public class TestIgniteReleaseFeatures_2_19_0 { /** */ public static final IgniteFeature ROLLING_UPGRADE_FEATURE = new IgniteCoreFeature(0); - - /** */ - public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = new IgniteCoreFeature(1); } From 5e91a9397278773757a7cb98207afd4a867a9a1f Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 14:19:21 +0300 Subject: [PATCH 11/44] extended tests --- .../ignite/util/GridCommandHandlerTest.java | 141 ++++++++++++++---- 1 file changed, 114 insertions(+), 27 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 8cd6c3d950b22..6ed41f1176605 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -78,6 +78,7 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.IgniteVersionUtils; import org.apache.ignite.internal.Order; import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.dto.IgniteDataTransferObject; @@ -117,6 +118,10 @@ import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor; import org.apache.ignite.internal.processors.datastreamer.DataStreamerRequest; import org.apache.ignite.internal.processors.metric.MetricRegistryImpl; +import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor; +import org.apache.ignite.internal.processors.rollingupgrade.RollingUpgradeProcessor; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeatureSet; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureSet; import org.apache.ignite.internal.util.BasicRateLimiter; import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.internal.util.distributed.DistributedProcess; @@ -136,7 +141,11 @@ import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.metric.MetricRegistry; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.plugin.PluginProvider; import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.IgniteNodeValidationResult; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.spi.metric.Metric; @@ -240,20 +249,16 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb /** */ protected @Nullable Supplier communicationSpiSupp; + /** */ + protected @Nullable PluginProvider pluginProvider; + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); initDiagnosticDir(); - cleanDiagnosticDir(); - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - super.afterTest(); - - listeningLog = null; + cleanPersistenceDir(); } /** {@inheritDoc} */ @@ -273,6 +278,9 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb if (communicationSpiSupp != null) cfg.setCommunicationSpi(communicationSpiSupp.get()); + if (pluginProvider != null) + cfg.setPluginProviders(pluginProvider); + return cfg; } @@ -469,7 +477,7 @@ public void testIdleVerifyCancelWhileCalcPartitionHashRunning() throws Exception * @param checkCrc If {@code true} then run idle verify with --check-crc argument. * @param waitAtStart If {@code true} then forces nodes to wait after the procedure start and to proceed only after * the {@code prepare}'s before-latch switched. - */ + */ private void doTestCancelIdleVerify( BiConsumer prepare, boolean checkCrc, @@ -1245,7 +1253,7 @@ public void testConnectivityCommandWithFailedNodes() throws Exception { String what = "There is no connectivity between the following nodes"; assertContains(log, out.replaceAll("[\\W_]+", "").trim(), - what.replaceAll("[\\W_]+", "").trim()); + what.replaceAll("[\\W_]+", "").trim()); } /** @@ -1679,14 +1687,14 @@ else if (entry.getKey().equals(node2)) { // Test kill by xid. validate(h, map -> { - assertEquals(1, map.size()); + assertEquals(1, map.size()); - Map.Entry killedEntry = map.entrySet().iterator().next(); + Map.Entry killedEntry = map.entrySet().iterator().next(); - TxInfo info = killedEntry.getValue().getInfos().get(0); + TxInfo info = killedEntry.getValue().getInfos().get(0); - assertEquals(toKill[0].getXid(), info.getXid()); - }, "--tx", "--kill", + assertEquals(toKill[0].getXid(), info.getXid()); + }, "--tx", "--kill", "--xid", toKill[0].getXid().toString(), // Use saved on first run value. "--nodes", grid(0).localNode().consistentId().toString()); @@ -1917,8 +1925,8 @@ public void testBaselineAddOnNotActiveCluster() throws Exception { // Ignite instase 1 can be logged only in arguments list. boolean isInstance1Found = Arrays.stream(testOutStr.split("\n")) - .filter(s -> s.contains("Arguments:")) - .noneMatch(s -> s.contains(getTestIgniteInstanceName() + "1")); + .filter(s -> s.contains("Arguments:")) + .noneMatch(s -> s.contains(getTestIgniteInstanceName() + "1")); assertTrue(testOutStr, testOutStr.contains("Node not found for consistent ID:")); @@ -3052,7 +3060,7 @@ public void testClusterCreateSnapshotWarning() throws Exception { String expWarn = dataStmrDetected ? DataStreamerUpdatesHandler.WRN_MSG : String.format("Cache partitions differ for cache groups [%s]. ", CU.cacheId(DEFAULT_CACHE_NAME)) - + SnapshotPartitionsQuickVerifyHandler.WRN_MSG; + + SnapshotPartitionsQuickVerifyHandler.WRN_MSG; assertContains(log, out, expWarn); @@ -3668,11 +3676,70 @@ public void testSnapshotRestoreCancelAndStatus() throws Exception { assertNull(ig.cache(DEFAULT_CACHE_NAME)); } - /** Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. */ + /** + * Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. + * All nodes support the check status. + */ + @Test + public void testRestoreSnapshotMetricsAtStartAllNodesSupport() throws Exception { + doTestRestoreSnapshotMetricsAtStart(null); + } + + /** + * Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. + * One node doesn't support the check status. + */ + @Test + public void testRestoreSnapshotMetricsAtStartOneNodeDoesntSupport() throws Exception { + doTestRestoreSnapshotMetricsAtStart(false); + } + + /** + * Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. + * All nodes don't support the check status. + */ @Test - public void testRestoreSnapshotMetricsAtStart() throws Exception { + public void testRestoreSnapshotMetricsAtStartNoSupport() throws Exception { + doTestRestoreSnapshotMetricsAtStart(true); + } + + /** + * @param allNodesNotSupporting Flag of how many nodes don't support the snapshot check status. + * If {@code null}, all nodes support. + * If {@code true}, none of nodes support the snapshot check status. + * If {@code false}, just one node doesn't support the snapshot check status. + */ + private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSupporting) throws Exception { communicationSpiSupp = TestRecordingCommunicationSpi::new; + // Creates empty feature set unsupporting the snpshot check ststus if required. + pluginProvider = allNodesNotSupporting == null ? null : new AbstractTestPluginProvider() { + @Override public String name() { + return "Test features provider"; + } + + @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { + if (!cls.equals(DiscoveryNodeValidationProcessor.class)) + return null; + + assert allNodesNotSupporting != null; + + boolean doNotSupport = allNodesNotSupporting + || ctx.igniteConfiguration().getIgniteInstanceName().equals(getTestIgniteInstanceName(2)); + + return (T)new RollingUpgradeProcessor( + ((IgniteEx)ctx.grid()).context(), + doNotSupport ? new IgniteCoreFeatureSet(IgniteVersionUtils.VER, new IgniteFeatureSet()) : IgniteCoreFeatureSet.local() + ) { + @Override public @Nullable IgniteNodeValidationResult validateNode(ClusterNode joiningNode) { + return null; + } + }; + } + }; + + listeningLog = new ListeningTestLogger(log); + startGrids(3).cluster().state(ClusterState.ACTIVE); createCacheAndPreload(grid(1), 8192); @@ -3711,17 +3778,37 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { injectTestSystemOut(); + LogListener logLsnr = null; + + if (Boolean.FALSE.equals(allNodesNotSupporting)) { + logLsnr = LogListener.matches("Node %s doesn't support the snapshot-check-aware status feature" + .formatted(grid(2).localNode().id())).build(); + + listeningLog.registerListener(logLsnr); + } else if (Boolean.TRUE.equals(allNodesNotSupporting)) { + logLsnr = LogListener.matches("The snapshot-check-aware status feature isn't enabled").build(); + + listeningLog.registerListener(logLsnr); + } + int code = execute("--snapshot", "status"); - // Ensures that there is a status despite unstarted restore process. assertEquals("Unexpected exit code", EXIT_CODE_OK, code); var out = testOut.toString(); - assertContains(log, out, "Check snapshot operation is in progress"); - assertContains(log, out, "Snapshot name: test_snapshot"); - assertContains(log, out, "Incremental: false"); - assertContains(log, out, "Estimated operation progress:"); + if (allNodesNotSupporting == null) { + assertContains(log, out, "Check snapshot operation is in progress"); + assertContains(log, out, "Snapshot name: test_snapshot"); + assertContains(log, out, "Incremental: false"); + assertContains(log, out, "Estimated operation progress:"); + } else { + assert logLsnr != null; + + assertTrue(logLsnr.check(getTestTimeout())); + + assertContains(log, out, "There is no create or restore snapshot operation in progress"); + } // Ensure that no snapshot restoration started or finished. assertFalse("Snapshot future has finished", restoreFut.isDone()); @@ -3961,8 +4048,8 @@ public void testCacheIdleVerifyLogLevelDebug() throws Exception { ignite.cluster().state(ACTIVE); IgniteCache cache = ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) - .setAffinity(new RendezvousAffinityFunction(false, 32)) - .setBackups(1)); + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(1)); cache.put("key", "value"); From ab62f2dd9eae022ea1473aa1f9b930bbb35c97bf Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 15:54:46 +0300 Subject: [PATCH 12/44] minority --- .../internal/management/snapshot/SnapshotStatusCommand.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 819aaa585d1c2..5b5c702615ac4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -65,8 +65,8 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand 0; assert (status instanceof SnapshotStatusTask.SnapshotStatusV2) == !(isCreating || isRestoring); From c223b8fb7213a22005d7227a467bdc1c6dd6c88e Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 16:31:22 +0300 Subject: [PATCH 13/44] fix --- .../ignite/util/GridCommandHandlerTest.java | 24 +- .../snapshot/SnapshotStatusCommand.java | 9 +- .../snapshot/SnapshotStatusTask.java | 232 +++-------------- .../snapshot/SnapshotStatusTaskV2.java | 239 ++++++++++++++++++ .../util/distributed/SingleNodeMessage.java | 6 - .../ignite/spi/discovery/tcp/ClientImpl.java | 28 +- .../resources/META-INF/classnames.properties | 5 +- 7 files changed, 303 insertions(+), 240 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 6ed41f1176605..2fcfeca5c7b18 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -477,7 +477,7 @@ public void testIdleVerifyCancelWhileCalcPartitionHashRunning() throws Exception * @param checkCrc If {@code true} then run idle verify with --check-crc argument. * @param waitAtStart If {@code true} then forces nodes to wait after the procedure start and to proceed only after * the {@code prepare}'s before-latch switched. - */ + */ private void doTestCancelIdleVerify( BiConsumer prepare, boolean checkCrc, @@ -1253,7 +1253,7 @@ public void testConnectivityCommandWithFailedNodes() throws Exception { String what = "There is no connectivity between the following nodes"; assertContains(log, out.replaceAll("[\\W_]+", "").trim(), - what.replaceAll("[\\W_]+", "").trim()); + what.replaceAll("[\\W_]+", "").trim()); } /** @@ -1687,14 +1687,14 @@ else if (entry.getKey().equals(node2)) { // Test kill by xid. validate(h, map -> { - assertEquals(1, map.size()); + assertEquals(1, map.size()); - Map.Entry killedEntry = map.entrySet().iterator().next(); + Map.Entry killedEntry = map.entrySet().iterator().next(); - TxInfo info = killedEntry.getValue().getInfos().get(0); + TxInfo info = killedEntry.getValue().getInfos().get(0); - assertEquals(toKill[0].getXid(), info.getXid()); - }, "--tx", "--kill", + assertEquals(toKill[0].getXid(), info.getXid()); + }, "--tx", "--kill", "--xid", toKill[0].getXid().toString(), // Use saved on first run value. "--nodes", grid(0).localNode().consistentId().toString()); @@ -1925,8 +1925,8 @@ public void testBaselineAddOnNotActiveCluster() throws Exception { // Ignite instase 1 can be logged only in arguments list. boolean isInstance1Found = Arrays.stream(testOutStr.split("\n")) - .filter(s -> s.contains("Arguments:")) - .noneMatch(s -> s.contains(getTestIgniteInstanceName() + "1")); + .filter(s -> s.contains("Arguments:")) + .noneMatch(s -> s.contains(getTestIgniteInstanceName() + "1")); assertTrue(testOutStr, testOutStr.contains("Node not found for consistent ID:")); @@ -3060,7 +3060,7 @@ public void testClusterCreateSnapshotWarning() throws Exception { String expWarn = dataStmrDetected ? DataStreamerUpdatesHandler.WRN_MSG : String.format("Cache partitions differ for cache groups [%s]. ", CU.cacheId(DEFAULT_CACHE_NAME)) - + SnapshotPartitionsQuickVerifyHandler.WRN_MSG; + + SnapshotPartitionsQuickVerifyHandler.WRN_MSG; assertContains(log, out, expWarn); @@ -4048,8 +4048,8 @@ public void testCacheIdleVerifyLogLevelDebug() throws Exception { ignite.cluster().state(ACTIVE); IgniteCache cache = ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) - .setAffinity(new RendezvousAffinityFunction(false, 32)) - .setBackups(1)); + .setAffinity(new RendezvousAffinityFunction(false, 32)) + .setBackups(1)); cache.put("key", "value"); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 5b5c702615ac4..52238cc119075 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -53,8 +53,8 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand taskClass() { - return SnapshotStatusTask.class; + @Override public Class taskClass() { + return SnapshotStatusTaskV2.class; } /** {@inheritDoc} */ @@ -69,12 +69,13 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand 0; - assert (status instanceof SnapshotStatusTask.SnapshotStatusV2) == !(isCreating || isRestoring); + assert (status instanceof SnapshotStatusTaskV2.SnapshotStatusV2) == !(isCreating || isRestoring) + : "No create or restore snapshot operation found but the status os not of V2 status."; // The check operation can be run in parallel for different snapshots. List multipleOpsView = isCreating || isRestoring ? Collections.singletonList(status) - : ((SnapshotStatusTask.SnapshotStatusV2)status).allCheckStatuses; + : ((SnapshotStatusTaskV2.SnapshotStatusV2)status).allCheckStatuses; // Flag of additional line delimiter. AtomicBoolean oneOp = new AtomicBoolean(true); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 1093e892789c9..ecc13fd9c7e95 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -18,34 +18,24 @@ package org.apache.ignite.internal.management.snapshot; import java.io.Serializable; -import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import org.apache.ignite.IgniteException; -import org.apache.ignite.IgniteLogger; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.management.api.NoArg; -import org.apache.ignite.internal.managers.discovery.IgniteClusterNode; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; -import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperationRequest; -import org.apache.ignite.internal.processors.metric.impl.MetricUtils; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; -import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; import org.apache.ignite.internal.processors.task.GridInternal; -import org.apache.ignite.internal.util.lang.GridFunc; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T5; import org.apache.ignite.internal.util.typedef.internal.CU; -import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorMultiNodeTask; import org.apache.ignite.internal.visor.VisorTaskArgument; import org.apache.ignite.metric.MetricRegistry; -import org.apache.ignite.resources.LoggerResource; -import org.apache.ignite.spi.metric.BooleanMetric; import org.apache.ignite.spi.metric.IntMetric; import org.apache.ignite.spi.metric.LongMetric; import org.jetbrains.annotations.Nullable; @@ -63,49 +53,9 @@ public class SnapshotStatusTask extends VisorMultiNodeTask job(NoArg arg) { - if (checkStatusSupported == null) - resolveCheckStatusSupported(); - - return checkStatusSupported ? new SnapshotStatusJobV2(arg, debug) : new SnapshotStatusJob(arg, debug); - } - - /** */ - private void resolveCheckStatusSupported() { - var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); - - if (!ignite.context().rollingUpgrade().features().isActive(feature)) { - log.warning("The snapshot-check-aware status feature isn't enabled. The status is available only for " + - "snapshot creation and restoration."); - - checkStatusSupported = false; - - return; - } - - for (var n : ignite.cluster().nodes()) { - if (!(n instanceof IgniteClusterNode cn) || !cn.features().contains(feature)) { - log.warning(String.format( - "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + - "for snapshot creation and restoration.", - n.id() - )); - - checkStatusSupported = false; - - return; - } - } - - checkStatusSupported = true; + return new SnapshotStatusJob(arg, debug); } /** {@inheritDoc} */ @@ -114,7 +64,7 @@ private void resolveCheckStatusSupported() { } /** {@inheritDoc} */ - @Override protected @Nullable SnapshotStatus reduce0(List results) { + @Nullable @Override protected SnapshotStatus reduce0(List results) { if (results.isEmpty()) throw new IgniteException("Failed to get the snapshot status. Topology is empty."); @@ -130,46 +80,21 @@ private void resolveCheckStatusSupported() { if (res.isEmpty()) return null; - SnapshotStatus firstRes = F.first(res); + SnapshotStatus s0 = F.first(res); // Filter out differing requests due to concurrent updates on nodes. - Collection sameRqRes = F.view(res, s -> s.reqId.equals(firstRes.reqId)); - - if (firstRes instanceof SnapshotStatusV2 firstResV2) { - assert !F.isEmpty(firstResV2.allCheckStatuses); - - // Check status: snpName, per node collection. - Map statusesMap = U.newHashMap(sameRqRes.size()); - - sameRqRes.forEach(s -> { - assert s instanceof SnapshotStatusTask.SnapshotStatusV2; - - for (SnapshotStatus s0 : ((SnapshotStatusTask.SnapshotStatusV2)s).allCheckStatuses) { - var prev = statusesMap.putIfAbsent(s0.name, s0); - - if (prev == null) - continue; - - // Merge nodes progress. - prev.progress().putAll(s0.progress()); - } - - firstResV2.allCheckStatuses = new ArrayList<>(statusesMap.values()); - }); - - return firstResV2; - } + res = F.view(res, s -> s.requestId.equals(s0.requestId)); // Merge nodes progress. - Map> mergedProgress = U.newHashMap(sameRqRes.size()); + Map> progress = new HashMap<>(); - sameRqRes.forEach(s -> mergedProgress.putAll(s.progress)); + res.forEach(s -> progress.putAll(s.progress)); - return new SnapshotStatus(firstRes.op, firstRes.name, firstRes.incIdx, firstRes.reqId, firstRes.startTime, mergedProgress); + return new SnapshotStatus(s0.op, s0.name, s0.incIdx, s0.requestId, s0.startTime, progress); } /** */ - private static class SnapshotStatusJob extends SnapshotJob { + protected static class SnapshotStatusJob extends SnapshotJob { /** */ private static final long serialVersionUID = 0L; @@ -177,12 +102,12 @@ private static class SnapshotStatusJob extends SnapshotJob> progress; /** */ - private SnapshotStatus( - @Nullable SnapshotOperation op, + public SnapshotStatus( + SnapshotOperation op, String name, int incIdx, - String reqId, + String requestId, long startTime, Map> progress ) { this.op = op; this.name = name; this.incIdx = incIdx; - this.reqId = reqId; + this.requestId = requestId; this.startTime = startTime; this.progress = progress; } - /** @return Operation type. {@code Null} for other operation types. */ - @Nullable SnapshotOperation operation() { + /** @return Operation type. */ + public SnapshotOperation operation() { return op; } /** @return Snapshot name. */ - String name() { + public String name() { return name; } /** @return Incremental snapshot index. */ - int incrementIndex() { + public int incrementIndex() { return incIdx; } /** @return Request ID. */ - String requestId() { - return reqId; + public String requestId() { + return requestId; } /** @return Start time. */ - long startTime() { + public long startTime() { return startTime; } /** @return Progress of operation on nodes. */ - Map> progress() { + public Map> progress() { return progress; } } /** Snapshot operation type. */ - enum SnapshotOperation { - /** Snapshot creation. */ + public enum SnapshotOperation { + /** Create snapshot. */ CREATE, - /** Snapshot restoration. */ + /** Restore snapshot. */ RESTORE } - - /** */ - private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { - /** */ - private static final long serialVersionUID = 0L; - - /** */ - private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { - super(arg, debug); - } - - /** {@inheritDoc} */ - @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { - var res1 = super.run(arg); - - // Create or restore status detected. - if (res1 != null) - return res1; - - List checkStatuses = null; - - for (var snpCheckMReg : ignite.context().metric()) { - if (!snpCheckMReg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) - continue; - - if (checkStatuses == null) - checkStatuses = new ArrayList<>(); - - int incIdx = snpCheckMReg.findMetric("incrementIndex") == null - ? 0 - : ((IntMetric)snpCheckMReg.findMetric("incrementIndex")).value(); - - T5 metrics; - - if (incIdx > 0) { - metrics = new T5<>( - (long)snpCheckMReg.findMetric("processedWalSegments").value(), - (long)snpCheckMReg.findMetric("totalWalSegments").value(), - -1L, - -1L, - -1L - ); - } - else { - metrics = new T5<>( - snpCheckMReg.findMetric("checkPartitions").value() ? 1L : 0L, - (long)snpCheckMReg.findMetric("processedPartitions").value(), - (long)snpCheckMReg.findMetric("totalPartitions").value(), - (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), - (long)snpCheckMReg.findMetric("snapshotPartsToProcess").value() - ); - } - - var status = new SnapshotStatus( - null, - MetricUtils.fromFullName(snpCheckMReg.name()).get2(), - incIdx, - snpCheckMReg.findMetric("requestId").getAsString(), - ((LongMetric)snpCheckMReg.findMetric("startTime")).value(), - GridFunc.asMap(ignite.localNode().id(), metrics) - ); - - checkStatuses.add(status); - } - - return checkStatuses == null ? null : new SnapshotStatusV2(checkStatuses); - } - } - - /** {@link SnapshotStatus} with support of snapshot check process. */ - public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { - /** */ - private static final long serialVersionUID = 0L; - - /** Nodes' statuses of all snapshot check operations. */ - @Nullable List allCheckStatuses; - - /** */ - private SnapshotStatusV2(SnapshotStatus s1) { - super(s1.op, s1.name, s1.incIdx, s1.reqId, s1.startTime, s1.progress); - } - - /** */ - private SnapshotStatusV2(List allCheckStatuses) { - // Single, V1 status holds first found check status. - super( - null, - allCheckStatuses.get(0).name, - allCheckStatuses.get(0).incIdx, - allCheckStatuses.get(0).reqId, - allCheckStatuses.get(0).startTime, - allCheckStatuses.get(0).progress - ); - - this.allCheckStatuses = allCheckStatuses; - } - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java new file mode 100644 index 0000000000000..af629fd394b59 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.snapshot; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.internal.management.api.NoArg; +import org.apache.ignite.internal.managers.discovery.IgniteClusterNode; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; +import org.apache.ignite.internal.processors.metric.impl.MetricUtils; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; +import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; +import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.util.lang.GridFunc; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.T5; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.resources.LoggerResource; +import org.apache.ignite.spi.metric.BooleanMetric; +import org.apache.ignite.spi.metric.IntMetric; +import org.apache.ignite.spi.metric.LongMetric; +import org.jetbrains.annotations.Nullable; + +/** + * Task to get the status of the current snapshot operation in the cluster. + */ +@GridInternal +public class SnapshotStatusTaskV2 extends SnapshotStatusTask { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + @LoggerResource + private transient IgniteLogger log; + + /** */ + private transient @Nullable Boolean checkStatusSupported; + + /** @inheritDoc} */ + @Override protected VisorJob job(NoArg arg) { + if (checkStatusSupported == null) + resolveCheckStatusSupported(); + + return checkStatusSupported ? new SnapshotStatusJobV2(arg, debug) : new SnapshotStatusJob(arg, debug); + } + + /** */ + private void resolveCheckStatusSupported() { + var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); + + if (!ignite.context().rollingUpgrade().features().isActive(feature)) { + log.warning("The snapshot-check-aware status feature isn't enabled. The status is available only for " + + "snapshot creation and restoration."); + + checkStatusSupported = false; + + return; + } + + for (var n : ignite.cluster().nodes()) { + if (!(n instanceof IgniteClusterNode cn) || !cn.features().contains(feature)) { + log.warning(String.format( + "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + + "for snapshot creation and restoration.", + n.id() + )); + + checkStatusSupported = false; + + return; + } + } + + checkStatusSupported = true; + } + + /** {@inheritDoc} */ + @Override protected @Nullable SnapshotStatus reduce0(List results) { + SnapshotStatus res0 = super.reduce0(results); + + // No results received at all. + if (res0 == null) + return null; + + // Found crate or restore result. + if (res0.operation() != null) + return res0; + + Collection sameRqRes = F.viewReadOnly(results, ComputeJobResult::getData, + r -> r.getData() != null && ((SnapshotStatus)r.getData()).requestId().equals(res0.requestId())); + + assert !F.isEmpty(sameRqRes); + + SnapshotStatus firstRes = F.first(sameRqRes); + + assert firstRes instanceof SnapshotStatusV2 : "Expected V2 snapshot status result"; + + SnapshotStatusV2 firstResV2 = (SnapshotStatusV2)firstRes; + + // Check status: snpName, per node collection. + Map statusesMap = U.newHashMap(sameRqRes.size()); + + sameRqRes.forEach(s -> { + assert s instanceof SnapshotStatusV2; + + for (SnapshotStatus s0 : ((SnapshotStatusV2)s).allCheckStatuses) { + var prev = statusesMap.putIfAbsent(s0.name(), s0); + + if (prev == null) + continue; + + // Merge nodes progress. + prev.progress().putAll(s0.progress()); + } + + firstResV2.allCheckStatuses = new ArrayList<>(statusesMap.values()); + }); + + return firstResV2; + } + + /** */ + private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { + var res1 = super.run(arg); + + // Create or restore status detected. + if (res1 != null) + return res1; + + List checkStatuses = null; + + for (var snpCheckMReg : ignite.context().metric()) { + if (!snpCheckMReg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) + continue; + + if (checkStatuses == null) + checkStatuses = new ArrayList<>(); + + int incIdx = snpCheckMReg.findMetric("incrementIndex") == null + ? 0 + : ((IntMetric)snpCheckMReg.findMetric("incrementIndex")).value(); + + T5 metrics; + + if (incIdx > 0) { + metrics = new T5<>( + (long)snpCheckMReg.findMetric("processedWalSegments").value(), + (long)snpCheckMReg.findMetric("totalWalSegments").value(), + -1L, + -1L, + -1L + ); + } + else { + metrics = new T5<>( + snpCheckMReg.findMetric("checkPartitions").value() ? 1L : 0L, + (long)snpCheckMReg.findMetric("processedPartitions").value(), + (long)snpCheckMReg.findMetric("totalPartitions").value(), + (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), + (long)snpCheckMReg.findMetric("snapshotPartsToProcess").value() + ); + } + + var status = new SnapshotStatus( + null, + MetricUtils.fromFullName(snpCheckMReg.name()).get2(), + incIdx, + snpCheckMReg.findMetric("requestId").getAsString(), + ((LongMetric)snpCheckMReg.findMetric("startTime")).value(), + GridFunc.asMap(ignite.localNode().id(), metrics) + ); + + checkStatuses.add(status); + } + + return checkStatuses == null ? null : new SnapshotStatusV2(checkStatuses); + } + } + + /** {@link SnapshotStatus} with support of snapshot check process. */ + public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { + /** */ + private static final long serialVersionUID = 0L; + + /** Nodes' statuses of all snapshot check operations. */ + @Nullable List allCheckStatuses; + + /** */ + private SnapshotStatusV2(SnapshotStatus s1) { + super(s1.operation(), s1.name(), s1.incrementIndex(), s1.requestId(), s1.startTime(), s1.progress()); + } + + /** */ + private SnapshotStatusV2(List allCheckStatuses) { + // Single, V1 status holds first found check status. + super( + null, + allCheckStatuses.get(0).name(), + allCheckStatuses.get(0).incrementIndex(), + allCheckStatuses.get(0).requestId(), + allCheckStatuses.get(0).startTime(), + allCheckStatuses.get(0).progress() + ); + + this.allCheckStatuses = allCheckStatuses; + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java index 5c9d9cb276ef8..fc6d5943bbee1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java @@ -22,7 +22,6 @@ import org.apache.ignite.internal.Order; import org.apache.ignite.internal.util.ErrorMessage; import org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType; -import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageFactory; import org.jetbrains.annotations.Nullable; @@ -100,9 +99,4 @@ public boolean hasError() { @Nullable public Throwable error() { return ErrorMessage.error(errMsg); } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(SingleNodeMessage.class, this); - } } diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java index 3cfacdd43b9a9..7af45bd9de712 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java @@ -535,7 +535,7 @@ else if (state == DISCONNECTED) { * @throws IgniteSpiException If failed. * @see TcpDiscoverySpi#joinTimeout */ - @Nullable private TcpDiscoveryIoSession joinTopology( + @Nullable private SocketStream joinTopology( InetSocketAddress prevAddr, long timeout, @Nullable Runnable beforeEachSleep, @@ -581,12 +581,12 @@ else if (state == DISCONNECTED) { Collection addrs0 = new ArrayList<>(addrs); - T2> waitAndRes = sendJoinRequests(prevAddr != null, addrs); + T2> waitAndRes = sendJoinRequests(prevAddr != null, addrs); addrs.clear(); boolean wait = waitAndRes.get1(); - T2 res = waitAndRes.get2(); + T2 res = waitAndRes.get2(); if (res != null) return res.get1(); @@ -611,7 +611,7 @@ else if (addrs.isEmpty()) { } /** */ - private T2> sendJoinRequests( + private T2> sendJoinRequests( boolean recon, Collection addrs ) throws InterruptedException { @@ -619,21 +619,21 @@ private T2> sendJoinRequests( if (Thread.currentThread().isInterrupted()) throw new InterruptedException(); - T2 joinRes = sendJoinRequest(recon, addr); + T2 sockAndRes = sendJoinRequest(recon, addr); - if (joinRes == null) + if (sockAndRes == null) continue; - assert joinRes.get1() != null && joinRes.get2() != null : joinRes; + assert sockAndRes.get1() != null && sockAndRes.get2() != null : sockAndRes; - Socket sock = joinRes.get1().socket(); + Socket sock = sockAndRes.get1().socket(); if (log.isDebugEnabled()) - log.debug("Received response to join request [addr=" + addr + ", res=" + joinRes.get2() + ']'); + log.debug("Received response to join request [addr=" + addr + ", res=" + sockAndRes.get2() + ']'); - switch (joinRes.get2()) { + switch (sockAndRes.get2()) { case RES_OK: - return new T2<>(false, joinRes); + return new T2<>(false, sockAndRes); case RES_CONTINUE_JOIN: case RES_WAIT: @@ -643,7 +643,7 @@ private T2> sendJoinRequests( default: if (log.isDebugEnabled()) - log.debug("Received unexpected response to join request: " + joinRes.get2()); + log.debug("Received unexpected response to join request: " + sockAndRes.get2()); U.closeQuiet(sock); } @@ -671,7 +671,7 @@ private static void sleepEx(long millis, Runnable before, Runnable after) throws * @param addr Address. * @return Socket, connect response and client acknowledge support flag. */ - @Nullable private T2 sendJoinRequest(boolean recon, + @Nullable private T2 sendJoinRequest(boolean recon, InetSocketAddress addr) throws InterruptedException { assert addr != null; @@ -731,7 +731,7 @@ private static void sleepEx(long millis, Runnable before, Runnable after) throws if (log.isInfoEnabled()) log.info("Reconnecting to the addresses of a proper DC [addrs=" + redirectAddrs + ']'); - T2> redirectedRes = sendJoinRequests(recon, redirectAddrs); + T2> redirectedRes = sendJoinRequests(recon, redirectAddrs); return redirectedRes.get2(); } diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index 11594924fe6fd..54dd1595b9b00 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -658,9 +658,10 @@ org.apache.ignite.internal.management.snapshot.SnapshotRestoreTask$SnapshotStart org.apache.ignite.internal.management.snapshot.SnapshotStatusTask org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatus -org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusV2 org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusJob -org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusJobV2 +org.apache.ignite.internal.management.snapshot.SnapshotStatusTaskV2 +org.apache.ignite.internal.management.snapshot.SnapshotStatusTaskV2$SnapshotStatusV2 +org.apache.ignite.internal.management.snapshot.SnapshotStatusTaskV2$SnapshotStatusJobV2 org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationCommandArg org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationResetAllCommandArg org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationResetCommandArg From 77dbb35838c5acae8d93a4764f3e4c0cb39ac2d7 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 16:38:07 +0300 Subject: [PATCH 14/44] + taskV2 for thin client --- .../ignite/util/GridCommandHandlerTest.java | 3 +- .../snapshot/SnapshotStatusTaskV2.java | 9 +++--- .../ignite/spi/discovery/tcp/ClientImpl.java | 28 +++++++++---------- .../TestRecordingCommunicationSpi.java | 2 +- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 2fcfeca5c7b18..741a8bcea99e5 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3785,7 +3785,8 @@ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSu .formatted(grid(2).localNode().id())).build(); listeningLog.registerListener(logLsnr); - } else if (Boolean.TRUE.equals(allNodesNotSupporting)) { + } + else if (Boolean.TRUE.equals(allNodesNotSupporting)) { logLsnr = LogListener.matches("The snapshot-check-aware status feature isn't enabled").build(); listeningLog.registerListener(logLsnr); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java index af629fd394b59..9b185a67007c4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java @@ -42,9 +42,7 @@ import org.apache.ignite.spi.metric.LongMetric; import org.jetbrains.annotations.Nullable; -/** - * Task to get the status of the current snapshot operation in the cluster. - */ +/** V2 of {@link SnapshotStatusTask} with the support of snapshot check status. */ @GridInternal public class SnapshotStatusTaskV2 extends SnapshotStatusTask { /** */ @@ -140,7 +138,7 @@ private void resolveCheckStatusSupported() { return firstResV2; } - /** */ + /** V2 of {@link SnapshotStatusJob} with the support of snapshot check status. */ private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { /** */ private static final long serialVersionUID = 0L; @@ -208,7 +206,8 @@ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { } } - /** {@link SnapshotStatus} with support of snapshot check process. */ + /** V2 of {@link SnapshotStatus} with the support of snapshot check status. */ + public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java index 7af45bd9de712..3cfacdd43b9a9 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java @@ -535,7 +535,7 @@ else if (state == DISCONNECTED) { * @throws IgniteSpiException If failed. * @see TcpDiscoverySpi#joinTimeout */ - @Nullable private SocketStream joinTopology( + @Nullable private TcpDiscoveryIoSession joinTopology( InetSocketAddress prevAddr, long timeout, @Nullable Runnable beforeEachSleep, @@ -581,12 +581,12 @@ else if (state == DISCONNECTED) { Collection addrs0 = new ArrayList<>(addrs); - T2> waitAndRes = sendJoinRequests(prevAddr != null, addrs); + T2> waitAndRes = sendJoinRequests(prevAddr != null, addrs); addrs.clear(); boolean wait = waitAndRes.get1(); - T2 res = waitAndRes.get2(); + T2 res = waitAndRes.get2(); if (res != null) return res.get1(); @@ -611,7 +611,7 @@ else if (addrs.isEmpty()) { } /** */ - private T2> sendJoinRequests( + private T2> sendJoinRequests( boolean recon, Collection addrs ) throws InterruptedException { @@ -619,21 +619,21 @@ private T2> sendJoinRequests( if (Thread.currentThread().isInterrupted()) throw new InterruptedException(); - T2 sockAndRes = sendJoinRequest(recon, addr); + T2 joinRes = sendJoinRequest(recon, addr); - if (sockAndRes == null) + if (joinRes == null) continue; - assert sockAndRes.get1() != null && sockAndRes.get2() != null : sockAndRes; + assert joinRes.get1() != null && joinRes.get2() != null : joinRes; - Socket sock = sockAndRes.get1().socket(); + Socket sock = joinRes.get1().socket(); if (log.isDebugEnabled()) - log.debug("Received response to join request [addr=" + addr + ", res=" + sockAndRes.get2() + ']'); + log.debug("Received response to join request [addr=" + addr + ", res=" + joinRes.get2() + ']'); - switch (sockAndRes.get2()) { + switch (joinRes.get2()) { case RES_OK: - return new T2<>(false, sockAndRes); + return new T2<>(false, joinRes); case RES_CONTINUE_JOIN: case RES_WAIT: @@ -643,7 +643,7 @@ private T2> sendJoinRequests( default: if (log.isDebugEnabled()) - log.debug("Received unexpected response to join request: " + sockAndRes.get2()); + log.debug("Received unexpected response to join request: " + joinRes.get2()); U.closeQuiet(sock); } @@ -671,7 +671,7 @@ private static void sleepEx(long millis, Runnable before, Runnable after) throws * @param addr Address. * @return Socket, connect response and client acknowledge support flag. */ - @Nullable private T2 sendJoinRequest(boolean recon, + @Nullable private T2 sendJoinRequest(boolean recon, InetSocketAddress addr) throws InterruptedException { assert addr != null; @@ -731,7 +731,7 @@ private static void sleepEx(long millis, Runnable before, Runnable after) throws if (log.isInfoEnabled()) log.info("Reconnecting to the addresses of a proper DC [addrs=" + redirectAddrs + ']'); - T2> redirectedRes = sendJoinRequests(recon, redirectAddrs); + T2> redirectedRes = sendJoinRequests(recon, redirectAddrs); return redirectedRes.get2(); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java index 5d08d2d18ca0a..c8e529f84b789 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java @@ -251,7 +251,7 @@ public void waitForRecorded() throws InterruptedException { /** */ public List blockedMessages() { - var res = this.blockedMsgs; + var res = blockedMsgs; return F.isEmpty(res) ? Collections.emptyList() : Collections.unmodifiableList(res); } From 65addb2af6191a1316787fb31bc175685cad7001 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 2 Sep 2026 17:07:40 +0300 Subject: [PATCH 15/44] minor --- .../java/org/apache/ignite/util/GridCommandHandlerTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 741a8bcea99e5..5e814445a892f 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3801,8 +3801,6 @@ else if (Boolean.TRUE.equals(allNodesNotSupporting)) { if (allNodesNotSupporting == null) { assertContains(log, out, "Check snapshot operation is in progress"); assertContains(log, out, "Snapshot name: test_snapshot"); - assertContains(log, out, "Incremental: false"); - assertContains(log, out, "Estimated operation progress:"); } else { assert logLsnr != null; From 0b534fee3d260bd7b7c4572baa33e1b5f0d8a1f2 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Thu, 3 Sep 2026 08:06:28 +0300 Subject: [PATCH 16/44] fixes, + check status test --- .../ignite/util/GridCommandHandlerTest.java | 137 +++++++++++++----- .../snapshot/SnapshotStatusCommand.java | 35 ++--- .../snapshot/SnapshotStatusTaskV2.java | 43 +++--- 3 files changed, 132 insertions(+), 83 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 5e814445a892f..2275cab14a5df 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3706,24 +3706,22 @@ public void testRestoreSnapshotMetricsAtStartNoSupport() throws Exception { /** * @param allNodesNotSupporting Flag of how many nodes don't support the snapshot check status. * If {@code null}, all nodes support. - * If {@code true}, none of nodes support the snapshot check status. - * If {@code false}, just one node doesn't support the snapshot check status. + * If {@code true}, none supports. + * If {@code false}, just one node doesn't support. */ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSupporting) throws Exception { communicationSpiSupp = TestRecordingCommunicationSpi::new; - // Creates empty feature set unsupporting the snpshot check ststus if required. + // Creates empty feature set unsupporting the snapshot check ststus if required. pluginProvider = allNodesNotSupporting == null ? null : new AbstractTestPluginProvider() { @Override public String name() { - return "Test features provider"; + return "Test Ignite features provider"; } @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { if (!cls.equals(DiscoveryNodeValidationProcessor.class)) return null; - assert allNodesNotSupporting != null; - boolean doNotSupport = allNodesNotSupporting || ctx.igniteConfiguration().getIgniteInstanceName().equals(getTestIgniteInstanceName(2)); @@ -3756,38 +3754,31 @@ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSu TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); // Block one of the process' first messages of snapshot restoring or snapshot checking. - for (var cm : Arrays.asList(cm1, cm2)) { - cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm - && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() || sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) - ); - } + F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm + && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() || sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()))); - // Snapshot restoration should get paused at the preceeding checking. + // Snapshot restoration should get paused at the preceeding snapshot check. IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot("test_snapshot", null, null, 0, true); - // Waiting for the nodes each to send snapshot check single result. - for (var cm : Arrays.asList(cm1, cm2)) { - assertTrue(waitForCondition( - () -> cm.blockedMessages().stream().anyMatch( - m -> m.ioMessage().message() instanceof SingleNodeMessage sm - && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() - ), - getTestTimeout() - )); + // Waiting for each node to send snapshot check single result. + for (var cm : F.asList(cm1, cm2)) { + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().anyMatch(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()), + getTestTimeout())); } injectTestSystemOut(); LogListener logLsnr = null; - if (Boolean.FALSE.equals(allNodesNotSupporting)) { - logLsnr = LogListener.matches("Node %s doesn't support the snapshot-check-aware status feature" - .formatted(grid(2).localNode().id())).build(); + if (Boolean.TRUE.equals(allNodesNotSupporting)) { + logLsnr = LogListener.matches("The snapshot-check-aware status feature isn't enabled").build(); listeningLog.registerListener(logLsnr); } - else if (Boolean.TRUE.equals(allNodesNotSupporting)) { - logLsnr = LogListener.matches("The snapshot-check-aware status feature isn't enabled").build(); + else if (Boolean.FALSE.equals(allNodesNotSupporting)) { + logLsnr = LogListener.matches("Node %s doesn't support the snapshot check status feature" + .formatted(grid(2).localNode().id())).build(); listeningLog.registerListener(logLsnr); } @@ -3801,31 +3792,99 @@ else if (Boolean.TRUE.equals(allNodesNotSupporting)) { if (allNodesNotSupporting == null) { assertContains(log, out, "Check snapshot operation is in progress"); assertContains(log, out, "Snapshot name: test_snapshot"); - } else { - assert logLsnr != null; - + } + else { assertTrue(logLsnr.check(getTestTimeout())); - assertContains(log, out, "There is no create or restore snapshot operation in progress"); } - // Ensure that no snapshot restoration started or finished. + // Ensure that no snapshot restore started or finished. assertFalse("Snapshot future has finished", restoreFut.isDone()); - for (var cm : Arrays.asList(cm1, cm2)) { - // Ensure that the restore process didn't start. - assertTrue(cm.blockedMessages().stream().noneMatch( - m -> m.ioMessage().message() instanceof SingleNodeMessage sm - && sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()) - ); + F.asList(cm1, cm2).forEach(cm -> { + assertTrue(cm.blockedMessages().stream().noneMatch(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal())); cm.stopBlock(); - } + }); - // Wait for future to finish in order to avoid excessive message about task cancellation. restoreFut.get(getTestTimeout()); } + /** */ + @Test + public void testOneSnapshotCheckStatus() throws Exception { + doTestSnapshotsChecksStatus(false); + } + + /** */ + @Test + public void testTwoSnapshotsChecksStatus() throws Exception { + doTestSnapshotsChecksStatus(true); + } + + /** */ + private void doTestSnapshotsChecksStatus(boolean twoSnapshots) throws Exception { + communicationSpiSupp = TestRecordingCommunicationSpi::new; + + startGrids(3).cluster().state(ClusterState.ACTIVE); + + IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); + + createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 4096, 64, null); + snapshotMgr.createSnapshot("testSnapshot0").get(getTestTimeout()); + + if (twoSnapshots) { + createCacheAndPreload(grid(1), "cache2", 4096, 64, null); + snapshotMgr.createSnapshot("testSnapshot1").get(getTestTimeout()); + } + + grid(0).destroyCaches(twoSnapshots ? F.asList(DEFAULT_CACHE_NAME, "cache2") : F.asList(DEFAULT_CACHE_NAME)); + + awaitPartitionMapExchange(); + + TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); + TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); + + F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> + msg instanceof SingleNodeMessage sm && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()))); + + var checkFut0 = runAsync(() -> execute("--snapshot", "check", "testSnapshot0")); + var checkFut1 = twoSnapshots ? runAsync(() -> execute("--snapshot", "check", "testSnapshot1")) : null; + + // Waiting for the nodes each to send snapshot check single result. + for (var cm : F.asList(cm1, cm2)) { + assertTrue(waitForCondition( + () -> cm.blockedMessages().stream().filter( + m -> m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoSnapshots ? 2 : 1), + getTestTimeout() + )); + } + + injectTestSystemOut(); + + assertEquals("Unexpected exit code", EXIT_CODE_OK, execute("--snapshot", "status")); + + var out = testOut.toString(); + + if (log.isInfoEnabled()) + log.info("Test out:" + U.nl() + out); + + assertTrue(out.contains(twoSnapshots ? "Check snapshot operations are in progress" : "Check snapshot operation is in progress")); + assertTrue(out.contains("Snapshot name: testSnapshot0")); + if (twoSnapshots) + assertTrue(out.contains("Snapshot name: testSnapshot1")); + + F.asList(cm1, cm2).forEach(TestRecordingCommunicationSpi::stopBlock); + + checkFut0.get(); + + if (twoSnapshots) + checkFut1.get(); + } + /** @throws Exception If fails. */ @Test public void testSnapshotStatusInMemory() throws Exception { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 52238cc119075..53735c84000cf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.stream.Collectors; import org.apache.ignite.internal.management.SystemViewCommand; @@ -75,29 +74,21 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand multipleOpsView = isCreating || isRestoring ? Collections.singletonList(status) - : ((SnapshotStatusTaskV2.SnapshotStatusV2)status).allCheckStatuses; + : ((SnapshotStatusTaskV2.SnapshotStatusV2)status).checkStatuses(); - // Flag of additional line delimiter. - AtomicBoolean oneOp = new AtomicBoolean(true); + assert multipleOpsView.size() == 1 || !(isCreating || isRestoring) : "Only snapshot check supports multiple operations."; - multipleOpsView.forEach(s0 -> { - GridStringBuilder s = new GridStringBuilder(); - - if (!oneOp.get()) - printer.accept(U.nl()); + if (isCreating) + printer.accept("Create snapshot operation is in progress."); + else if (isRestoring) + printer.accept("Restore snapshot operation is in progress."); + else + printer.accept("Check snapshot operation" + (multipleOpsView.size() < 2 ? " is " : "s are ") + "in progress."); - if (isCreating) { - assert multipleOpsView.size() == 1; - - s.a("Create snapshot operation is in progress.").nl(); - } - else if (isRestoring) { - assert multipleOpsView.size() == 1; + printer.accept(U.nl()); - s.a("Restore snapshot operation is in progress.").nl(); - } - else - s.a("Check snapshot operation" + (multipleOpsView.size() < 2 ? " is " : "s are ") + "in progress.").nl(); + multipleOpsView.forEach(s0 -> { + GridStringBuilder s = new GridStringBuilder(); s.a("Snapshot name: ").a(s0.name()).nl(); s.a("Incremental: ").a(isIncremental).nl(); @@ -129,8 +120,6 @@ else if (isRestoring) SystemViewCommand.printTable(desc.titles(), desc.types(), rows, printer); printer.accept(U.nl()); - - oneOp.set(false); }); } @@ -279,7 +268,7 @@ private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressD private final boolean incremental; /** */ - CheckSnapshotTaskProgressDesc(boolean incremental) { + private CheckSnapshotTaskProgressDesc(boolean incremental) { super(incremental ? F.asList("Node ID", "processedWalSegments", "totalWalSegments", "percent") : F.asList("Node ID", "fullCheck", "processedPartitions", "totalPartitions", diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java index 9b185a67007c4..c699a097ab876 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java @@ -42,7 +42,7 @@ import org.apache.ignite.spi.metric.LongMetric; import org.jetbrains.annotations.Nullable; -/** V2 of {@link SnapshotStatusTask} with the support of snapshot check status. */ +/** V2 of {@link SnapshotStatusTask} with support of the snapshot check status. */ @GridInternal public class SnapshotStatusTaskV2 extends SnapshotStatusTask { /** */ @@ -78,11 +78,8 @@ private void resolveCheckStatusSupported() { for (var n : ignite.cluster().nodes()) { if (!(n instanceof IgniteClusterNode cn) || !cn.features().contains(feature)) { - log.warning(String.format( - "Node %s doesn't support the snapshot-check-aware status feature. The status is available only " + - "for snapshot creation and restoration.", - n.id() - )); + log.warning(String.format("Node %s doesn't support the snapshot check status feature. The status " + + "is available only for snapshot creation and restoration.", n.id())); checkStatusSupported = false; @@ -101,7 +98,7 @@ private void resolveCheckStatusSupported() { if (res0 == null) return null; - // Found crate or restore result. + // Found create or restore result. if (res0.operation() != null) return res0; @@ -122,7 +119,7 @@ private void resolveCheckStatusSupported() { sameRqRes.forEach(s -> { assert s instanceof SnapshotStatusV2; - for (SnapshotStatus s0 : ((SnapshotStatusV2)s).allCheckStatuses) { + for (SnapshotStatus s0 : ((SnapshotStatusV2)s).checkStatuses) { var prev = statusesMap.putIfAbsent(s0.name(), s0); if (prev == null) @@ -132,13 +129,13 @@ private void resolveCheckStatusSupported() { prev.progress().putAll(s0.progress()); } - firstResV2.allCheckStatuses = new ArrayList<>(statusesMap.values()); + firstResV2.checkStatuses = new ArrayList<>(statusesMap.values()); }); return firstResV2; } - /** V2 of {@link SnapshotStatusJob} with the support of snapshot check status. */ + /** V2 of {@link SnapshotStatusJob} with support of the snapshot check status. */ private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { /** */ private static final long serialVersionUID = 0L; @@ -206,14 +203,13 @@ private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { } } - /** V2 of {@link SnapshotStatus} with the support of snapshot check status. */ - + /** V2 of {@link SnapshotStatus} with support of the snapshot check status. */ public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { /** */ private static final long serialVersionUID = 0L; /** Nodes' statuses of all snapshot check operations. */ - @Nullable List allCheckStatuses; + private List checkStatuses; /** */ private SnapshotStatusV2(SnapshotStatus s1) { @@ -221,18 +217,23 @@ private SnapshotStatusV2(SnapshotStatus s1) { } /** */ - private SnapshotStatusV2(List allCheckStatuses) { - // Single, V1 status holds first found check status. + private SnapshotStatusV2(List checkStatuses) { + // Single, V1 status holds first check status. super( null, - allCheckStatuses.get(0).name(), - allCheckStatuses.get(0).incrementIndex(), - allCheckStatuses.get(0).requestId(), - allCheckStatuses.get(0).startTime(), - allCheckStatuses.get(0).progress() + checkStatuses.get(0).name(), + checkStatuses.get(0).incrementIndex(), + checkStatuses.get(0).requestId(), + checkStatuses.get(0).startTime(), + checkStatuses.get(0).progress() ); - this.allCheckStatuses = allCheckStatuses; + this.checkStatuses = checkStatuses; + } + + /** */ + public List checkStatuses() { + return checkStatuses; } } } From 215c7dd18fbf559bfdd3e26f813aec1093d0092c Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Thu, 3 Sep 2026 09:07:57 +0300 Subject: [PATCH 17/44] + incremental snp test --- .../ignite/util/GridCommandHandlerTest.java | 88 +++++++++++++++---- 1 file changed, 73 insertions(+), 15 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 2275cab14a5df..6739ecafea6d2 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -252,6 +252,9 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb /** */ protected @Nullable PluginProvider pluginProvider; + /** */ + protected boolean walCompaction; + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); @@ -281,6 +284,8 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb if (pluginProvider != null) cfg.setPluginProviders(pluginProvider); + cfg.getDataStorageConfiguration().setWalCompactionEnabled(walCompaction); + return cfg; } @@ -3740,11 +3745,11 @@ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSu startGrids(3).cluster().state(ClusterState.ACTIVE); - createCacheAndPreload(grid(1), 8192); + createCacheAndPreload(grid(1), 1000); - IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); + IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot(); - snapshotMgr.createSnapshot("test_snapshot").get(getTestTimeout()); + snpMgr.createSnapshot("test_snapshot").get(getTestTimeout()); grid(0).destroyCache(DEFAULT_CACHE_NAME); @@ -3758,7 +3763,7 @@ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSu && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() || sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()))); // Snapshot restoration should get paused at the preceeding snapshot check. - IgniteFutureImpl restoreFut = snapshotMgr.restoreSnapshot("test_snapshot", null, null, 0, true); + IgniteFutureImpl restoreFut = snpMgr.restoreSnapshot("test_snapshot", null, null, 0, true); // Waiting for each node to send snapshot check single result. for (var cm : F.asList(cm1, cm2)) { @@ -3815,29 +3820,61 @@ else if (Boolean.FALSE.equals(allNodesNotSupporting)) { /** */ @Test public void testOneSnapshotCheckStatus() throws Exception { - doTestSnapshotsChecksStatus(false); + doTestSnapshotsChecksStatus(false, false); + } + + /** */ + @Test + public void testOneIncrementalSnapshotCheckStatus() throws Exception { + doTestSnapshotsChecksStatus(false, true); } /** */ @Test public void testTwoSnapshotsChecksStatus() throws Exception { - doTestSnapshotsChecksStatus(true); + doTestSnapshotsChecksStatus(true, false); } /** */ - private void doTestSnapshotsChecksStatus(boolean twoSnapshots) throws Exception { + @Test + public void testTwoIncrementalsSnapshotsChecksStatus() throws Exception { + doTestSnapshotsChecksStatus(true, true); + } + + /** */ + private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean incremental) throws Exception { communicationSpiSupp = TestRecordingCommunicationSpi::new; + walCompaction = incremental; + startGrids(3).cluster().state(ClusterState.ACTIVE); - IgniteSnapshotManager snapshotMgr = (IgniteSnapshotManager)grid(0).snapshot(); + IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot(); - createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 4096, 64, null); - snapshotMgr.createSnapshot("testSnapshot0").get(getTestTimeout()); + createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 1000, 32, null); + snpMgr.createSnapshot("testSnapshot0").get(getTestTimeout()); + + if (incremental) { + try (IgniteDataStreamer streamer = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 1000; i < 2000; i++) + streamer.addData(i, i); + } + + snpMgr.createIncrementalSnapshot("testSnapshot0").get(getTestTimeout()); + } if (twoSnapshots) { - createCacheAndPreload(grid(1), "cache2", 4096, 64, null); - snapshotMgr.createSnapshot("testSnapshot1").get(getTestTimeout()); + createCacheAndPreload(grid(1), "cache2", 1000, 32, null); + snpMgr.createSnapshot("testSnapshot1").get(getTestTimeout()); + + if (incremental) { + try (IgniteDataStreamer streamer = grid(0).dataStreamer("cache2")) { + for (int i = 1000; i < 2000; i++) + streamer.addData(i, i); + } + + snpMgr.createIncrementalSnapshot("testSnapshot1").get(getTestTimeout()); + } } grid(0).destroyCaches(twoSnapshots ? F.asList(DEFAULT_CACHE_NAME, "cache2") : F.asList(DEFAULT_CACHE_NAME)); @@ -3850,8 +3887,15 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots) throws Exception F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()))); - var checkFut0 = runAsync(() -> execute("--snapshot", "check", "testSnapshot0")); - var checkFut1 = twoSnapshots ? runAsync(() -> execute("--snapshot", "check", "testSnapshot1")) : null; + var checkFut0 = runAsync(() -> incremental + ? execute("--snapshot", "check", "testSnapshot0", "--increment", "1") + : execute("--snapshot", "check", "testSnapshot0")); + + var checkFut1 = twoSnapshots + ? runAsync(() -> incremental + ? execute("--snapshot", "check", "testSnapshot1", "--increment", "1") + : execute("--snapshot", "check", "testSnapshot1")) + : null; // Waiting for the nodes each to send snapshot check single result. for (var cm : F.asList(cm1, cm2)) { @@ -3874,9 +3918,23 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots) throws Exception assertTrue(out.contains(twoSnapshots ? "Check snapshot operations are in progress" : "Check snapshot operation is in progress")); assertTrue(out.contains("Snapshot name: testSnapshot0")); - if (twoSnapshots) + + if (incremental) + assertTrue(out.contains("Increment index: 1")); + + if (twoSnapshots) { assertTrue(out.contains("Snapshot name: testSnapshot1")); + if (incremental) { + // Number of 'Increment index: 1' entries. + var sum = Arrays.stream(out.split(U.nl())) + .mapToInt(l -> (l.length() - l.replace("Increment index: 1", "").length()) / "Increment index: 1".length()) + .sum(); + + assertEquals(2, sum); + } + } + F.asList(cm1, cm2).forEach(TestRecordingCommunicationSpi::stopBlock); checkFut0.get(); From 93a442e46f80fe2a80d6111984ed85a411440c55 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 4 Sep 2026 17:45:24 +0300 Subject: [PATCH 18/44] review fixes --- .../ignite/util/GridCommandHandlerTest.java | 126 +++++++-- .../snapshot/SnapshotStatusCommand.java | 109 ++++---- .../snapshot/SnapshotStatusTask.java | 250 +++++++++++++++--- .../snapshot/SnapshotStatusTaskV2.java | 239 ----------------- .../snapshot/SnapshotCheckProcess.java | 17 +- .../snapshot/SnapshotMXBeanImpl.java | 61 ++++- .../resources/META-INF/classnames.properties | 3 - .../TestRecordingCommunicationSpi.java | 9 +- 8 files changed, 423 insertions(+), 391 deletions(-) delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 6739ecafea6d2..ad5a603eb4f2f 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -48,7 +48,6 @@ import java.util.function.BooleanSupplier; import java.util.function.Consumer; import java.util.function.Function; -import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -195,6 +194,7 @@ import static org.apache.ignite.internal.processors.job.GridJobProcessor.JOBS_VIEW; import static org.apache.ignite.internal.processors.task.GridTaskProcessor.TASKS_VIEW; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_PARTS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.END_SNAPSHOT; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; import static org.apache.ignite.testframework.GridTestUtils.assertContains; import static org.apache.ignite.testframework.GridTestUtils.assertNotContains; @@ -246,22 +246,16 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb /** */ protected ListeningTestLogger listeningLog; - /** */ - protected @Nullable Supplier communicationSpiSupp; - /** */ protected @Nullable PluginProvider pluginProvider; - /** */ - protected boolean walCompaction; - /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); initDiagnosticDir(); - cleanPersistenceDir(); + cleanDiagnosticDir(); } /** {@inheritDoc} */ @@ -278,14 +272,9 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb if (listeningLog != null) cfg.setGridLogger(listeningLog); - if (communicationSpiSupp != null) - cfg.setCommunicationSpi(communicationSpiSupp.get()); - if (pluginProvider != null) cfg.setPluginProviders(pluginProvider); - cfg.getDataStorageConfiguration().setWalCompactionEnabled(walCompaction); - return cfg; } @@ -3715,8 +3704,6 @@ public void testRestoreSnapshotMetricsAtStartNoSupport() throws Exception { * If {@code false}, just one node doesn't support. */ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSupporting) throws Exception { - communicationSpiSupp = TestRecordingCommunicationSpi::new; - // Creates empty feature set unsupporting the snapshot check ststus if required. pluginProvider = allNodesNotSupporting == null ? null : new AbstractTestPluginProvider() { @Override public String name() { @@ -3735,6 +3722,7 @@ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSu doNotSupport ? new IgniteCoreFeatureSet(IgniteVersionUtils.VER, new IgniteFeatureSet()) : IgniteCoreFeatureSet.local() ) { @Override public @Nullable IgniteNodeValidationResult validateNode(ClusterNode joiningNode) { + // Simulates started rolling updrade allowing node with other features join cluster. return null; } }; @@ -3843,9 +3831,7 @@ public void testTwoIncrementalsSnapshotsChecksStatus() throws Exception { /** */ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean incremental) throws Exception { - communicationSpiSupp = TestRecordingCommunicationSpi::new; - - walCompaction = incremental; + walCompactionEnabled(incremental); startGrids(3).cluster().state(ClusterState.ACTIVE); @@ -3877,10 +3863,6 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment } } - grid(0).destroyCaches(twoSnapshots ? F.asList(DEFAULT_CACHE_NAME, "cache2") : F.asList(DEFAULT_CACHE_NAME)); - - awaitPartitionMapExchange(); - TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); @@ -3899,9 +3881,8 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment // Waiting for the nodes each to send snapshot check single result. for (var cm : F.asList(cm1, cm2)) { - assertTrue(waitForCondition( - () -> cm.blockedMessages().stream().filter( - m -> m.ioMessage().message() instanceof SingleNodeMessage sm + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoSnapshots ? 2 : 1), getTestTimeout() )); @@ -3916,7 +3897,12 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment if (log.isInfoEnabled()) log.info("Test out:" + U.nl() + out); - assertTrue(out.contains(twoSnapshots ? "Check snapshot operations are in progress" : "Check snapshot operation is in progress")); + String chkSnpE = "Check snapshot operation is in progress"; + var chkSnpESum = Arrays.stream(out.split(U.nl())) + .mapToInt(l -> (l.length() - l.replace(chkSnpE, "").length()) / chkSnpE.length()) + .sum(); + assertEquals(twoSnapshots ? 2 : 1, chkSnpESum); + assertTrue(out.contains("Snapshot name: testSnapshot0")); if (incremental) @@ -3943,6 +3929,94 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment checkFut1.get(); } + /** */ + @Test + public void testSnapshotCheckStatusWithParallelCreate() throws Exception { + doTestSnapshotsChecksStatusWithParallelCreate(false); + } + + /** */ + @Test + public void testTwoSnapshotsChecksStatusWithParallelCreate() throws Exception { + doTestSnapshotsChecksStatusWithParallelCreate(true); + } + + /** */ + private void doTestSnapshotsChecksStatusWithParallelCreate(boolean twoChecks) throws Exception { + startGrids(3).cluster().state(ClusterState.ACTIVE); + + IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot(); + + createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 1000, 32, null); + snpMgr.createSnapshot("testSnapshot0").get(getTestTimeout()); + + if (twoChecks) { + createCacheAndPreload(grid(1), "cache2", 1000, 32, null); + snpMgr.createSnapshot("testSnapshot1").get(getTestTimeout()); + } + + createCacheAndPreload(grid(1), "cache3", 1000, 32, null); + + TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); + TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); + + F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> + msg instanceof SingleNodeMessage sm && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() + || sm.type() == END_SNAPSHOT.ordinal())) + ); + + var createFut = snpMgr.createSnapshot("testSnapshot2"); + + var checkFut0 = runAsync(() -> execute("--snapshot", "check", "testSnapshot0")); + var checkFut1 = twoChecks ? runAsync(() -> execute("--snapshot", "check", "testSnapshot1")) : null; + + // Waiting for the nodes each to send snapshot create single result. + for (var cm : F.asList(cm1, cm2)) { + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().anyMatch(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == END_SNAPSHOT.ordinal()), + getTestTimeout() + )); + } + + // Waiting for the nodes each to send snapshot check single result. + for (var cm : F.asList(cm1, cm2)) { + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoChecks ? 2 : 1), + getTestTimeout() + )); + } + + injectTestSystemOut(); + + assertEquals("Unexpected exit code", EXIT_CODE_OK, execute("--snapshot", "status")); + + var out = testOut.toString(); + + if (log.isInfoEnabled()) + log.info("Test out:" + U.nl() + out); + + assertTrue(out.contains("Create snapshot operation is in progress")); + assertTrue(out.contains("Snapshot name: testSnapshot2")); + + String chkLogE = "Check snapshot operation is in progress"; + var chkLogSum = Arrays.stream(out.split(U.nl())) + .mapToInt(l -> (l.length() - l.replace(chkLogE, "").length()) / chkLogE.length()) + .sum(); + assertEquals(twoChecks ? 2 : 1, chkLogSum); + + assertTrue(out.contains("Snapshot name: testSnapshot0")); + if (twoChecks) + assertTrue(out.contains("Snapshot name: testSnapshot1")); + + F.asList(cm1, cm2).forEach(TestRecordingCommunicationSpi::stopBlock); + + createFut.get(); + checkFut0.get(); + if (twoChecks) + checkFut1.get(); + } + /** @throws Exception If fails. */ @Test public void testSnapshotStatusInMemory() throws Exception { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 53735c84000cf..347d177f4e5fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -26,6 +26,7 @@ import java.util.UUID; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.ignite.internal.management.SystemViewCommand; import org.apache.ignite.internal.management.SystemViewTask; import org.apache.ignite.internal.management.api.NoArg; @@ -52,8 +53,8 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand taskClass() { - return SnapshotStatusTaskV2.class; + @Override public Class taskClass() { + return SnapshotStatusTask.class; } /** {@inheritDoc} */ @@ -64,36 +65,46 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand 0; - - assert (status instanceof SnapshotStatusTaskV2.SnapshotStatusV2) == !(isCreating || isRestoring) - : "No create or restore snapshot operation found but the status os not of V2 status."; + assert status.checkStatuses() != null || status.operation() != SnapshotStatusTask.SnapshotOperation.CHECK + : "No create or restore snapshot operation found but the check statuses are also empty."; // The check operation can be run in parallel for different snapshots. - List multipleOpsView = isCreating || isRestoring - ? Collections.singletonList(status) - : ((SnapshotStatusTaskV2.SnapshotStatusV2)status).checkStatuses(); + List multipleOpsView; + + if (status.operation() == SnapshotStatusTask.SnapshotOperation.CHECK) { + // Check operation always has itself in its aggregated check sattuses. + multipleOpsView = status.checkStatuses(); + } + else { + // If the operation is not check, attach possible parallel checks after. + multipleOpsView = status.checkStatuses() == null + ? Collections.singletonList(status) + : Stream.concat(Stream.of(status), status.checkStatuses().stream()).collect(Collectors.toList()); + } + + boolean first = true; - assert multipleOpsView.size() == 1 || !(isCreating || isRestoring) : "Only snapshot check supports multiple operations."; + for (SnapshotStatus s0 : multipleOpsView) { + if (!first) + printer.accept(U.nl()); - if (isCreating) - printer.accept("Create snapshot operation is in progress."); - else if (isRestoring) - printer.accept("Restore snapshot operation is in progress."); - else - printer.accept("Check snapshot operation" + (multipleOpsView.size() < 2 ? " is " : "s are ") + "in progress."); + if (s0.operation() == SnapshotStatusTask.SnapshotOperation.CREATE) + printer.accept("Create snapshot operation is in progress."); + else if (s0.operation() == SnapshotStatusTask.SnapshotOperation.RESTORE) + printer.accept("Restore snapshot operation is in progress."); + else + printer.accept("Check snapshot operation is in progress."); - printer.accept(U.nl()); + printer.accept(""); - multipleOpsView.forEach(s0 -> { GridStringBuilder s = new GridStringBuilder(); + boolean incremental = s0.incrementIndex() > 0; + s.a("Snapshot name: ").a(s0.name()).nl(); - s.a("Incremental: ").a(isIncremental).nl(); + s.a("Incremental: ").a(incremental).nl(); - if (isIncremental) + if (incremental) s.a("Increment index: ").a(s0.incrementIndex()).nl(); s.a("Operation request ID: ").a(s0.requestId()).nl(); @@ -106,12 +117,12 @@ else if (isRestoring) SnapshotTaskProgressDesc desc; - if (isCreating) - desc = isIncremental ? new CreateIncrementalSnapshotTaskProgressDesc() : new CreateFullSnapshotTaskProgressDesc(); - else if (isRestoring) - desc = isIncremental ? new RestoreIncrementalSnapshotTaskProgressDesc() : new RestoreFullSnapshotTaskProgressDesc(); + if (s0.operation() == SnapshotStatusTask.SnapshotOperation.CREATE) + desc = incremental ? new CreateIncrementalSnapshotTaskProgressDesc() : new CreateFullSnapshotTaskProgressDesc(); + else if (s0.operation() == SnapshotStatusTask.SnapshotOperation.RESTORE) + desc = incremental ? new RestoreIncrementalSnapshotTaskProgressDesc() : new RestoreFullSnapshotTaskProgressDesc(); else - desc = new CheckSnapshotTaskProgressDesc(isIncremental); + desc = new CheckSnapshotTaskProgressDesc(incremental); List> rows = s0.progress().entrySet().stream().sorted(Map.Entry.comparingByKey()) .map(e -> desc.buildRow(e.getKey(), e.getValue())) @@ -119,8 +130,8 @@ else if (isRestoring) SystemViewCommand.printTable(desc.titles(), desc.types(), rows, printer); - printer.accept(U.nl()); - }); + first = false; + } } /** Describes progress of a snapshot task. */ @@ -264,52 +275,26 @@ private static class RestoreIncrementalSnapshotTaskProgressDesc extends Snapshot /** */ private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { - /** */ - private final boolean incremental; - /** */ private CheckSnapshotTaskProgressDesc(boolean incremental) { super(incremental ? F.asList("Node ID", "processedWalSegments", "totalWalSegments", "percent") - : F.asList("Node ID", "fullCheck", "processedPartitions", "totalPartitions", - "processedSnapshotParts", "snapshotPartsToProcess", "percent") + : F.asList("Node ID", "processedPartitions", "totalPartitions", "percent") ); - - this.incremental = incremental; } /** {@inheritDoc} */ @Override public List buildRow(UUID nodeId, T5 progress) { - if (incremental) { - long processed = progress.get1(); - long total = progress.get2(); - - if (total <= 0) - return F.asList(nodeId, "unknown", "unknown", "unknown"); - - String percent = (int)(processed * 100 / total) + "%"; - - return F.asList(nodeId, processed, total, percent); - } + long total = progress.get2(); - long partitionsToCheck = progress.get3(); - long partsToCheck = progress.get5(); + if (total <= 0) + return F.asList(nodeId, "unknown", "unknown", "unknown"); - if (partitionsToCheck <= 0 || partsToCheck <= 0) - return F.asList(nodeId, "unknown", "unknown", "unknown", "unknown", "unknown", "unknown"); + long processed = progress.get1(); - // Ratio of checked partitions in current snapshot part * total parts ratio. - double totalRatio = ((double)progress.get2() / partitionsToCheck) * ((double)progress.get4() / partsToCheck); + String percent = (int)(processed * 100 / total) + "%"; - return F.asList( - nodeId, - progress.get1() == 0L ? "false" : "true", // Full check flag; - progress.get2(), // Checked partitions in current snapshot part; - partitionsToCheck, // Total partitions in current snapshot part; - progress.get4(), // Checked shapshot parts; - partsToCheck, // Total snapshot parts to check. - ((int)(totalRatio * 100.0d)) + '%' - ); + return F.asList(nodeId, processed, total, percent); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index ecc13fd9c7e95..63eb2443fce7d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -18,26 +18,38 @@ package org.apache.ignite.internal.management.snapshot; import java.io.Serializable; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.management.api.NoArg; +import org.apache.ignite.internal.managers.discovery.IgniteClusterNode; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperationRequest; +import org.apache.ignite.internal.processors.metric.impl.MetricUtils; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; +import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.util.lang.GridFunc; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T5; import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorMultiNodeTask; import org.apache.ignite.internal.visor.VisorTaskArgument; import org.apache.ignite.metric.MetricRegistry; +import org.apache.ignite.resources.LoggerResource; import org.apache.ignite.spi.metric.IntMetric; import org.apache.ignite.spi.metric.LongMetric; +import org.apache.ignite.spi.metric.Metric; import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.management.snapshot.SnapshotStatusTask.SnapshotStatus; @@ -74,23 +86,65 @@ public class SnapshotStatusTask extends VisorMultiNodeTask res = F.viewReadOnly(results, ComputeJobResult::getData, r -> r.getData() != null); + // First, find create or restore operation. Check sattuses, if are, contains any status. + Collection res = F.viewReadOnly(results, ComputeJobResult::getData, + r -> r.getData() != null && ((SnapshotStatus)r.getData()).operation() != SnapshotOperation.CHECK); + + if (res.isEmpty()) + res = F.viewReadOnly(results, ComputeJobResult::getData, r -> r.getData() != null); // There is no snapshot operation. if (res.isEmpty()) return null; - SnapshotStatus s0 = F.first(res); + SnapshotStatus mergerRes = F.first(res); + + // Try to find check statutes. + Map chkStatuses = null; + + for (SnapshotStatus s : res) { + if (s.checkStatuses == null) { + assert s.op != SnapshotOperation.CHECK; + + continue; + } + + if (chkStatuses == null) + chkStatuses = U.newHashMap(res.size()); + + for (SnapshotStatus s0 : s.checkStatuses) { + var prev = chkStatuses.putIfAbsent(s0.name(), s0); + + if (prev == null) + continue; + + // Merge nodes progress. + prev.progress.putAll(s0.progress()); + } + } + + var rqId = mergerRes.requestId; // Filter out differing requests due to concurrent updates on nodes. - res = F.view(res, s -> s.requestId.equals(s0.requestId)); + res = F.view(res, s -> s.requestId.equals(rqId)); + + // Create of restore. + if (mergerRes.operation() != SnapshotOperation.CHECK) { + // Merge nodes progress. + Map> progress = new HashMap<>(); - // Merge nodes progress. - Map> progress = new HashMap<>(); + res.forEach(s -> progress.putAll(s.progress)); - res.forEach(s -> progress.putAll(s.progress)); + mergerRes = new SnapshotStatus(mergerRes.op, mergerRes.name, mergerRes.incIdx, mergerRes.requestId, + mergerRes.startTime, progress); + + if (chkStatuses != null) + mergerRes.checkStatuses = new ArrayList<>(chkStatuses.values()); + } - return new SnapshotStatus(s0.op, s0.name, s0.incIdx, s0.requestId, s0.startTime, progress); + assert !F.isEmpty(mergerRes.progress); + + return mergerRes; } /** */ @@ -98,6 +152,33 @@ protected static class SnapshotStatusJob extends SnapshotJob metrics; @@ -129,7 +210,7 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { -1L, -1L, -1L); } - return new SnapshotStatus( + res = new SnapshotStatus( SnapshotOperation.CREATE, req.snapshotName(), req.incrementIndex(), @@ -138,32 +219,91 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { F.asMap(ignite.localNode().id(), metrics) ); } - - MetricRegistry mreg = ignite.context().metric().registry(SNAPSHOT_RESTORE_METRICS); - - long startTime = mreg.findMetric("startTime").value(); - - if (startTime > mreg.findMetric("endTime").value()) { - return new SnapshotStatus( - SnapshotOperation.RESTORE, - mreg.findMetric("snapshotName").getAsString(), - mreg.findMetric("incrementIndex").value(), - mreg.findMetric("requestId").getAsString(), - mreg.findMetric("startTime").value(), - F.asMap( - ignite.localNode().id(), - new T5<>( - (long)mreg.findMetric("processedPartitions").value(), - (long)mreg.findMetric("totalPartitions").value(), - (long)mreg.findMetric("processedWalSegments").value(), - (long)mreg.findMetric("totalWalSegments").value(), - mreg.findMetric("processedWalEntries").value() + else { + MetricRegistry mreg = ignite.context().metric().registry(SNAPSHOT_RESTORE_METRICS); + + long startTime = mreg.findMetric("startTime").value(); + + if (startTime > mreg.findMetric("endTime").value()) { + res = new SnapshotStatus( + SnapshotOperation.RESTORE, + mreg.findMetric("snapshotName").getAsString(), + mreg.findMetric("incrementIndex").value(), + mreg.findMetric("requestId").getAsString(), + mreg.findMetric("startTime").value(), + F.asMap( + ignite.localNode().id(), + new T5<>( + (long)mreg.findMetric("processedPartitions").value(), + (long)mreg.findMetric("totalPartitions").value(), + (long)mreg.findMetric("processedWalSegments").value(), + (long)mreg.findMetric("totalWalSegments").value(), + mreg.findMetric("processedWalEntries").value() + ) ) - ) + ); + } + } + + if (!clusterSupportsSnapshotCheckStatus()) + return res; + + List checkStatuses = null; + + for (var snpCheckMReg : ignite.context().metric()) { + if (!snpCheckMReg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) + continue; + + Metric rqIdMetric = snpCheckMReg.findMetric("requestId"); + + // The requestId metric is registered last. + if (rqIdMetric == null) + continue; + + if (checkStatuses == null) + checkStatuses = new ArrayList<>(); + + int incIdx = snpCheckMReg.findMetric("incrementIndex") == null + ? 0 + : snpCheckMReg.findMetric("incrementIndex").value(); + + T5 metrics; + + if (incIdx > 0) { + metrics = new T5<>( + (long)snpCheckMReg.findMetric("processedWalSegments").value(), + (long)snpCheckMReg.findMetric("totalWalSegments").value(), + -1L, -1L, -1L + ); + } + else { + metrics = new T5<>( + (long)snpCheckMReg.findMetric("processedPartitions").value(), + (long)snpCheckMReg.findMetric("totalPartitions").value(), + -1L, -1L, -1L + ); + } + + var status = new SnapshotStatus( + null, + MetricUtils.fromFullName(snpCheckMReg.name()).get2(), + incIdx, + rqIdMetric.getAsString(), + ((LongMetric)snpCheckMReg.findMetric("startTime")).value(), + GridFunc.asMap(ignite.localNode().id(), metrics) ); + + checkStatuses.add(status); } - return null; + if (checkStatuses != null) { + if (res == null) + res = new SnapshotStatus(checkStatuses); + else + res.checkStatuses = checkStatuses; + } + + return res; } } @@ -190,8 +330,11 @@ public static class SnapshotStatus implements Serializable { /** Progress of operation on nodes. */ private final Map> progress; + /** Nodes' statuses of all snapshot check operations. */ + private @Nullable List checkStatuses; + /** */ - public SnapshotStatus( + SnapshotStatus( SnapshotOperation op, String name, int incIdx, @@ -207,34 +350,56 @@ public SnapshotStatus( this.progress = progress; } + /** */ + private SnapshotStatus(List checkStatuses) { + // Single, V1 status holds first check status. + this( + SnapshotOperation.CHECK, + checkStatuses.get(0).name(), + checkStatuses.get(0).incrementIndex(), + checkStatuses.get(0).requestId(), + checkStatuses.get(0).startTime(), + checkStatuses.get(0).progress() + ); + + assert !F.isEmpty(checkStatuses); + + this.checkStatuses = checkStatuses; + } + /** @return Operation type. */ - public SnapshotOperation operation() { + SnapshotOperation operation() { return op; } /** @return Snapshot name. */ - public String name() { + String name() { return name; } /** @return Incremental snapshot index. */ - public int incrementIndex() { + int incrementIndex() { return incIdx; } /** @return Request ID. */ - public String requestId() { + String requestId() { return requestId; } /** @return Start time. */ - public long startTime() { + long startTime() { return startTime; } /** @return Progress of operation on nodes. */ - public Map> progress() { - return progress; + Map> progress() { + return Collections.unmodifiableMap(progress); + } + + /** @return Statuses of parallel check. */ + @Nullable List checkStatuses() { + return checkStatuses; } } @@ -244,6 +409,9 @@ public enum SnapshotOperation { CREATE, /** Restore snapshot. */ - RESTORE + RESTORE, + + /** Check snapshot. */ + CHECK } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java deleted file mode 100644 index c699a097ab876..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.management.snapshot; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import org.apache.ignite.IgniteException; -import org.apache.ignite.IgniteLogger; -import org.apache.ignite.compute.ComputeJobResult; -import org.apache.ignite.internal.management.api.NoArg; -import org.apache.ignite.internal.managers.discovery.IgniteClusterNode; -import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; -import org.apache.ignite.internal.processors.metric.impl.MetricUtils; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; -import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; -import org.apache.ignite.internal.processors.task.GridInternal; -import org.apache.ignite.internal.util.lang.GridFunc; -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.T5; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.internal.visor.VisorJob; -import org.apache.ignite.resources.LoggerResource; -import org.apache.ignite.spi.metric.BooleanMetric; -import org.apache.ignite.spi.metric.IntMetric; -import org.apache.ignite.spi.metric.LongMetric; -import org.jetbrains.annotations.Nullable; - -/** V2 of {@link SnapshotStatusTask} with support of the snapshot check status. */ -@GridInternal -public class SnapshotStatusTaskV2 extends SnapshotStatusTask { - /** */ - private static final long serialVersionUID = 0L; - - /** */ - @LoggerResource - private transient IgniteLogger log; - - /** */ - private transient @Nullable Boolean checkStatusSupported; - - /** @inheritDoc} */ - @Override protected VisorJob job(NoArg arg) { - if (checkStatusSupported == null) - resolveCheckStatusSupported(); - - return checkStatusSupported ? new SnapshotStatusJobV2(arg, debug) : new SnapshotStatusJob(arg, debug); - } - - /** */ - private void resolveCheckStatusSupported() { - var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); - - if (!ignite.context().rollingUpgrade().features().isActive(feature)) { - log.warning("The snapshot-check-aware status feature isn't enabled. The status is available only for " + - "snapshot creation and restoration."); - - checkStatusSupported = false; - - return; - } - - for (var n : ignite.cluster().nodes()) { - if (!(n instanceof IgniteClusterNode cn) || !cn.features().contains(feature)) { - log.warning(String.format("Node %s doesn't support the snapshot check status feature. The status " + - "is available only for snapshot creation and restoration.", n.id())); - - checkStatusSupported = false; - - return; - } - } - - checkStatusSupported = true; - } - - /** {@inheritDoc} */ - @Override protected @Nullable SnapshotStatus reduce0(List results) { - SnapshotStatus res0 = super.reduce0(results); - - // No results received at all. - if (res0 == null) - return null; - - // Found create or restore result. - if (res0.operation() != null) - return res0; - - Collection sameRqRes = F.viewReadOnly(results, ComputeJobResult::getData, - r -> r.getData() != null && ((SnapshotStatus)r.getData()).requestId().equals(res0.requestId())); - - assert !F.isEmpty(sameRqRes); - - SnapshotStatus firstRes = F.first(sameRqRes); - - assert firstRes instanceof SnapshotStatusV2 : "Expected V2 snapshot status result"; - - SnapshotStatusV2 firstResV2 = (SnapshotStatusV2)firstRes; - - // Check status: snpName, per node collection. - Map statusesMap = U.newHashMap(sameRqRes.size()); - - sameRqRes.forEach(s -> { - assert s instanceof SnapshotStatusV2; - - for (SnapshotStatus s0 : ((SnapshotStatusV2)s).checkStatuses) { - var prev = statusesMap.putIfAbsent(s0.name(), s0); - - if (prev == null) - continue; - - // Merge nodes progress. - prev.progress().putAll(s0.progress()); - } - - firstResV2.checkStatuses = new ArrayList<>(statusesMap.values()); - }); - - return firstResV2; - } - - /** V2 of {@link SnapshotStatusJob} with support of the snapshot check status. */ - private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { - /** */ - private static final long serialVersionUID = 0L; - - /** */ - private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { - super(arg, debug); - } - - /** {@inheritDoc} */ - @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { - var res1 = super.run(arg); - - // Create or restore status detected. - if (res1 != null) - return res1; - - List checkStatuses = null; - - for (var snpCheckMReg : ignite.context().metric()) { - if (!snpCheckMReg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) - continue; - - if (checkStatuses == null) - checkStatuses = new ArrayList<>(); - - int incIdx = snpCheckMReg.findMetric("incrementIndex") == null - ? 0 - : ((IntMetric)snpCheckMReg.findMetric("incrementIndex")).value(); - - T5 metrics; - - if (incIdx > 0) { - metrics = new T5<>( - (long)snpCheckMReg.findMetric("processedWalSegments").value(), - (long)snpCheckMReg.findMetric("totalWalSegments").value(), - -1L, - -1L, - -1L - ); - } - else { - metrics = new T5<>( - snpCheckMReg.findMetric("checkPartitions").value() ? 1L : 0L, - (long)snpCheckMReg.findMetric("processedPartitions").value(), - (long)snpCheckMReg.findMetric("totalPartitions").value(), - (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), - (long)snpCheckMReg.findMetric("snapshotPartsToProcess").value() - ); - } - - var status = new SnapshotStatus( - null, - MetricUtils.fromFullName(snpCheckMReg.name()).get2(), - incIdx, - snpCheckMReg.findMetric("requestId").getAsString(), - ((LongMetric)snpCheckMReg.findMetric("startTime")).value(), - GridFunc.asMap(ignite.localNode().id(), metrics) - ); - - checkStatuses.add(status); - } - - return checkStatuses == null ? null : new SnapshotStatusV2(checkStatuses); - } - } - - /** V2 of {@link SnapshotStatus} with support of the snapshot check status. */ - public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { - /** */ - private static final long serialVersionUID = 0L; - - /** Nodes' statuses of all snapshot check operations. */ - private List checkStatuses; - - /** */ - private SnapshotStatusV2(SnapshotStatus s1) { - super(s1.operation(), s1.name(), s1.incrementIndex(), s1.requestId(), s1.startTime(), s1.progress()); - } - - /** */ - private SnapshotStatusV2(List checkStatuses) { - // Single, V1 status holds first check status. - super( - null, - checkStatuses.get(0).name(), - checkStatuses.get(0).incrementIndex(), - checkStatuses.get(0).requestId(), - checkStatuses.get(0).startTime(), - checkStatuses.get(0).progress() - ); - - this.checkStatuses = checkStatuses; - } - - /** */ - public List checkStatuses() { - return checkStatuses; - } - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index f069919248f91..a4f4e98d587b8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -732,13 +732,6 @@ private void registerMetrics(SnapshotCheckContext ctx) { mreg.register("startTime", U::currentTimeMillis, "The system time of the start of the cluster snapshot check operation on current node."); - mreg.register( - "requestId", - () -> ctx.req.requestId().toString(), - String.class, - "The request ID of the last running cluster snapshot restore operation on this node." - ); - if (ctx.req.incrementalIndex() > 0) { mreg.register("incrementIndex", ctx.req::incrementalIndex, "The index of incremental snapshot of the snapshot check operation."); @@ -758,6 +751,14 @@ private void registerMetrics(SnapshotCheckContext ctx) { mreg.register("processedSnapshotParts", ctx.checkedSnapshotParts::get, "Number of checked snapshot parts (nodes data) on current node."); } + + // Persistence of this metrics also marks that the registry is complete. + mreg.register( + "requestId", + () -> ctx.req.requestId().toString(), + String.class, + "The request ID of the last running cluster snapshot check operation on this node." + ); } /** */ @@ -784,7 +785,7 @@ private static final class SnapshotCheckContext { /** All the snapshot metadatas. */ @Nullable private Map> clusterMetas; - /** Common counter of total work units to process on current node. */ + /** Common counter of total work units to process on current node, of all snapshot parts ({@link #checkedSnapshotParts}). */ @GridToStringExclude private final AtomicInteger totalCounter = new AtomicInteger(-1); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java index 5cf956155c8b5..bedd6243724ec 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java @@ -23,11 +23,14 @@ import java.util.stream.Collectors; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.processors.metric.GridMetricManager; +import org.apache.ignite.internal.processors.metric.impl.MetricUtils; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.metric.MetricRegistry; import org.apache.ignite.mxbean.SnapshotMXBean; import org.apache.ignite.spi.metric.IntMetric; +import org.apache.ignite.spi.metric.Metric; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.DFLT_CHECK_ON_RESTORE; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotRestoreProcess.SNAPSHOT_RESTORE_METRICS; @@ -116,16 +119,17 @@ public SnapshotMXBeanImpl(GridKernalContext ctx) { /** {@inheritDoc} */ @Override public String status() { + StringBuilder res = null; + SnapshotOperationRequest req = mgr.currentCreateRequest(); if (req != null) { - return "Create snapshot operation is in progress [name=" + req.snapshotName() + + res = new StringBuilder("Create snapshot operation is in progress [name=" + req.snapshotName() + ", incremental=" + req.incremental() + (req.incremental() ? (", incrementIndex=" + req.incrementIndex()) : "") + - ", id=" + req.requestId() + ']'; + ", id=" + req.requestId() + ']'); } - - if (mgr.isRestoring()) { + else if (mgr.isRestoring()) { MetricRegistry mreg = metricMgr.registry(SNAPSHOT_RESTORE_METRICS); String name = mreg.findMetric("snapshotName").getAsString(); @@ -134,10 +138,53 @@ public SnapshotMXBeanImpl(GridKernalContext ctx) { boolean incremental = incIdx > 0; - return "Restore snapshot operation is in progress [name=" + name + ", incremental=" + incremental + - (incremental ? ", incrementIndex=" + incIdx : "") + ", id=" + id + ']'; + res = new StringBuilder("Restore snapshot operation is in progress [name=" + name + ", incremental=" + incremental + + (incremental ? ", incrementIndex=" + incIdx : "") + ", id=" + id + ']'); + } + + boolean one = true; + + for (var mreg : metricMgr) { + if (!mreg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) + continue; + + Metric rqIdMetric = mreg.findMetric("requestId"); + + // The requestId metric is registered last. + if (rqIdMetric == null) + continue; + + String name = MetricUtils.fromFullName(mreg.name()).get2(); + int incIdx = mreg.findMetric("incrementIndex").value(); + String id = rqIdMetric.getAsString(); + + if (one) { + if (res == null) + res = new StringBuilder(); + else + res.append(U.nl()).append(U.nl()); + } + + res.append("Check snapshot operations are in progress ["); + + StringBuilder sb0 = new StringBuilder(); + + if (!one) + sb0.append(", "); + + sb0.append("[name=").append(name).append(", incremental=").append(incIdx > 0) + .append(incIdx > 0 ? ", incrementIndex=" + incIdx : "").append(", id=").append(id).append(']'); + + res.append(sb0); + + one = false; } - return "There is no create or restore snapshot operation in progress."; + if (res == null) + return "No snapshot operation is in progress."; + + res.append(']'); + + return res.toString(); } } diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index 54dd1595b9b00..65f0244da47b7 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -659,9 +659,6 @@ org.apache.ignite.internal.management.snapshot.SnapshotStatusTask org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatus org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusJob -org.apache.ignite.internal.management.snapshot.SnapshotStatusTaskV2 -org.apache.ignite.internal.management.snapshot.SnapshotStatusTaskV2$SnapshotStatusV2 -org.apache.ignite.internal.management.snapshot.SnapshotStatusTaskV2$SnapshotStatusJobV2 org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationCommandArg org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationResetAllCommandArg org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationResetCommandArg diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java index c8e529f84b789..d0f83b89dccac 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java @@ -32,7 +32,6 @@ import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; -import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiInClosure; @@ -60,7 +59,7 @@ public class TestRecordingCommunicationSpi extends TcpCommunicationSpi { private List recordedMsgs = new ArrayList<>(); /** */ - private List blockedMsgs = new ArrayList<>(); + private final List blockedMsgs = new ArrayList<>(); /** */ private Map, Set> blockCls = new HashMap<>(); @@ -251,9 +250,9 @@ public void waitForRecorded() throws InterruptedException { /** */ public List blockedMessages() { - var res = blockedMsgs; - - return F.isEmpty(res) ? Collections.emptyList() : Collections.unmodifiableList(res); + synchronized (this) { + return Collections.unmodifiableList(blockedMsgs); + } } /** From 1bc87c38a3ab2ecbeaba04502ebf702d895a0fbd Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 4 Sep 2026 17:45:24 +0300 Subject: [PATCH 19/44] review fixes --- docs/_docs/snapshots/snapshots.adoc | 27 +- .../ignite/util/GridCommandHandlerTest.java | 166 ++++++++---- .../snapshot/SnapshotStatusCommand.java | 111 ++++---- .../snapshot/SnapshotStatusTask.java | 253 +++++++++++++++--- .../snapshot/SnapshotStatusTaskV2.java | 239 ----------------- .../snapshot/SnapshotCheckProcess.java | 17 +- .../apache/ignite/mxbean/SnapshotMXBean.java | 7 +- .../resources/META-INF/classnames.properties | 3 - .../TestRecordingCommunicationSpi.java | 9 +- 9 files changed, 401 insertions(+), 431 deletions(-) delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java diff --git a/docs/_docs/snapshots/snapshots.adoc b/docs/_docs/snapshots/snapshots.adoc index fe61b3c49234a..eb2ab5024de8b 100644 --- a/docs/_docs/snapshots/snapshots.adoc +++ b/docs/_docs/snapshots/snapshots.adoc @@ -289,35 +289,36 @@ control.(sh|bat) --snapshot restore snapshot_09062021 --increment 1 == Getting Snapshot Operation Status -The status of the current snapshot operation in the cluster can be obtained using the `control.sh|bat` script or JMX interface: +The status of current snapshot operations in the cluster can be obtained using the `control.sh|bat` script. + +Snapshot operations include: │ +* **Create** - Creating a full or incremental snapshot │ +* **Restore** - Restoring cache groups from a snapshot │ +* **Check** - Verifying snapshot consistency (can run in parallel for multiple snapshots) │ [tabs] -- tab:Unix[] [source,shell] ---- -# Get the status of the snapshot operation. +# Get the status of snapshot operation. control.sh --snapshot status ---- tab:Windows[] [source,shell] ---- -# Get the status of the snapshot operation. +# Get the status of snapshot operation. control.bat --snapshot status ---- - -tab:JMX[] -You can also get the current snapshot status via the `SnapshotMXBean` interface: -[source,java] ----- -SnapshotMXBean mxBean = ...; - -// The status of a current snapshot operation in the cluster. -String status = mxBean.status(); ----- -- + [NOTE] │ +==== │ +Parallel and multiple snapshot check operations can be running concurrently with any other snapshot operation. +But only one check for one snapshot. Create and restore operations cannot run in parallel - only one of each type can +be active at a time. The status is printed for each running snapshot operation (if multiple) separately, one after another. + == Cancelling Snapshot Operation To abort create/restore snapshot operation you need to obtain an `operation request ID`. diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 6739ecafea6d2..b58e2eb756d18 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -48,7 +48,6 @@ import java.util.function.BooleanSupplier; import java.util.function.Consumer; import java.util.function.Function; -import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -195,6 +194,7 @@ import static org.apache.ignite.internal.processors.job.GridJobProcessor.JOBS_VIEW; import static org.apache.ignite.internal.processors.task.GridTaskProcessor.TASKS_VIEW; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_PARTS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.END_SNAPSHOT; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; import static org.apache.ignite.testframework.GridTestUtils.assertContains; import static org.apache.ignite.testframework.GridTestUtils.assertNotContains; @@ -246,22 +246,16 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb /** */ protected ListeningTestLogger listeningLog; - /** */ - protected @Nullable Supplier communicationSpiSupp; - /** */ protected @Nullable PluginProvider pluginProvider; - /** */ - protected boolean walCompaction; - /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); initDiagnosticDir(); - cleanPersistenceDir(); + cleanDiagnosticDir(); } /** {@inheritDoc} */ @@ -278,14 +272,9 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb if (listeningLog != null) cfg.setGridLogger(listeningLog); - if (communicationSpiSupp != null) - cfg.setCommunicationSpi(communicationSpiSupp.get()); - if (pluginProvider != null) cfg.setPluginProviders(pluginProvider); - cfg.getDataStorageConfiguration().setWalCompactionEnabled(walCompaction); - return cfg; } @@ -3687,7 +3676,7 @@ public void testSnapshotRestoreCancelAndStatus() throws Exception { */ @Test public void testRestoreSnapshotMetricsAtStartAllNodesSupport() throws Exception { - doTestRestoreSnapshotMetricsAtStart(null); + doTestRestoreSnapshotMetricsAtStart(false); } /** @@ -3696,29 +3685,15 @@ public void testRestoreSnapshotMetricsAtStartAllNodesSupport() throws Exception */ @Test public void testRestoreSnapshotMetricsAtStartOneNodeDoesntSupport() throws Exception { - doTestRestoreSnapshotMetricsAtStart(false); - } - - /** - * Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. - * All nodes don't support the check status. - */ - @Test - public void testRestoreSnapshotMetricsAtStartNoSupport() throws Exception { doTestRestoreSnapshotMetricsAtStart(true); } /** - * @param allNodesNotSupporting Flag of how many nodes don't support the snapshot check status. - * If {@code null}, all nodes support. - * If {@code true}, none supports. - * If {@code false}, just one node doesn't support. + * @param oneNodeDoesntSupport If {@code true}, one node will simulate not supporting snapshot check status. */ - private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSupporting) throws Exception { - communicationSpiSupp = TestRecordingCommunicationSpi::new; - + private void doTestRestoreSnapshotMetricsAtStart(boolean oneNodeDoesntSupport) throws Exception { // Creates empty feature set unsupporting the snapshot check ststus if required. - pluginProvider = allNodesNotSupporting == null ? null : new AbstractTestPluginProvider() { + pluginProvider = !oneNodeDoesntSupport ? null : new AbstractTestPluginProvider() { @Override public String name() { return "Test Ignite features provider"; } @@ -3727,14 +3702,14 @@ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSu if (!cls.equals(DiscoveryNodeValidationProcessor.class)) return null; - boolean doNotSupport = allNodesNotSupporting - || ctx.igniteConfiguration().getIgniteInstanceName().equals(getTestIgniteInstanceName(2)); + boolean doNotSupport = ctx.igniteConfiguration().getIgniteInstanceName().equals(getTestIgniteInstanceName(2)); return (T)new RollingUpgradeProcessor( ((IgniteEx)ctx.grid()).context(), doNotSupport ? new IgniteCoreFeatureSet(IgniteVersionUtils.VER, new IgniteFeatureSet()) : IgniteCoreFeatureSet.local() ) { @Override public @Nullable IgniteNodeValidationResult validateNode(ClusterNode joiningNode) { + // Simulates started rolling updrade allowing node with other features join cluster. return null; } }; @@ -3776,12 +3751,7 @@ private void doTestRestoreSnapshotMetricsAtStart(@Nullable Boolean allNodesNotSu LogListener logLsnr = null; - if (Boolean.TRUE.equals(allNodesNotSupporting)) { - logLsnr = LogListener.matches("The snapshot-check-aware status feature isn't enabled").build(); - - listeningLog.registerListener(logLsnr); - } - else if (Boolean.FALSE.equals(allNodesNotSupporting)) { + if (oneNodeDoesntSupport) { logLsnr = LogListener.matches("Node %s doesn't support the snapshot check status feature" .formatted(grid(2).localNode().id())).build(); @@ -3794,14 +3764,14 @@ else if (Boolean.FALSE.equals(allNodesNotSupporting)) { var out = testOut.toString(); - if (allNodesNotSupporting == null) { - assertContains(log, out, "Check snapshot operation is in progress"); - assertContains(log, out, "Snapshot name: test_snapshot"); - } - else { + if (oneNodeDoesntSupport) { assertTrue(logLsnr.check(getTestTimeout())); assertContains(log, out, "There is no create or restore snapshot operation in progress"); } + else { + assertContains(log, out, "Check snapshot operation is in progress"); + assertContains(log, out, "Snapshot name: test_snapshot"); + } // Ensure that no snapshot restore started or finished. assertFalse("Snapshot future has finished", restoreFut.isDone()); @@ -3843,9 +3813,7 @@ public void testTwoIncrementalsSnapshotsChecksStatus() throws Exception { /** */ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean incremental) throws Exception { - communicationSpiSupp = TestRecordingCommunicationSpi::new; - - walCompaction = incremental; + walCompactionEnabled(incremental); startGrids(3).cluster().state(ClusterState.ACTIVE); @@ -3877,10 +3845,6 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment } } - grid(0).destroyCaches(twoSnapshots ? F.asList(DEFAULT_CACHE_NAME, "cache2") : F.asList(DEFAULT_CACHE_NAME)); - - awaitPartitionMapExchange(); - TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); @@ -3899,9 +3863,8 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment // Waiting for the nodes each to send snapshot check single result. for (var cm : F.asList(cm1, cm2)) { - assertTrue(waitForCondition( - () -> cm.blockedMessages().stream().filter( - m -> m.ioMessage().message() instanceof SingleNodeMessage sm + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoSnapshots ? 2 : 1), getTestTimeout() )); @@ -3916,7 +3879,12 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment if (log.isInfoEnabled()) log.info("Test out:" + U.nl() + out); - assertTrue(out.contains(twoSnapshots ? "Check snapshot operations are in progress" : "Check snapshot operation is in progress")); + String chkSnpE = "Check snapshot operation is in progress"; + var chkSnpESum = Arrays.stream(out.split(U.nl())) + .mapToInt(l -> (l.length() - l.replace(chkSnpE, "").length()) / chkSnpE.length()) + .sum(); + assertEquals(twoSnapshots ? 2 : 1, chkSnpESum); + assertTrue(out.contains("Snapshot name: testSnapshot0")); if (incremental) @@ -3943,6 +3911,94 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment checkFut1.get(); } + /** */ + @Test + public void testSnapshotCheckStatusWithParallelCreate() throws Exception { + doTestSnapshotsChecksStatusWithParallelCreate(false); + } + + /** */ + @Test + public void testTwoSnapshotsChecksStatusWithParallelCreate() throws Exception { + doTestSnapshotsChecksStatusWithParallelCreate(true); + } + + /** */ + private void doTestSnapshotsChecksStatusWithParallelCreate(boolean twoChecks) throws Exception { + startGrids(3).cluster().state(ClusterState.ACTIVE); + + IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot(); + + createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 1000, 32, null); + snpMgr.createSnapshot("testSnapshot0").get(getTestTimeout()); + + if (twoChecks) { + createCacheAndPreload(grid(1), "cache2", 1000, 32, null); + snpMgr.createSnapshot("testSnapshot1").get(getTestTimeout()); + } + + createCacheAndPreload(grid(1), "cache3", 1000, 32, null); + + TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); + TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); + + F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> + msg instanceof SingleNodeMessage sm && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() + || sm.type() == END_SNAPSHOT.ordinal())) + ); + + var createFut = snpMgr.createSnapshot("testSnapshot2"); + + var checkFut0 = runAsync(() -> execute("--snapshot", "check", "testSnapshot0")); + var checkFut1 = twoChecks ? runAsync(() -> execute("--snapshot", "check", "testSnapshot1")) : null; + + // Waiting for the nodes each to send snapshot create single result. + for (var cm : F.asList(cm1, cm2)) { + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().anyMatch(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == END_SNAPSHOT.ordinal()), + getTestTimeout() + )); + } + + // Waiting for the nodes each to send snapshot check single result. + for (var cm : F.asList(cm1, cm2)) { + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoChecks ? 2 : 1), + getTestTimeout() + )); + } + + injectTestSystemOut(); + + assertEquals("Unexpected exit code", EXIT_CODE_OK, execute("--snapshot", "status")); + + var out = testOut.toString(); + + if (log.isInfoEnabled()) + log.info("Test out:" + U.nl() + out); + + assertTrue(out.contains("Create snapshot operation is in progress")); + assertTrue(out.contains("Snapshot name: testSnapshot2")); + + String chkLogE = "Check snapshot operation is in progress"; + var chkLogSum = Arrays.stream(out.split(U.nl())) + .mapToInt(l -> (l.length() - l.replace(chkLogE, "").length()) / chkLogE.length()) + .sum(); + assertEquals(twoChecks ? 2 : 1, chkLogSum); + + assertTrue(out.contains("Snapshot name: testSnapshot0")); + if (twoChecks) + assertTrue(out.contains("Snapshot name: testSnapshot1")); + + F.asList(cm1, cm2).forEach(TestRecordingCommunicationSpi::stopBlock); + + createFut.get(); + checkFut0.get(); + if (twoChecks) + checkFut1.get(); + } + /** @throws Exception If fails. */ @Test public void testSnapshotStatusInMemory() throws Exception { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 53735c84000cf..b2210a7e96975 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -26,6 +26,7 @@ import java.util.UUID; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.ignite.internal.management.SystemViewCommand; import org.apache.ignite.internal.management.SystemViewTask; import org.apache.ignite.internal.management.api.NoArg; @@ -52,8 +53,8 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand taskClass() { - return SnapshotStatusTaskV2.class; + @Override public Class taskClass() { + return SnapshotStatusTask.class; } /** {@inheritDoc} */ @@ -64,36 +65,46 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand 0; - - assert (status instanceof SnapshotStatusTaskV2.SnapshotStatusV2) == !(isCreating || isRestoring) - : "No create or restore snapshot operation found but the status os not of V2 status."; + assert status.checkStatuses() != null || status.operation() != SnapshotStatusTask.SnapshotOperation.CHECK + : "No create or restore snapshot operation found but the check statuses are also empty."; // The check operation can be run in parallel for different snapshots. - List multipleOpsView = isCreating || isRestoring - ? Collections.singletonList(status) - : ((SnapshotStatusTaskV2.SnapshotStatusV2)status).checkStatuses(); + List multipleOpsView; + + if (status.operation() == SnapshotStatusTask.SnapshotOperation.CHECK) { + // Check operation always has itself in its aggregated check statuses. + multipleOpsView = status.checkStatuses(); + } + else { + // If the operation is not check, attach possible parallel checks after. + multipleOpsView = status.checkStatuses() == null + ? Collections.singletonList(status) + : Stream.concat(Stream.of(status), status.checkStatuses().stream()).collect(Collectors.toList()); + } + + boolean first = true; - assert multipleOpsView.size() == 1 || !(isCreating || isRestoring) : "Only snapshot check supports multiple operations."; + for (SnapshotStatus s0 : multipleOpsView) { + if (!first) + printer.accept(U.nl()); - if (isCreating) - printer.accept("Create snapshot operation is in progress."); - else if (isRestoring) - printer.accept("Restore snapshot operation is in progress."); - else - printer.accept("Check snapshot operation" + (multipleOpsView.size() < 2 ? " is " : "s are ") + "in progress."); + if (s0.operation() == SnapshotStatusTask.SnapshotOperation.CREATE) + printer.accept("Create snapshot operation is in progress."); + else if (s0.operation() == SnapshotStatusTask.SnapshotOperation.RESTORE) + printer.accept("Restore snapshot operation is in progress."); + else + printer.accept("Check snapshot operation is in progress."); - printer.accept(U.nl()); + printer.accept(""); - multipleOpsView.forEach(s0 -> { GridStringBuilder s = new GridStringBuilder(); + boolean incremental = s0.incrementIndex() > 0; + s.a("Snapshot name: ").a(s0.name()).nl(); - s.a("Incremental: ").a(isIncremental).nl(); + s.a("Incremental: ").a(incremental).nl(); - if (isIncremental) + if (incremental) s.a("Increment index: ").a(s0.incrementIndex()).nl(); s.a("Operation request ID: ").a(s0.requestId()).nl(); @@ -106,12 +117,12 @@ else if (isRestoring) SnapshotTaskProgressDesc desc; - if (isCreating) - desc = isIncremental ? new CreateIncrementalSnapshotTaskProgressDesc() : new CreateFullSnapshotTaskProgressDesc(); - else if (isRestoring) - desc = isIncremental ? new RestoreIncrementalSnapshotTaskProgressDesc() : new RestoreFullSnapshotTaskProgressDesc(); + if (s0.operation() == SnapshotStatusTask.SnapshotOperation.CREATE) + desc = incremental ? new CreateIncrementalSnapshotTaskProgressDesc() : new CreateFullSnapshotTaskProgressDesc(); + else if (s0.operation() == SnapshotStatusTask.SnapshotOperation.RESTORE) + desc = incremental ? new RestoreIncrementalSnapshotTaskProgressDesc() : new RestoreFullSnapshotTaskProgressDesc(); else - desc = new CheckSnapshotTaskProgressDesc(isIncremental); + desc = new CheckSnapshotTaskProgressDesc(incremental); List> rows = s0.progress().entrySet().stream().sorted(Map.Entry.comparingByKey()) .map(e -> desc.buildRow(e.getKey(), e.getValue())) @@ -119,8 +130,8 @@ else if (isRestoring) SystemViewCommand.printTable(desc.titles(), desc.types(), rows, printer); - printer.accept(U.nl()); - }); + first = false; + } } /** Describes progress of a snapshot task. */ @@ -264,52 +275,26 @@ private static class RestoreIncrementalSnapshotTaskProgressDesc extends Snapshot /** */ private static class CheckSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { - /** */ - private final boolean incremental; - /** */ private CheckSnapshotTaskProgressDesc(boolean incremental) { super(incremental - ? F.asList("Node ID", "processedWalSegments", "totalWalSegments", "percent") - : F.asList("Node ID", "fullCheck", "processedPartitions", "totalPartitions", - "processedSnapshotParts", "snapshotPartsToProcess", "percent") + ? F.asList("Node ID", "Processed WAL segments", "Total WAL segments", "Percent") + : F.asList("Node ID", "Processed partitions", "Total partitions", "Percent") ); - - this.incremental = incremental; } /** {@inheritDoc} */ @Override public List buildRow(UUID nodeId, T5 progress) { - if (incremental) { - long processed = progress.get1(); - long total = progress.get2(); - - if (total <= 0) - return F.asList(nodeId, "unknown", "unknown", "unknown"); - - String percent = (int)(processed * 100 / total) + "%"; - - return F.asList(nodeId, processed, total, percent); - } + long total = progress.get2(); - long partitionsToCheck = progress.get3(); - long partsToCheck = progress.get5(); + if (total <= 0) + return F.asList(nodeId, "unknown", "unknown", "unknown"); - if (partitionsToCheck <= 0 || partsToCheck <= 0) - return F.asList(nodeId, "unknown", "unknown", "unknown", "unknown", "unknown", "unknown"); + long processed = progress.get1(); - // Ratio of checked partitions in current snapshot part * total parts ratio. - double totalRatio = ((double)progress.get2() / partitionsToCheck) * ((double)progress.get4() / partsToCheck); + String percent = (int)(processed * 100 / total) + "%"; - return F.asList( - nodeId, - progress.get1() == 0L ? "false" : "true", // Full check flag; - progress.get2(), // Checked partitions in current snapshot part; - partitionsToCheck, // Total partitions in current snapshot part; - progress.get4(), // Checked shapshot parts; - partsToCheck, // Total snapshot parts to check. - ((int)(totalRatio * 100.0d)) + '%' - ); + return F.asList(nodeId, processed, total, percent); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index ecc13fd9c7e95..d092339d039e9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -18,26 +18,37 @@ package org.apache.ignite.internal.management.snapshot; import java.io.Serializable; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.management.api.NoArg; +import org.apache.ignite.internal.managers.discovery.IgniteClusterNode; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperationRequest; +import org.apache.ignite.internal.processors.metric.impl.MetricUtils; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; +import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T5; import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorMultiNodeTask; import org.apache.ignite.internal.visor.VisorTaskArgument; import org.apache.ignite.metric.MetricRegistry; +import org.apache.ignite.resources.LoggerResource; import org.apache.ignite.spi.metric.IntMetric; import org.apache.ignite.spi.metric.LongMetric; +import org.apache.ignite.spi.metric.Metric; import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.management.snapshot.SnapshotStatusTask.SnapshotStatus; @@ -74,30 +85,103 @@ public class SnapshotStatusTask extends VisorMultiNodeTask res = F.viewReadOnly(results, ComputeJobResult::getData, r -> r.getData() != null); + // Try to find check statuses. + Map chkStatuses = null; + + // Not empty results. + Collection statuses = new ArrayList<>(results.size()); + + for (ComputeJobResult jobRes : results) { + if (jobRes.getData() == null) + continue; + + assert jobRes.getData() instanceof SnapshotStatus; + + SnapshotStatus s = jobRes.getData(); + + statuses.add(s); + + if (s.checkStatuses == null) { + assert s.op != SnapshotOperation.CHECK; + + continue; + } + + if (chkStatuses == null) + chkStatuses = U.newHashMap(results.size()); + + for (SnapshotStatus s0 : s.checkStatuses) { + var prev = chkStatuses.putIfAbsent(s0.name(), s0); + + if (prev == null) + continue; + + // Merge nodes progress. + prev.progress.putAll(s0.progress()); + } + } + + // First, find create or restore operation. Check statuses, if are, contains any status. + Collection res = F.viewReadOnly(statuses, s -> s, s -> s.operation() != SnapshotOperation.CHECK); + + if (res.isEmpty()) + res = statuses; // There is no snapshot operation. if (res.isEmpty()) return null; - SnapshotStatus s0 = F.first(res); + SnapshotStatus mergerRes = F.first(res); + + var rqId = mergerRes.requestId; // Filter out differing requests due to concurrent updates on nodes. - res = F.view(res, s -> s.requestId.equals(s0.requestId)); + res = F.view(res, s -> s.requestId.equals(rqId)); + + // Create or restore. + if (mergerRes.operation() != SnapshotOperation.CHECK) { + // Merge nodes progress. + Map> progress = new HashMap<>(); + + res.forEach(s -> progress.putAll(s.progress)); - // Merge nodes progress. - Map> progress = new HashMap<>(); + mergerRes = new SnapshotStatus(mergerRes.op, mergerRes.name, mergerRes.incIdx, mergerRes.requestId, + mergerRes.startTime, progress); + } + + if (chkStatuses != null) + mergerRes.checkStatuses = new ArrayList<>(chkStatuses.values()); - res.forEach(s -> progress.putAll(s.progress)); + assert !F.isEmpty(mergerRes.progress); - return new SnapshotStatus(s0.op, s0.name, s0.incIdx, s0.requestId, s0.startTime, progress); + return mergerRes; } /** */ - protected static class SnapshotStatusJob extends SnapshotJob { + private static class SnapshotStatusJob extends SnapshotJob { /** */ private static final long serialVersionUID = 0L; + /** */ + @LoggerResource + private transient IgniteLogger log; + + /** */ + private boolean clusterSupportsSnapshotCheckStatus() { + var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); + + for (var n : ignite.cluster().forServers().nodes()) { + if (!(n instanceof IgniteClusterNode cn) || !cn.features().contains(feature)) { + log.warning(String.format("Node %s doesn't support the snapshot check status feature. The status " + + "is available only for snapshot creation and restoration.", n.id())); + + return false; + } + } + + return true; + } + /** * @param arg Job argument. * @param debug Flag indicating whether debug information should be printed into node log. @@ -107,13 +191,13 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { } /** {@inheritDoc} */ - @Override protected SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { + @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { if (!CU.isPersistenceEnabled(ignite.context().config())) return null; IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); - SnapshotOperationRequest req = snpMgr.currentCreateRequest(); + SnapshotStatus res = null; if (req != null) { T5 metrics; @@ -129,7 +213,7 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { -1L, -1L, -1L); } - return new SnapshotStatus( + res = new SnapshotStatus( SnapshotOperation.CREATE, req.snapshotName(), req.incrementIndex(), @@ -138,32 +222,91 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { F.asMap(ignite.localNode().id(), metrics) ); } - - MetricRegistry mreg = ignite.context().metric().registry(SNAPSHOT_RESTORE_METRICS); - - long startTime = mreg.findMetric("startTime").value(); - - if (startTime > mreg.findMetric("endTime").value()) { - return new SnapshotStatus( - SnapshotOperation.RESTORE, - mreg.findMetric("snapshotName").getAsString(), - mreg.findMetric("incrementIndex").value(), - mreg.findMetric("requestId").getAsString(), - mreg.findMetric("startTime").value(), - F.asMap( - ignite.localNode().id(), - new T5<>( - (long)mreg.findMetric("processedPartitions").value(), - (long)mreg.findMetric("totalPartitions").value(), - (long)mreg.findMetric("processedWalSegments").value(), - (long)mreg.findMetric("totalWalSegments").value(), - mreg.findMetric("processedWalEntries").value() + else { + MetricRegistry mreg = ignite.context().metric().registry(SNAPSHOT_RESTORE_METRICS); + + long startTime = mreg.findMetric("startTime").value(); + + if (startTime > mreg.findMetric("endTime").value()) { + res = new SnapshotStatus( + SnapshotOperation.RESTORE, + mreg.findMetric("snapshotName").getAsString(), + mreg.findMetric("incrementIndex").value(), + mreg.findMetric("requestId").getAsString(), + mreg.findMetric("startTime").value(), + F.asMap( + ignite.localNode().id(), + new T5<>( + (long)mreg.findMetric("processedPartitions").value(), + (long)mreg.findMetric("totalPartitions").value(), + (long)mreg.findMetric("processedWalSegments").value(), + (long)mreg.findMetric("totalWalSegments").value(), + mreg.findMetric("processedWalEntries").value() + ) ) - ) + ); + } + } + + if (!clusterSupportsSnapshotCheckStatus()) + return res; + + List checkStatuses = null; + + for (var snpCheckMReg : ignite.context().metric()) { + if (!snpCheckMReg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) + continue; + + Metric rqIdMetric = snpCheckMReg.findMetric("requestId"); + + // The requestId metric is registered last. + if (rqIdMetric == null) + continue; + + if (checkStatuses == null) + checkStatuses = new ArrayList<>(); + + int incIdx = snpCheckMReg.findMetric("incrementIndex") == null + ? 0 + : snpCheckMReg.findMetric("incrementIndex").value(); + + T5 metrics; + + if (incIdx > 0) { + metrics = new T5<>( + (long)snpCheckMReg.findMetric("processedWalSegments").value(), + (long)snpCheckMReg.findMetric("totalWalSegments").value(), + -1L, -1L, -1L + ); + } + else { + metrics = new T5<>( + (long)snpCheckMReg.findMetric("processedPartitions").value(), + (long)snpCheckMReg.findMetric("totalPartitions").value(), + -1L, -1L, -1L + ); + } + + var status = new SnapshotStatus( + SnapshotOperation.CHECK, + MetricUtils.fromFullName(snpCheckMReg.name()).get2(), + incIdx, + rqIdMetric.getAsString(), + ((LongMetric)snpCheckMReg.findMetric("startTime")).value(), + F.asMap(ignite.localNode().id(), metrics) ); + + checkStatuses.add(status); } - return null; + if (checkStatuses != null) { + if (res == null) + res = new SnapshotStatus(checkStatuses); + else + res.checkStatuses = checkStatuses; + } + + return res; } } @@ -190,8 +333,11 @@ public static class SnapshotStatus implements Serializable { /** Progress of operation on nodes. */ private final Map> progress; + /** Nodes' statuses of all snapshot check operations. */ + private @Nullable List checkStatuses; + /** */ - public SnapshotStatus( + SnapshotStatus( SnapshotOperation op, String name, int incIdx, @@ -207,34 +353,54 @@ public SnapshotStatus( this.progress = progress; } + /** */ + private SnapshotStatus(List checkStatuses) { + // Single, V1 status holds first check status. + this( + SnapshotOperation.CHECK, + checkStatuses.get(0).name(), + checkStatuses.get(0).incrementIndex(), + checkStatuses.get(0).requestId(), + checkStatuses.get(0).startTime(), + checkStatuses.get(0).progress() + ); + + this.checkStatuses = checkStatuses; + } + /** @return Operation type. */ - public SnapshotOperation operation() { + SnapshotOperation operation() { return op; } /** @return Snapshot name. */ - public String name() { + String name() { return name; } /** @return Incremental snapshot index. */ - public int incrementIndex() { + int incrementIndex() { return incIdx; } /** @return Request ID. */ - public String requestId() { + String requestId() { return requestId; } /** @return Start time. */ - public long startTime() { + long startTime() { return startTime; } /** @return Progress of operation on nodes. */ - public Map> progress() { - return progress; + Map> progress() { + return Collections.unmodifiableMap(progress); + } + + /** @return Statuses of parallel check. */ + @Nullable List checkStatuses() { + return checkStatuses; } } @@ -244,6 +410,9 @@ public enum SnapshotOperation { CREATE, /** Restore snapshot. */ - RESTORE + RESTORE, + + /** Check snapshot. */ + CHECK } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java deleted file mode 100644 index c699a097ab876..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTaskV2.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.management.snapshot; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import org.apache.ignite.IgniteException; -import org.apache.ignite.IgniteLogger; -import org.apache.ignite.compute.ComputeJobResult; -import org.apache.ignite.internal.management.api.NoArg; -import org.apache.ignite.internal.managers.discovery.IgniteClusterNode; -import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; -import org.apache.ignite.internal.processors.metric.impl.MetricUtils; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; -import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; -import org.apache.ignite.internal.processors.task.GridInternal; -import org.apache.ignite.internal.util.lang.GridFunc; -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.T5; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.internal.visor.VisorJob; -import org.apache.ignite.resources.LoggerResource; -import org.apache.ignite.spi.metric.BooleanMetric; -import org.apache.ignite.spi.metric.IntMetric; -import org.apache.ignite.spi.metric.LongMetric; -import org.jetbrains.annotations.Nullable; - -/** V2 of {@link SnapshotStatusTask} with support of the snapshot check status. */ -@GridInternal -public class SnapshotStatusTaskV2 extends SnapshotStatusTask { - /** */ - private static final long serialVersionUID = 0L; - - /** */ - @LoggerResource - private transient IgniteLogger log; - - /** */ - private transient @Nullable Boolean checkStatusSupported; - - /** @inheritDoc} */ - @Override protected VisorJob job(NoArg arg) { - if (checkStatusSupported == null) - resolveCheckStatusSupported(); - - return checkStatusSupported ? new SnapshotStatusJobV2(arg, debug) : new SnapshotStatusJob(arg, debug); - } - - /** */ - private void resolveCheckStatusSupported() { - var feature = new IgniteCoreFeature(SupportedFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE.id()); - - if (!ignite.context().rollingUpgrade().features().isActive(feature)) { - log.warning("The snapshot-check-aware status feature isn't enabled. The status is available only for " + - "snapshot creation and restoration."); - - checkStatusSupported = false; - - return; - } - - for (var n : ignite.cluster().nodes()) { - if (!(n instanceof IgniteClusterNode cn) || !cn.features().contains(feature)) { - log.warning(String.format("Node %s doesn't support the snapshot check status feature. The status " + - "is available only for snapshot creation and restoration.", n.id())); - - checkStatusSupported = false; - - return; - } - } - - checkStatusSupported = true; - } - - /** {@inheritDoc} */ - @Override protected @Nullable SnapshotStatus reduce0(List results) { - SnapshotStatus res0 = super.reduce0(results); - - // No results received at all. - if (res0 == null) - return null; - - // Found create or restore result. - if (res0.operation() != null) - return res0; - - Collection sameRqRes = F.viewReadOnly(results, ComputeJobResult::getData, - r -> r.getData() != null && ((SnapshotStatus)r.getData()).requestId().equals(res0.requestId())); - - assert !F.isEmpty(sameRqRes); - - SnapshotStatus firstRes = F.first(sameRqRes); - - assert firstRes instanceof SnapshotStatusV2 : "Expected V2 snapshot status result"; - - SnapshotStatusV2 firstResV2 = (SnapshotStatusV2)firstRes; - - // Check status: snpName, per node collection. - Map statusesMap = U.newHashMap(sameRqRes.size()); - - sameRqRes.forEach(s -> { - assert s instanceof SnapshotStatusV2; - - for (SnapshotStatus s0 : ((SnapshotStatusV2)s).checkStatuses) { - var prev = statusesMap.putIfAbsent(s0.name(), s0); - - if (prev == null) - continue; - - // Merge nodes progress. - prev.progress().putAll(s0.progress()); - } - - firstResV2.checkStatuses = new ArrayList<>(statusesMap.values()); - }); - - return firstResV2; - } - - /** V2 of {@link SnapshotStatusJob} with support of the snapshot check status. */ - private static class SnapshotStatusJobV2 extends SnapshotStatusTask.SnapshotStatusJob { - /** */ - private static final long serialVersionUID = 0L; - - /** */ - private SnapshotStatusJobV2(@Nullable NoArg arg, boolean debug) { - super(arg, debug); - } - - /** {@inheritDoc} */ - @Override protected @Nullable SnapshotStatus run(@Nullable NoArg arg) throws IgniteException { - var res1 = super.run(arg); - - // Create or restore status detected. - if (res1 != null) - return res1; - - List checkStatuses = null; - - for (var snpCheckMReg : ignite.context().metric()) { - if (!snpCheckMReg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) - continue; - - if (checkStatuses == null) - checkStatuses = new ArrayList<>(); - - int incIdx = snpCheckMReg.findMetric("incrementIndex") == null - ? 0 - : ((IntMetric)snpCheckMReg.findMetric("incrementIndex")).value(); - - T5 metrics; - - if (incIdx > 0) { - metrics = new T5<>( - (long)snpCheckMReg.findMetric("processedWalSegments").value(), - (long)snpCheckMReg.findMetric("totalWalSegments").value(), - -1L, - -1L, - -1L - ); - } - else { - metrics = new T5<>( - snpCheckMReg.findMetric("checkPartitions").value() ? 1L : 0L, - (long)snpCheckMReg.findMetric("processedPartitions").value(), - (long)snpCheckMReg.findMetric("totalPartitions").value(), - (long)snpCheckMReg.findMetric("processedSnapshotParts").value(), - (long)snpCheckMReg.findMetric("snapshotPartsToProcess").value() - ); - } - - var status = new SnapshotStatus( - null, - MetricUtils.fromFullName(snpCheckMReg.name()).get2(), - incIdx, - snpCheckMReg.findMetric("requestId").getAsString(), - ((LongMetric)snpCheckMReg.findMetric("startTime")).value(), - GridFunc.asMap(ignite.localNode().id(), metrics) - ); - - checkStatuses.add(status); - } - - return checkStatuses == null ? null : new SnapshotStatusV2(checkStatuses); - } - } - - /** V2 of {@link SnapshotStatus} with support of the snapshot check status. */ - public static class SnapshotStatusV2 extends SnapshotStatusTask.SnapshotStatus { - /** */ - private static final long serialVersionUID = 0L; - - /** Nodes' statuses of all snapshot check operations. */ - private List checkStatuses; - - /** */ - private SnapshotStatusV2(SnapshotStatus s1) { - super(s1.operation(), s1.name(), s1.incrementIndex(), s1.requestId(), s1.startTime(), s1.progress()); - } - - /** */ - private SnapshotStatusV2(List checkStatuses) { - // Single, V1 status holds first check status. - super( - null, - checkStatuses.get(0).name(), - checkStatuses.get(0).incrementIndex(), - checkStatuses.get(0).requestId(), - checkStatuses.get(0).startTime(), - checkStatuses.get(0).progress() - ); - - this.checkStatuses = checkStatuses; - } - - /** */ - public List checkStatuses() { - return checkStatuses; - } - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index f069919248f91..88a3f25ae451a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -732,13 +732,6 @@ private void registerMetrics(SnapshotCheckContext ctx) { mreg.register("startTime", U::currentTimeMillis, "The system time of the start of the cluster snapshot check operation on current node."); - mreg.register( - "requestId", - () -> ctx.req.requestId().toString(), - String.class, - "The request ID of the last running cluster snapshot restore operation on this node." - ); - if (ctx.req.incrementalIndex() > 0) { mreg.register("incrementIndex", ctx.req::incrementalIndex, "The index of incremental snapshot of the snapshot check operation."); @@ -758,6 +751,14 @@ private void registerMetrics(SnapshotCheckContext ctx) { mreg.register("processedSnapshotParts", ctx.checkedSnapshotParts::get, "Number of checked snapshot parts (nodes data) on current node."); } + + // Presence of this metrics also marks that the registry is complete. + mreg.register( + "requestId", + () -> ctx.req.requestId().toString(), + String.class, + "The request ID of the last running cluster snapshot check operation on this node." + ); } /** */ @@ -784,7 +785,7 @@ private static final class SnapshotCheckContext { /** All the snapshot metadatas. */ @Nullable private Map> clusterMetas; - /** Common counter of total work units to process on current node. */ + /** Common counter of total work units to process on current node, of all snapshot parts ({@link #checkedSnapshotParts}). */ @GridToStringExclude private final AtomicInteger totalCounter = new AtomicInteger(-1); diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java index dee7c59a42e7f..dcd3192d3d353 100644 --- a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java +++ b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java @@ -129,10 +129,11 @@ public void restoreSnapshot( public void cancelSnapshotRestore(@MXBeanParameter(name = "snpName", description = "Snapshot name.") String name); /** - * Get the status of the current snapshot operation in the cluster. + * Get the status of the current snapshot check or restore operation in the cluster. Doesn't provide status of + * snapshots check operations. * - * @return The status of a current snapshot operation in the cluster. + * @return The status of a current snapshot create or restore operation in the cluster. */ - @MXBeanDescription("The status of a current snapshot operation in the cluster.") + @MXBeanDescription("The status of a current snapshot create or restore operation in the cluster.") public String status(); } diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index 54dd1595b9b00..65f0244da47b7 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -659,9 +659,6 @@ org.apache.ignite.internal.management.snapshot.SnapshotStatusTask org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatus org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotStatusJob -org.apache.ignite.internal.management.snapshot.SnapshotStatusTaskV2 -org.apache.ignite.internal.management.snapshot.SnapshotStatusTaskV2$SnapshotStatusV2 -org.apache.ignite.internal.management.snapshot.SnapshotStatusTaskV2$SnapshotStatusJobV2 org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationCommandArg org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationResetAllCommandArg org.apache.ignite.internal.management.tracing.TracingConfigurationCommand$TracingConfigurationResetCommandArg diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java index c8e529f84b789..5ba3cecc3dd29 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java @@ -32,7 +32,6 @@ import org.apache.ignite.internal.managers.communication.GridIoMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; -import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiInClosure; @@ -60,7 +59,7 @@ public class TestRecordingCommunicationSpi extends TcpCommunicationSpi { private List recordedMsgs = new ArrayList<>(); /** */ - private List blockedMsgs = new ArrayList<>(); + private final List blockedMsgs = new ArrayList<>(); /** */ private Map, Set> blockCls = new HashMap<>(); @@ -251,9 +250,9 @@ public void waitForRecorded() throws InterruptedException { /** */ public List blockedMessages() { - var res = blockedMsgs; - - return F.isEmpty(res) ? Collections.emptyList() : Collections.unmodifiableList(res); + synchronized (this) { + return new ArrayList<>(blockedMsgs); + } } /** From c13083aaa1272f64f611d3bc0e8700af8a69c2e1 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 4 Sep 2026 20:17:59 +0300 Subject: [PATCH 20/44] minor doc fix --- docs/_docs/snapshots/snapshots.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/_docs/snapshots/snapshots.adoc b/docs/_docs/snapshots/snapshots.adoc index eb2ab5024de8b..5afeb689779a3 100644 --- a/docs/_docs/snapshots/snapshots.adoc +++ b/docs/_docs/snapshots/snapshots.adoc @@ -292,9 +292,9 @@ control.(sh|bat) --snapshot restore snapshot_09062021 --increment 1 The status of current snapshot operations in the cluster can be obtained using the `control.sh|bat` script. Snapshot operations include: │ -* **Create** - Creating a full or incremental snapshot │ -* **Restore** - Restoring cache groups from a snapshot │ -* **Check** - Verifying snapshot consistency (can run in parallel for multiple snapshots) │ +* `Create` - Full or incremental snapshot creation +* `Restore` - Cache group restoration from snapshot +* `Check` - Snapshot consistency verification (parallel execution allowed) │ [tabs] -- From 9bfe62a9c241d7b32116340a17a931ecc8b4e182 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 14 Oct 2026 19:40:13 +0300 Subject: [PATCH 21/44] + master, fix --- .../snapshot/SnapshotStatusTask.java | 26 +++---------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index d092339d039e9..7ec2214fc2b3a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -29,13 +29,11 @@ import org.apache.ignite.IgniteLogger; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.management.api.NoArg; -import org.apache.ignite.internal.managers.discovery.IgniteClusterNode; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperationRequest; import org.apache.ignite.internal.processors.metric.impl.MetricUtils; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; -import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; +import org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T5; @@ -71,6 +69,9 @@ public class SnapshotStatusTask extends VisorMultiNodeTask jobNodes(VisorTaskArgument arg) { + if (!ignite.context().rollingUpgrade().features().isActive(CoreFeatureRegistry.ROLLING_UPGRADE_FEATURE)) + throw new IgniteException("The snapshot lists feature isn't active yet on the current cluster."); + return nodeIds(ignite.cluster().forServers().nodes()); } @@ -166,22 +167,6 @@ private static class SnapshotStatusJob extends SnapshotJob checkStatuses = null; for (var snpCheckMReg : ignite.context().metric()) { From 4f93605d6b2fe514ace25ccdd85149d57b864ac2 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 14 Oct 2026 19:40:13 +0300 Subject: [PATCH 22/44] + master, fix --- .../snapshot/SnapshotStatusCommand.java | 3 +- .../snapshot/SnapshotStatusTask.java | 62 ++-- .../internal/visor/VisorTaskArgument.java | 5 + .../TestRecordingCommunicationSpi.java | 2 +- ...SnapshotCheckStatusRollingUpgradeTest.java | 309 ++++++++++++++++++ .../AbstractRollingUpgradeTest.java | 1 + .../ManagementApiVersionValidationTest.java | 2 +- .../TestIgniteReleaseFeatures_2_19_4.java | 27 ++ 8 files changed, 380 insertions(+), 31 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_4.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index b2210a7e96975..7c811e24da13e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -36,6 +36,7 @@ import org.apache.ignite.internal.util.typedef.T5; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; +import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.management.SystemViewTask.SimpleType.NUMBER; import static org.apache.ignite.internal.management.SystemViewTask.SimpleType.STRING; @@ -58,7 +59,7 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand printer) { + @Override public void printResult(NoArg arg, @Nullable SnapshotStatus status, Consumer printer) { if (status == null) { printer.accept("There is no create or restore snapshot operation in progress."); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index d092339d039e9..2035fd45f34c7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -29,13 +29,11 @@ import org.apache.ignite.IgniteLogger; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.internal.management.api.NoArg; -import org.apache.ignite.internal.managers.discovery.IgniteClusterNode; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperationRequest; import org.apache.ignite.internal.processors.metric.impl.MetricUtils; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature; -import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; +import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeatureSet; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T5; @@ -54,6 +52,7 @@ import static org.apache.ignite.internal.management.snapshot.SnapshotStatusTask.SnapshotStatus; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.SNAPSHOT_METRICS; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotRestoreProcess.SNAPSHOT_RESTORE_METRICS; +import static org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE; import static org.apache.ignite.internal.util.lang.ClusterNodeFunc.nodeIds; /** @@ -64,6 +63,9 @@ public class SnapshotStatusTask extends VisorMultiNodeTask job(NoArg arg) { return new SnapshotStatusJob(arg, debug); @@ -71,6 +73,8 @@ public class SnapshotStatusTask extends VisorMultiNodeTask jobNodes(VisorTaskArgument arg) { + cmdInitiatorFeatures = arg.initiatorFeatures(); + return nodeIds(ignite.cluster().forServers().nodes()); } @@ -85,6 +89,10 @@ public class SnapshotStatusTask extends VisorMultiNodeTask chkStatuses = null; @@ -99,6 +107,16 @@ public class SnapshotStatusTask extends VisorMultiNodeTask checkStatuses = null; @@ -354,7 +360,7 @@ public static class SnapshotStatus implements Serializable { } /** */ - private SnapshotStatus(List checkStatuses) { + private SnapshotStatus(@Nullable List checkStatuses) { // Single, V1 status holds first check status. this( SnapshotOperation.CHECK, @@ -369,37 +375,37 @@ private SnapshotStatus(List checkStatuses) { } /** @return Operation type. */ - SnapshotOperation operation() { + public SnapshotOperation operation() { return op; } /** @return Snapshot name. */ - String name() { + public String name() { return name; } /** @return Incremental snapshot index. */ - int incrementIndex() { + public int incrementIndex() { return incIdx; } /** @return Request ID. */ - String requestId() { + public String requestId() { return requestId; } /** @return Start time. */ - long startTime() { + public long startTime() { return startTime; } /** @return Progress of operation on nodes. */ - Map> progress() { + public Map> progress() { return Collections.unmodifiableMap(progress); } /** @return Statuses of parallel check. */ - @Nullable List checkStatuses() { + public @Nullable List checkStatuses() { return checkStatuses; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorTaskArgument.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorTaskArgument.java index 55602bbb8f853..4b1bd9671e664 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorTaskArgument.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorTaskArgument.java @@ -137,6 +137,11 @@ public boolean isDebug() { return debug; } + /** */ + public IgniteCoreFeatureSet initiatorFeatures() { + return cmdInitiatorFeatures; + } + /** {@inheritDoc} */ @Override public void writeExternal(ObjectOutput out) throws IOException { out.writeInt(MAGIC); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java index 5ba3cecc3dd29..1a0324d92b310 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java @@ -62,7 +62,7 @@ public class TestRecordingCommunicationSpi extends TcpCommunicationSpi { private final List blockedMsgs = new ArrayList<>(); /** */ - private Map, Set> blockCls = new HashMap<>(); + private final Map, Set> blockCls = new HashMap<>(); /** */ private IgniteBiPredicate blockP; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java new file mode 100644 index 0000000000000..58a84f01514d8 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.nio.file.DirectoryStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.Collections; +import java.util.UUID; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.client.Config; +import org.apache.ignite.client.IgniteClient; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.ClientConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.management.api.NoArg; +import org.apache.ignite.internal.management.snapshot.SnapshotStatusTask; +import org.apache.ignite.internal.processors.rollingupgrade.AbstractRollingUpgradeTest; +import org.apache.ignite.internal.util.distributed.SingleNodeMessage; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.VisorTaskArgument; +import org.apache.ignite.internal.visor.VisorTaskResult; +import org.jetbrains.annotations.Nullable; +import org.junit.Test; + +import static java.nio.file.Files.newDirectoryStream; +import static org.apache.ignite.internal.TestRecordingCommunicationSpi.spi; +import static org.apache.ignite.internal.processors.rollingupgrade.feature.ManagementApiVersionValidationTest.withCoreVersion; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_METAS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RU_PREPARE_VERSION_FINALIZATION; +import static org.apache.ignite.testframework.GridTestUtils.runAsync; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** */ +public class IgniteClusterSnapshotCheckStatusRollingUpgradeTest extends AbstractRollingUpgradeTest { + /** */ + private static final int ALL_GRIDS = 4; + + /** */ + private static final int CLIENTS = 1; + + /** */ + private static final String SNP_NAME = "testSnapshot"; + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void cleanPersistenceDir() throws Exception { + super.cleanPersistenceDir(); + + // Clean all: also separated snapshot working directories. + try (DirectoryStream files = newDirectoryStream(Paths.get(U.defaultWorkDirectory()))) { + for (Path path : files) + U.delete(path); + } + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName, String ver) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName, ver); + + cfg.setDataStorageConfiguration( + new DataStorageConfiguration() + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setPersistenceEnabled(true) + .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE) + ) + ); + + cfg.setWorkDirectory(new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath()); + + return cfg; + } + + /** */ + @Test + public void testClientOfThePreviousVersion() throws Exception { + for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) + startGrid(i, "2.19.4", false); + + grid(0).cluster().state(ClusterState.ACTIVE); + + createCacheAndSnapshot(0); + + Collection nodes = Collections.singleton(nodeId(0)); + + // Client not supporting the snapshot check status. + VisorTaskArgument vtArg = withCoreVersion("2.18.0", () -> new VisorTaskArgument<>(nodes, new NoArg(), false)); + + ensureSnapshotCheckStatus(false, vtArg); + } + + /** */ + @Test + public void testParallelRollingUpgradeInProgress() throws Exception { + for (int i = 0; i < ALL_GRIDS; i++) + startGrid(i, "2.18.0", i >= ALL_GRIDS - CLIENTS); + + grid(0).cluster().active(true); + + int testNodeIx = ALL_GRIDS - CLIENTS - 1; + + createCacheAndSnapshot(testNodeIx); + + ensureSnapshotCheckStatus(false); + + ru(grid(testNodeIx)).enableVersionUpgrade(); + + for (int i = 0; i < ALL_GRIDS; i++) { + assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); + + upgradeNodeVersion(i, "2.18.0", "2.19.4"); + } + + spi(grid(testNodeIx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && + snm.type() == RU_PREPARE_VERSION_FINALIZATION.ordinal()); + + IgniteInternalFuture finalizeFut = runAsync(() -> ru(testNodeIx).finalizeClusterVersion()); + + assertTrue(spi(grid(testNodeIx)).waitForBlocked(1, getTestTimeout())); + + ensureSnapshotCheckStatus(false); + + spi(grid(testNodeIx)).stopBlock(); + + assertFalse(spi(grid(testNodeIx)).hasBlockedMessages()); + + finalizeFut.get(getTestTimeout()); + + for (int i = 0; i < ALL_GRIDS; i++) { + int i0 = i; + + assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout())); + } + + ensureSnapshotCheckStatus(true); + } + + /** */ + @Test + public void testSnapshotListFeature() throws Exception { + for (int i = 0; i < ALL_GRIDS; i++) + startGrid(i, "2.18.0", i >= ALL_GRIDS - CLIENTS); + + grid(0).cluster().active(true); + + createCacheAndSnapshot(1); + + ensureSnapshotCheckStatus(false); + + ru(grid(0)).enableVersionUpgrade(); + + for (int i = 0; i < ALL_GRIDS; i++) { + assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); + + upgradeNodeVersion(i, "2.18.0", "2.19.4"); + } + + ensureSnapshotCheckStatus(false); + + ru(grid(1)).finalizeClusterVersion(); + + for (int i = 0; i < ALL_GRIDS; i++) { + int i0 = i; + + assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout())); + } + + ensureSnapshotCheckStatus(true); + } + + /** */ + private void createCacheAndSnapshot(int gridIdx) { + int partsCnt = 4; + int keysCnt = partsCnt * 5; + + grid(gridIdx).createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setCacheMode(CacheMode.REPLICATED) + .setAffinity(new RendezvousAffinityFunction().setPartitions(partsCnt)) + .setAtomicityMode(CacheAtomicityMode.ATOMIC)); + + try (IgniteDataStreamer ds = grid(gridIdx).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 0; i < keysCnt; i++) + ds.addData(i, i); + } + + createSnapshot(gridIdx); + } + + /** */ + private void createSnapshot(int gridIdx) { + snp(gridIdx).createSnapshot(SNP_NAME).get(getTestTimeout()); + } + + /** */ + private void ensureSnapshotCheckStatus(boolean positive, @Nullable VisorTaskArgument clientArg) throws Exception { + int blockNodeIdx = 1; + + spi(grid(blockNodeIdx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && + snm.type() == CHECK_SNAPSHOT_METAS.ordinal()); + + IgniteInternalFuture chkFut = snp(1).checkSnapshot(SNP_NAME, null); + + assertTrue(waitForCondition(() -> spi(grid(blockNodeIdx)).hasBlockedMessages(), getTestTimeout())); + + SnapshotStatusTask.SnapshotStatus snpStatus; + + if (clientArg != null) { + snpStatus = snapshotStatusCli(clientArg); + + if (positive) + assertTrue(snpStatus != null && snpStatus.checkStatuses() != null && snpStatus.checkStatuses().size() == 1); + else + assertTrue(snpStatus == null); + } + else { + for (Ignite ig : G.allGrids()) { + snpStatus = snapshotStatusTask(ig); + + if (positive) + assertTrue(snpStatus != null && snpStatus.checkStatuses() != null && snpStatus.checkStatuses().size() == 1); + else + assertTrue(snpStatus == null); + } + } + + spi(grid(blockNodeIdx)).stopBlock(); + + assertTrue(waitForCondition(() -> !spi(grid(blockNodeIdx)).hasBlockedMessages(), getTestTimeout())); + + // Ensure that the snapshot check process correctly finished too. + SnapshotPartitionsVerifyResult chkRes = chkFut.get(getTestTimeout()); + + assertTrue(F.isEmpty(chkRes.exceptions())); + assertFalse(F.isEmpty(chkRes.metas())); + assertTrue(F.isEmpty(chkRes.idleVerifyResult().exceptions())); + assertFalse(chkRes.idleVerifyResult().hasConflicts()); + } + + /** */ + private void ensureSnapshotCheckStatus(boolean positive) throws Exception { + ensureSnapshotCheckStatus(positive, null); + } + + /** */ + private IgniteSnapshotManager snp(int gridIdx) { + return grid(gridIdx).context().cache().context().snapshotMgr(); + } + + /** */ + private static SnapshotStatusTask.SnapshotStatus snapshotStatusTask(Ignite grid) throws Exception { + Collection nodes = grid.cluster().nodes().stream().map(ClusterNode::id).toList(); + + VisorTaskArgument vtArg = new VisorTaskArgument<>(nodes, new NoArg(), false); + + return grid.compute().execute(SnapshotStatusTask.class, vtArg).result(); + } + + /** */ + private SnapshotStatusTask.SnapshotStatus snapshotStatusCli(VisorTaskArgument vtArg) throws Exception { + try (IgniteClient cli = Ignition.startClient(new ClientConfiguration().setAddresses(Config.SERVER))) { + VisorTaskResult vtRes = cli.compute().execute(SnapshotStatusTask.class.getName(), vtArg); + + return vtRes.result(); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java index 3d7f76477c251..4399c0ded6059 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java @@ -91,6 +91,7 @@ * 2.19.1 0-1 * 2.19.2 0-2 * 2.19.3 0-2,6 + * 2.19.4 0-1 * 2.20.0 2-5 * 2.20.1 2-6 * 2.21.0 6 diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/ManagementApiVersionValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/ManagementApiVersionValidationTest.java index b8af58f9f7cda..6cd07a0cc81e7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/ManagementApiVersionValidationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/ManagementApiVersionValidationTest.java @@ -94,7 +94,7 @@ private VisorTaskArgument createCommandArgument(int destNodeIdx, String } /** */ - private static R withCoreVersion(String ver, Callable action) throws Exception { + public static R withCoreVersion(String ver, Callable action) throws Exception { IgniteCoreFeatureSet prev = IgniteCoreFeatureSet.INSTANCE; IgniteCoreFeatureSet.INSTANCE = createCoreFeatureSet(ver); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_4.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_4.java new file mode 100644 index 0000000000000..1448c0adb263b --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_4.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.rollingupgrade.feature; + +/** */ +public class TestIgniteReleaseFeatures_2_19_4 { + /** */ + public static final IgniteFeature ROLLING_UPGRADE_FEATURE = TestIgniteReleaseFeatures_2_18_0.ROLLING_UPGRADE_FEATURE; + + /** */ + public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = CoreFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE; +} From e9ba27a91a7571a1058a47c0c27560cc21235a16 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Thu, 15 Oct 2026 15:53:15 +0300 Subject: [PATCH 23/44] reimpls iof teh result --- .../snapshot/SnapshotStatusCommand.java | 14 +- .../snapshot/SnapshotStatusTask.java | 127 ++++++++++++------ 2 files changed, 94 insertions(+), 47 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 7c811e24da13e..53679a5256600 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -66,21 +66,25 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand multipleOpsView; if (status.operation() == SnapshotStatusTask.SnapshotOperation.CHECK) { + assert s2 != null; + // Check operation always has itself in its aggregated check statuses. - multipleOpsView = status.checkStatuses(); + multipleOpsView = s2.checkStatuses(); } else { // If the operation is not check, attach possible parallel checks after. - multipleOpsView = status.checkStatuses() == null + multipleOpsView = s2 == null || s2.checkStatuses() == null ? Collections.singletonList(status) - : Stream.concat(Stream.of(status), status.checkStatuses().stream()).collect(Collectors.toList()); + : Stream.concat(Stream.of(status), s2.checkStatuses().stream()).collect(Collectors.toList()); } boolean first = true; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 2035fd45f34c7..4ab638d58faca 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -33,6 +33,7 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperationRequest; import org.apache.ignite.internal.processors.metric.impl.MetricUtils; +import org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry; import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeatureSet; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.typedef.F; @@ -91,7 +92,8 @@ public class SnapshotStatusTask extends VisorMultiNodeTask chkStatuses = null; @@ -108,18 +110,20 @@ public class SnapshotStatusTask extends VisorMultiNodeTask checkStatuses = s instanceof SnapshotOperationStatus s2 ? s2.checkStatuses() : null; + + if (checkStatuses == null) { assert s.op != SnapshotOperation.CHECK; continue; @@ -128,7 +132,7 @@ public class SnapshotStatusTask extends VisorMultiNodeTask progress.putAll(s.progress)); - mergerRes = new SnapshotStatus(mergerRes.op, mergerRes.name, mergerRes.incIdx, mergerRes.requestId, - mergerRes.startTime, progress); - } + if (checkStatusSupport) { + mergerRes = new SnapshotOperationStatus(mergerRes.op, mergerRes.name, mergerRes.incIdx, mergerRes.requestId, + mergerRes.startTime, progress); - if (chkStatuses != null) - mergerRes.checkStatuses = new ArrayList<>(chkStatuses.values()); + ((SnapshotOperationStatus)mergerRes).checkStatuses = new ArrayList<>(chkStatuses.values()); + } + else { + mergerRes = new SnapshotStatus(mergerRes.op, mergerRes.name, mergerRes.incIdx, mergerRes.requestId, + mergerRes.startTime, progress); + + assert chkStatuses == null; + } + } else { + assert checkStatusSupport; + assert mergerRes instanceof SnapshotOperationStatus; + + ((SnapshotOperationStatus)mergerRes).checkStatuses = new ArrayList<>(chkStatuses.values()); + } assert !F.isEmpty(mergerRes.progress); @@ -257,7 +273,10 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { return res; } - List checkStatuses = null; + if (res != null) + res = new SnapshotOperationStatus(res.op, res.name, res.incIdx, res.requestId, res.startTime, res.progress); + + List checkStatuses = null; for (var snpCheckMReg : ignite.context().metric()) { if (!snpCheckMReg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) @@ -293,7 +312,7 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { ); } - var status = new SnapshotStatus( + var status = new SnapshotOperationStatus( SnapshotOperation.CHECK, MetricUtils.fromFullName(snpCheckMReg.name()).get2(), incIdx, @@ -307,9 +326,12 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { if (checkStatuses != null) { if (res == null) - res = new SnapshotStatus(checkStatuses); - else - res.checkStatuses = checkStatuses; + res = new SnapshotOperationStatus(checkStatuses); + else { + assert res instanceof SnapshotOperationStatus; + + ((SnapshotOperationStatus)res).checkStatuses = checkStatuses; + } } return res; @@ -339,9 +361,6 @@ public static class SnapshotStatus implements Serializable { /** Progress of operation on nodes. */ private final Map> progress; - /** Nodes' statuses of all snapshot check operations. */ - private @Nullable List checkStatuses; - /** */ SnapshotStatus( SnapshotOperation op, @@ -359,21 +378,6 @@ public static class SnapshotStatus implements Serializable { this.progress = progress; } - /** */ - private SnapshotStatus(@Nullable List checkStatuses) { - // Single, V1 status holds first check status. - this( - SnapshotOperation.CHECK, - checkStatuses.get(0).name(), - checkStatuses.get(0).incrementIndex(), - checkStatuses.get(0).requestId(), - checkStatuses.get(0).startTime(), - checkStatuses.get(0).progress() - ); - - this.checkStatuses = checkStatuses; - } - /** @return Operation type. */ public SnapshotOperation operation() { return op; @@ -403,11 +407,6 @@ public long startTime() { public Map> progress() { return Collections.unmodifiableMap(progress); } - - /** @return Statuses of parallel check. */ - public @Nullable List checkStatuses() { - return checkStatuses; - } } /** Snapshot operation type. */ @@ -418,7 +417,51 @@ public enum SnapshotOperation { /** Restore snapshot. */ RESTORE, - /** Check snapshot. */ + /** + * Check snapshot. + * NOTE: Is supported only by {@link SnapshotOperationStatus}. + * + * @see CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE + */ CHECK } + + /** Snapshot operation status with the support {@link SnapshotOperation#CHECK}. */ + public static class SnapshotOperationStatus extends SnapshotStatus { + private static final long serialVersionUID = 0L; + + /** Nodes' statuses of all snapshot check operations. */ + private @Nullable List checkStatuses; + + /** */ + SnapshotOperationStatus( + SnapshotOperation op, + String name, + int incIdx, String requestId, + long startTime, + Map> progress + ) { + super(null, name, incIdx, requestId, startTime, progress); + } + + /** Constructor only for {@link SnapshotOperation#CHECK}. */ + SnapshotOperationStatus(List checkStatuses) { + // Holds first check status. + this( + SnapshotOperation.CHECK, + checkStatuses.get(0).name(), + checkStatuses.get(0).incrementIndex(), + checkStatuses.get(0).requestId(), + checkStatuses.get(0).startTime(), + checkStatuses.get(0).progress() + ); + + this.checkStatuses = checkStatuses; + } + + /** @return Statuses of parallel check. */ + public @Nullable List checkStatuses() { + return checkStatuses; + } + } } From 79fb992f408158df806e9fe537b9de29e48cd4d7 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Thu, 15 Oct 2026 16:13:45 +0300 Subject: [PATCH 24/44] reimppl iof teh result in progress --- .../management/snapshot/SnapshotStatusTask.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 4ab638d58faca..9fc956b79334d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -96,7 +96,7 @@ public class SnapshotStatusTask extends VisorMultiNodeTask chkStatuses = null; + Map chkStatuses = null; // Not empty results. Collection statuses = new ArrayList<>(results.size()); @@ -121,7 +121,7 @@ public class SnapshotStatusTask extends VisorMultiNodeTask checkStatuses = s instanceof SnapshotOperationStatus s2 ? s2.checkStatuses() : null; + List checkStatuses = s instanceof SnapshotOperationStatus s2 ? s2.checkStatuses() : null; if (checkStatuses == null) { assert s.op != SnapshotOperation.CHECK; @@ -132,7 +132,7 @@ public class SnapshotStatusTask extends VisorMultiNodeTask checkStatuses; + private @Nullable List checkStatuses; /** */ SnapshotOperationStatus( @@ -445,7 +445,7 @@ public static class SnapshotOperationStatus extends SnapshotStatus { } /** Constructor only for {@link SnapshotOperation#CHECK}. */ - SnapshotOperationStatus(List checkStatuses) { + SnapshotOperationStatus(List checkStatuses) { // Holds first check status. this( SnapshotOperation.CHECK, @@ -460,7 +460,7 @@ public static class SnapshotOperationStatus extends SnapshotStatus { } /** @return Statuses of parallel check. */ - public @Nullable List checkStatuses() { + public @Nullable List checkStatuses() { return checkStatuses; } } From 84ace45333f212bc1b696e17c43bf624efbb1ec5 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Thu, 15 Oct 2026 18:38:58 +0300 Subject: [PATCH 25/44] compiled + checkstyle --- .../snapshot/SnapshotStatusCommand.java | 11 ++++---- .../snapshot/SnapshotStatusTask.java | 27 +++++++++---------- ...SnapshotCheckStatusRollingUpgradeTest.java | 12 +++++---- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 53679a5256600..10fa459c7c50a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -66,13 +66,13 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand multipleOpsView; + List multipleOpsView; if (status.operation() == SnapshotStatusTask.SnapshotOperation.CHECK) { assert s2 != null; @@ -90,9 +90,12 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand { /** */ @@ -90,7 +88,7 @@ public class SnapshotStatusTask extends VisorMultiNodeTask> progress; + protected final Map> progress; /** */ - SnapshotStatus( + private SnapshotStatus( SnapshotOperation op, String name, int incIdx, @@ -426,26 +424,27 @@ public enum SnapshotOperation { CHECK } - /** Snapshot operation status with the support {@link SnapshotOperation#CHECK}. */ + /** Snapshot operation status with the support {@link SnapshotOperation#CHECK} and statuses of several shapshots checks. */ public static class SnapshotOperationStatus extends SnapshotStatus { private static final long serialVersionUID = 0L; - /** Nodes' statuses of all snapshot check operations. */ + /** Nodes' statuses of all snapshot checks operations. */ private @Nullable List checkStatuses; /** */ - SnapshotOperationStatus( + private SnapshotOperationStatus( SnapshotOperation op, String name, - int incIdx, String requestId, + int incIdx, + String reqId, long startTime, Map> progress ) { - super(null, name, incIdx, requestId, startTime, progress); + super(op, name, incIdx, reqId, startTime, progress); } /** Constructor only for {@link SnapshotOperation#CHECK}. */ - SnapshotOperationStatus(List checkStatuses) { + private SnapshotOperationStatus(List checkStatuses) { // Holds first check status. this( SnapshotOperation.CHECK, diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java index 58a84f01514d8..c52029c23d805 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java @@ -245,7 +245,7 @@ private void ensureSnapshotCheckStatus(boolean positive, @Nullable VisorTaskArgu assertTrue(waitForCondition(() -> spi(grid(blockNodeIdx)).hasBlockedMessages(), getTestTimeout())); - SnapshotStatusTask.SnapshotStatus snpStatus; + SnapshotStatusTask.SnapshotOperationStatus snpStatus; if (clientArg != null) { snpStatus = snapshotStatusCli(clientArg); @@ -290,18 +290,20 @@ private IgniteSnapshotManager snp(int gridIdx) { } /** */ - private static SnapshotStatusTask.SnapshotStatus snapshotStatusTask(Ignite grid) throws Exception { + private static T snapshotStatusTask(Ignite grid) throws Exception { Collection nodes = grid.cluster().nodes().stream().map(ClusterNode::id).toList(); VisorTaskArgument vtArg = new VisorTaskArgument<>(nodes, new NoArg(), false); - return grid.compute().execute(SnapshotStatusTask.class, vtArg).result(); + return (T)grid.compute().execute(SnapshotStatusTask.class, vtArg).result(); } /** */ - private SnapshotStatusTask.SnapshotStatus snapshotStatusCli(VisorTaskArgument vtArg) throws Exception { + private T snapshotStatusCli(VisorTaskArgument vtArg) throws Exception { + String taskName = SnapshotStatusTask.class.getName(); + try (IgniteClient cli = Ignition.startClient(new ClientConfiguration().setAddresses(Config.SERVER))) { - VisorTaskResult vtRes = cli.compute().execute(SnapshotStatusTask.class.getName(), vtArg); + VisorTaskResult vtRes = cli.compute().execute(taskName, vtArg); return vtRes.result(); } From 651581ed52f1f21453a833147c027daa909c9a1d Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Thu, 15 Oct 2026 20:22:07 +0300 Subject: [PATCH 26/44] checkstyle --- .../internal/management/snapshot/SnapshotStatusTask.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 8a68416ca8220..8cc65d7e0fe2d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -177,9 +177,10 @@ public class SnapshotStatusTask extends VisorMultiNodeTask(chkStatuses.values()); } From 075c2170a3e6f012fbe5a1a9c267bea52e1660c0 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Mon, 19 Oct 2026 16:54:19 +0300 Subject: [PATCH 27/44] + master --- .../rollingupgrade/feature/CoreFeatureRegistry.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java index 664226a4b3f83..25991c4f48457 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java @@ -99,4 +99,9 @@ public class CoreFeatureRegistry { * positional fields of every message. */ public static final IgniteFeature RAW_FIELDS_FEATURE = new IgniteCoreFeature(1); + + /** + * Feature that introduces the snapshots list command. + */ + public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = new IgniteCoreFeature(3); } From 1606feaa0e7f9cd5a083848af7feb00b992f0c4f Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Mon, 19 Oct 2026 17:35:04 +0300 Subject: [PATCH 28/44] in progress --- .../snapshot/SnapshotStatusTask.java | 73 ++++++++++--------- .../feature/CoreFeatureRegistry.java | 5 ++ 2 files changed, 42 insertions(+), 36 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 6cab50b2079a1..146590ecbda20 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -48,9 +48,9 @@ import org.apache.ignite.spi.metric.IntMetric; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.spi.metric.Metric; +import org.apache.ignite.spi.metric.ReadOnlyMetricRegistry; import org.jetbrains.annotations.Nullable; -import static org.apache.ignite.internal.management.snapshot.SnapshotStatusTask.SnapshotStatus; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.SNAPSHOT_METRICS; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotRestoreProcess.SNAPSHOT_RESTORE_METRICS; import static org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE; @@ -97,11 +97,7 @@ public class SnapshotStatusTask extends VisorMultiNodeTask chkStatuses = null; + boolean featureEnabled = ignite.context().rollingUpgrade().features().isActive(SNAPSHOT_CHECK_STATUS_FEATURE); // Not empty results. Collection statuses = new ArrayList<>(results.size()); @@ -110,12 +106,23 @@ public class SnapshotStatusTask extends VisorMultiNodeTask : + "Unsupported job result: " + rawResult.getClass().getSimpleName(); + + assert !(rawResult instanceof Collection cll) || cll.isEmpty() || (F.first(cll) instanceof SnapshotStatus) : + "Statuses collection is empty or its first status is of unsupported type"; + + if (rawResult instanceof SnapshotStatus singleStatus) { - SnapshotStatus s = jobRes.getData(); + } + else { + assert cmdInitiatorFeatures.features().contains(SNAPSHOT_CHECK_STATUS_FEATURE.id()); + } // Command initiator doesn't support the snapshot check status yet. - if (!checkStatusSupport && s instanceof SnapshotOperationStatus) { + if (!initiatorSupportsStatuses && s instanceof SnapshotOperationStatus) { // Do not store the status if there are only stnapshot checks (no other oparations). if (s.operation() == SnapshotOperation.CHECK) continue; @@ -172,7 +179,7 @@ public class SnapshotStatusTask extends VisorMultiNodeTask progress.putAll(s.progress)); - if (checkStatusSupport) { + if (initiatorSupportsStatuses) { mergerRes = new SnapshotOperationStatus(mergerRes.op, mergerRes.name, mergerRes.incIdx, mergerRes.requestId, mergerRes.startTime, progress); @@ -185,7 +192,7 @@ public class SnapshotStatusTask extends VisorMultiNodeTask(chkStatuses.values()); @@ -229,7 +236,7 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { SnapshotOperationRequest req = snpMgr.currentCreateRequest(); // Only one operation like creation or restoration is allowed at the same time. - SnapshotStatus createOrRestore = null; + SnapshotStatus createOrRestoreStatus = null; if (req != null) { T5 metrics; @@ -245,7 +252,7 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { -1L, -1L, -1L); } - createOrRestore = new SnapshotStatus( + createOrRestoreStatus = new SnapshotStatus( SnapshotOperation.CREATE, req.snapshotName(), req.incrementIndex(), @@ -260,7 +267,7 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { long startTime = mreg.findMetric("startTime").value(); if (startTime > mreg.findMetric("endTime").value()) { - createOrRestore = new SnapshotStatus( + createOrRestoreStatus = new SnapshotStatus( SnapshotOperation.RESTORE, mreg.findMetric("snapshotName").getAsString(), mreg.findMetric("incrementIndex").value(), @@ -280,20 +287,25 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { } } - /** Return old simgle-one result */ + /** Return old single result. */ if (!ignite.context().rollingUpgrade().features().isActive(SNAPSHOT_CHECK_STATUS_FEATURE)) { log.warning("The snapshot check status feature isn't activated yet. The status is available only " + "for snapshot creation and restoration."); - return createOrRestore; + return createOrRestoreStatus; } - if (createOrRestore != null) - createOrRestore = new SnapshotOperationStatus(createOrRestore.op, createOrRestore.name, createOrRestore.incIdx, createOrRestore.requestId, createOrRestore.startTime, createOrRestore.progress); + // Many results. + ArrayList severalStatuses = null; - List checkStatuses = null; + if (createOrRestoreStatus != null) { + severalStatuses = new ArrayList<>(); - for (var snpCheckMReg : ignite.context().metric()) { + // Status of creation or restoration operation always goes firts. This is important for old command initiators (control.sh). + severalStatuses.add(createOrRestoreStatus); + } + + for (ReadOnlyMetricRegistry snpCheckMReg : ignite.context().metric()) { if (!snpCheckMReg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) continue; @@ -303,9 +315,6 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { if (rqIdMetric == null) continue; - if (checkStatuses == null) - checkStatuses = new ArrayList<>(); - int incIdx = snpCheckMReg.findMetric("incrementIndex") == null ? 0 : snpCheckMReg.findMetric("incrementIndex").value(); @@ -327,7 +336,7 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { ); } - SnapshotOperationStatus status = new SnapshotOperationStatus( + SnapshotStatus status = new SnapshotStatus( SnapshotOperation.CHECK, MetricUtils.fromFullName(snpCheckMReg.name()).get2(), incIdx, @@ -336,20 +345,13 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { F.asMap(ignite.localNode().id(), metrics) ); - checkStatuses.add(status); - } + if (severalStatuses == null) + severalStatuses = new ArrayList<>(); - if (checkStatuses != null) { - if (createOrRestore == null) - createOrRestore = new SnapshotOperationStatus(checkStatuses); - else { - assert createOrRestore instanceof SnapshotOperationStatus; - - ((SnapshotOperationStatus)createOrRestore).checkStatuses = checkStatuses; - } + severalStatuses.add(status); } - return createOrRestore; + return severalStatuses; } } @@ -434,7 +436,6 @@ public enum SnapshotOperation { /** * Check snapshot. - * NOTE: Is supported only by {@link SnapshotOperationStatus}. * * @see CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java index 664226a4b3f83..25991c4f48457 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java @@ -99,4 +99,9 @@ public class CoreFeatureRegistry { * positional fields of every message. */ public static final IgniteFeature RAW_FIELDS_FEATURE = new IgniteCoreFeature(1); + + /** + * Feature that introduces the snapshots list command. + */ + public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = new IgniteCoreFeature(3); } From 086a468a960de964eb43c817e2a251acabaad657 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Tue, 20 Oct 2026 00:40:45 +0300 Subject: [PATCH 29/44] reimpl result --- .../ignite/util/GridCommandHandlerTest.java | 78 +---------- .../snapshot/SnapshotStatusCommand.java | 35 ++--- .../snapshot/SnapshotStatusTask.java | 127 +++++++++--------- ...SnapshotCheckStatusRollingUpgradeTest.java | 14 +- 4 files changed, 89 insertions(+), 165 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 50bef27bd8e96..73d6a8ba7a044 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -77,7 +77,6 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteInterruptedCheckedException; -import org.apache.ignite.internal.IgniteVersionUtils; import org.apache.ignite.internal.Order; import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.dto.IgniteDataTransferObject; @@ -117,10 +116,6 @@ import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor; import org.apache.ignite.internal.processors.datastreamer.DataStreamerRequest; import org.apache.ignite.internal.processors.metric.MetricRegistryImpl; -import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor; -import org.apache.ignite.internal.processors.rollingupgrade.RollingUpgradeProcessor; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeatureSet; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureSet; import org.apache.ignite.internal.util.BasicRateLimiter; import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.internal.util.distributed.DistributedProcess; @@ -140,11 +135,8 @@ import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.metric.MetricRegistry; -import org.apache.ignite.plugin.AbstractTestPluginProvider; -import org.apache.ignite.plugin.PluginContext; import org.apache.ignite.plugin.PluginProvider; import org.apache.ignite.plugin.extensions.communication.Message; -import org.apache.ignite.spi.IgniteNodeValidationResult; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.spi.metric.Metric; @@ -3670,52 +3662,9 @@ public void testSnapshotRestoreCancelAndStatus() throws Exception { assertNull(ig.cache(DEFAULT_CACHE_NAME)); } - /** - * Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. - * All nodes support the check status. - */ - @Test - public void testRestoreSnapshotMetricsAtStartAllNodesSupport() throws Exception { - doTestRestoreSnapshotMetricsAtStart(false); - } - - /** - * Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. - * One node doesn't support the check status. - */ + /** Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. */ @Test - public void testRestoreSnapshotMetricsAtStartOneNodeDoesntSupport() throws Exception { - doTestRestoreSnapshotMetricsAtStart(true); - } - - /** - * @param oneNodeDoesntSupport If {@code true}, one node will simulate not supporting snapshot check status. - */ - private void doTestRestoreSnapshotMetricsAtStart(boolean oneNodeDoesntSupport) throws Exception { - // Creates empty feature set unsupporting the snapshot check ststus if required. - pluginProvider = !oneNodeDoesntSupport ? null : new AbstractTestPluginProvider() { - @Override public String name() { - return "Test Ignite features provider"; - } - - @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { - if (!cls.equals(DiscoveryNodeValidationProcessor.class)) - return null; - - boolean doNotSupport = ctx.igniteConfiguration().getIgniteInstanceName().equals(getTestIgniteInstanceName(2)); - - return (T)new RollingUpgradeProcessor( - ((IgniteEx)ctx.grid()).context(), - doNotSupport ? new IgniteCoreFeatureSet(IgniteVersionUtils.VER, new IgniteFeatureSet()) : IgniteCoreFeatureSet.local() - ) { - @Override public @Nullable IgniteNodeValidationResult validateNode(ClusterNode joiningNode) { - // Simulates started rolling updrade allowing node with other features join cluster. - return null; - } - }; - } - }; - + public void testRestoreSnapshotMetricsAtStart() throws Exception { listeningLog = new ListeningTestLogger(log); startGrids(3).cluster().state(ClusterState.ACTIVE); @@ -3735,7 +3684,7 @@ private void doTestRestoreSnapshotMetricsAtStart(boolean oneNodeDoesntSupport) t // Block one of the process' first messages of snapshot restoring or snapshot checking. F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm - && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() || sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()))); + && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() || sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()))); // Snapshot restoration should get paused at the preceeding snapshot check. IgniteFutureImpl restoreFut = snpMgr.restoreSnapshot("test_snapshot", null, null, 0, true); @@ -3749,29 +3698,14 @@ private void doTestRestoreSnapshotMetricsAtStart(boolean oneNodeDoesntSupport) t injectTestSystemOut(); - LogListener logLsnr = null; - - if (oneNodeDoesntSupport) { - logLsnr = LogListener.matches("Node %s doesn't support the snapshot check status feature" - .formatted(grid(2).localNode().id())).build(); - - listeningLog.registerListener(logLsnr); - } - int code = execute("--snapshot", "status"); assertEquals("Unexpected exit code", EXIT_CODE_OK, code); - var out = testOut.toString(); + String out = testOut.toString(); - if (oneNodeDoesntSupport) { - assertTrue(logLsnr.check(getTestTimeout())); - assertContains(log, out, "There is no create or restore snapshot operation in progress"); - } - else { - assertContains(log, out, "Check snapshot operation is in progress"); - assertContains(log, out, "Snapshot name: test_snapshot"); - } + assertContains(log, out, "Check snapshot operation is in progress"); + assertContains(log, out, "Snapshot name: test_snapshot"); // Ensure that no snapshot restore started or finished. assertFalse("Snapshot future has finished", restoreFut.isDone()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 10fa459c7c50a..33a61e9e300df 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -17,8 +17,10 @@ package org.apache.ignite.internal.management.snapshot; +import java.io.Serializable; import java.text.DateFormat; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; @@ -26,7 +28,6 @@ import java.util.UUID; import java.util.function.Consumer; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.ignite.internal.management.SystemViewCommand; import org.apache.ignite.internal.management.SystemViewTask; import org.apache.ignite.internal.management.api.NoArg; @@ -42,7 +43,7 @@ import static org.apache.ignite.internal.management.SystemViewTask.SimpleType.STRING; /** */ -public class SnapshotStatusCommand extends AbstractSnapshotCommand { +public class SnapshotStatusCommand extends AbstractSnapshotCommand { /** {@inheritDoc} */ @Override public String description() { return "Get the status of the current snapshot operation"; @@ -59,37 +60,27 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand printer) { - if (status == null) { + @Override public void printResult(NoArg arg, @Nullable Serializable rawResult, Consumer printer) { + if (rawResult == null) { printer.accept("There is no create or restore snapshot operation in progress."); return; } - // Status with the support of the sanpshot checks operations. - SnapshotStatusTask.SnapshotOperationStatus s2 = status instanceof SnapshotStatusTask.SnapshotOperationStatus - ? (SnapshotStatusTask.SnapshotOperationStatus)status - : null; + assert rawResult instanceof SnapshotStatus || rawResult instanceof Collection : + "Unsupported job result: " + rawResult.getClass().getSimpleName(); - // The check operation can be run in parallel for different snapshots. - List multipleOpsView; + SnapshotStatus singleStatus = rawResult instanceof SnapshotStatus ? (SnapshotStatus)rawResult : null; + Collection statuses = rawResult instanceof Collection ? (Collection)rawResult : null; - if (status.operation() == SnapshotStatusTask.SnapshotOperation.CHECK) { - assert s2 != null; + assert statuses == null || (!statuses.isEmpty() || (F.first(statuses) instanceof SnapshotStatus)) : + "Statuses collection has unsupported result type"; - // Check operation always has itself in its aggregated check statuses. - multipleOpsView = s2.checkStatuses(); - } - else { - // If the operation is not check, attach possible parallel checks after. - multipleOpsView = s2 == null || s2.checkStatuses() == null - ? Collections.singletonList(status) - : Stream.concat(Stream.of(status), s2.checkStatuses().stream()).collect(Collectors.toList()); - } + statuses = statuses == null ? Collections.singleton(singleStatus) : statuses; boolean first = true; - for (SnapshotStatus s0 : multipleOpsView) { + for (SnapshotStatus s0 : statuses) { if (!first) { printer.accept(U.nl()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 146590ecbda20..30693dfe98390 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -85,7 +85,7 @@ public class SnapshotStatusTask extends VisorMultiNodeTask results) { + @Override protected @Nullable Serializable reduce0(List results) { if (results.isEmpty()) throw new IgniteException("Failed to get the snapshot status. Topology is empty."); @@ -98,109 +98,108 @@ public class SnapshotStatusTask extends VisorMultiNodeTask statuses = new ArrayList<>(results.size()); + Collection allStatuses = new ArrayList<>(results.size()); for (ComputeJobResult jobRes : results) { if (jobRes.getData() == null) continue; + /** Old single {@link SnapshotStatus} of creation or restoration status or new several results as a collection. */ Serializable rawResult = jobRes.getData(); assert rawResult instanceof SnapshotStatus || rawResult instanceof Collection : - "Unsupported job result: " + rawResult.getClass().getSimpleName(); + "Unsupported job result type: " + rawResult.getClass().getSimpleName(); assert !(rawResult instanceof Collection cll) || cll.isEmpty() || (F.first(cll) instanceof SnapshotStatus) : "Statuses collection is empty or its first status is of unsupported type"; - if (rawResult instanceof SnapshotStatus singleStatus) { + if (rawResult instanceof SnapshotStatus singleJobStatus) { + assert !featureEnabled : + "Single snapshot status result is allowed only if the snapshot check status feature is not enabled"; + assert singleJobStatus.operation() != SnapshotOperation.CHECK : + "Snapshot check status result is allowed only within the result list, not as a single result"; + allStatuses.add(singleJobStatus); } else { - assert cmdInitiatorFeatures.features().contains(SNAPSHOT_CHECK_STATUS_FEATURE.id()); - } + assert featureEnabled : + "Many snapshot status results are allowed only if the snapshot check status feature is enabled"; - // Command initiator doesn't support the snapshot check status yet. - if (!initiatorSupportsStatuses && s instanceof SnapshotOperationStatus) { - // Do not store the status if there are only stnapshot checks (no other oparations). - if (s.operation() == SnapshotOperation.CHECK) - continue; + Collection manyStatuses = (Collection)rawResult; - // Convert into the previious version for corrent serialization. - s = new SnapshotStatus(s.operation(), s.name(), s.incrementIndex(), s.requestId(), s.startTime(), s.progress()); - } + // Skip the check status if cluster supports, but not a thin client of previous version. + if (!initiatorSupport) { + manyStatuses = manyStatuses.stream().filter(s -> s.operation() != SnapshotOperation.CHECK).toList(); - statuses.add(s); + assert manyStatuses.isEmpty() || manyStatuses.size() == 1 : + "Snapshot creation or restoration are mutually exclusive, only one of them allowed at the same time."; + } - List checkStatuses = s instanceof SnapshotOperationStatus s2 ? s2.checkStatuses() : null; + allStatuses.addAll(manyStatuses); + } + } - if (checkStatuses == null) { - assert s.op != SnapshotOperation.CHECK; + // There is no snapshot operation or the check operation in not supported yet in the result. + if (allStatuses.isEmpty()) + return null; - continue; - } + ArrayList tastResultStatuses = null; - if (chkStatuses == null) - chkStatuses = U.newHashMap(results.size()); + // First, find create or restore operation. + Collection filtered = F.viewReadOnly(allStatuses, s -> s, s -> s.operation() != SnapshotOperation.CHECK); - for (SnapshotOperationStatus s0 : checkStatuses) { - SnapshotOperationStatus prev = chkStatuses.putIfAbsent(s0.name(), s0); + // Found create or restore statuses. + if (!filtered.isEmpty()) { + // Merged create or restore. + SnapshotStatus merged = F.first(filtered); - if (prev == null) - continue; + String mergedRqId = merged.requestId; - // Merge nodes progress. - prev.progress.putAll(s0.progress()); - } - } + // Filter out differing requests due to concurrent updates on nodes. + filtered = F.view(filtered, s -> s.requestId.equals(mergedRqId)); - // First, find create or restore operation. Check statuses, if are, contains any status. - Collection res = F.viewReadOnly(statuses, s -> s, s -> s.operation() != SnapshotOperation.CHECK); + // Merge nodes progress. + Map> progress = new HashMap<>(); - if (res.isEmpty()) - res = statuses; + filtered.forEach(s -> progress.putAll(s.progress)); - // There is no snapshot operation. - if (res.isEmpty()) - return null; + merged = new SnapshotStatus(merged.op, merged.name, merged.incIdx, merged.requestId, merged.startTime, progress); - SnapshotStatus mergerRes = F.first(res); + // Return the old style single result. + if (!initiatorSupport || !featureEnabled) + return merged; - var rqId = mergerRes.requestId; + if (tastResultStatuses == null) + tastResultStatuses = new ArrayList<>(); - // Filter out differing requests due to concurrent updates on nodes. - res = F.view(res, s -> s.requestId.equals(rqId)); + tastResultStatuses.add(merged); + } - // Create or restore. - if (mergerRes.operation() != SnapshotOperation.CHECK) { - // Merge nodes progress. - Map> progress = new HashMap<>(); + // Check operation statuses. + filtered = F.viewReadOnly(allStatuses, s -> s, s -> s.operation() == SnapshotOperation.CHECK); - res.forEach(s -> progress.putAll(s.progress)); + // Found check statuses. + if (!filtered.isEmpty()) { + // Merged check statuses per snapshot name. + Map mergedCheckStatuses = U.newHashMap(filtered.size()); - if (initiatorSupportsStatuses) { - mergerRes = new SnapshotOperationStatus(mergerRes.op, mergerRes.name, mergerRes.incIdx, mergerRes.requestId, - mergerRes.startTime, progress); + filtered.forEach(chkStatus -> { + SnapshotStatus chkStatus0 = mergedCheckStatuses.computeIfAbsent(chkStatus.name, (snpName -> chkStatus)); - ((SnapshotOperationStatus)mergerRes).checkStatuses = new ArrayList<>(chkStatuses.values()); - } - else { - mergerRes = new SnapshotStatus(mergerRes.op, mergerRes.name, mergerRes.incIdx, mergerRes.requestId, - mergerRes.startTime, progress); + if (chkStatus0 != chkStatus) + chkStatus0.progress.putAll(chkStatus.progress); + }); - assert chkStatuses == null; - } - } else { - assert initiatorSupportsStatuses; - assert mergerRes instanceof SnapshotOperationStatus; + if (tastResultStatuses == null) + tastResultStatuses = new ArrayList<>(); - ((SnapshotOperationStatus)mergerRes).checkStatuses = new ArrayList<>(chkStatuses.values()); + tastResultStatuses.addAll(mergedCheckStatuses.values()); } - assert !F.isEmpty(mergerRes.progress); - - return mergerRes; + return tastResultStatuses; } /** @@ -376,7 +375,7 @@ public static class SnapshotStatus implements Serializable { private final long startTime; /** Progress of operation on nodes. */ - protected final Map> progress; + Map> progress; /** */ private SnapshotStatus( diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java index c52029c23d805..a5ee0b55f5cee 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java @@ -245,24 +245,24 @@ private void ensureSnapshotCheckStatus(boolean positive, @Nullable VisorTaskArgu assertTrue(waitForCondition(() -> spi(grid(blockNodeIdx)).hasBlockedMessages(), getTestTimeout())); - SnapshotStatusTask.SnapshotOperationStatus snpStatus; + Collection snpStatuses; if (clientArg != null) { - snpStatus = snapshotStatusCli(clientArg); + snpStatuses = snapshotStatusCli(clientArg); if (positive) - assertTrue(snpStatus != null && snpStatus.checkStatuses() != null && snpStatus.checkStatuses().size() == 1); + assertTrue(snpStatuses != null && snpStatuses.size() == 1); else - assertTrue(snpStatus == null); + assertTrue(snpStatuses == null); } else { for (Ignite ig : G.allGrids()) { - snpStatus = snapshotStatusTask(ig); + snpStatuses = snapshotStatusTask(ig); if (positive) - assertTrue(snpStatus != null && snpStatus.checkStatuses() != null && snpStatus.checkStatuses().size() == 1); + assertTrue(snpStatuses != null && snpStatuses.size() == 1); else - assertTrue(snpStatus == null); + assertTrue(snpStatuses == null); } } From 50467cbc01339e7e45a98ce8823b6c46f5ad45d0 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Tue, 20 Oct 2026 20:43:56 +0300 Subject: [PATCH 30/44] fixes --- .../ignite/util/GridCommandHandlerTest.java | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 73d6a8ba7a044..ff14e7fe7e16b 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -135,7 +135,6 @@ import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.metric.MetricRegistry; -import org.apache.ignite.plugin.PluginProvider; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.spi.metric.LongMetric; @@ -150,7 +149,6 @@ import org.apache.ignite.transactions.TransactionRollbackException; import org.apache.ignite.transactions.TransactionTimeoutException; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.junit.Assume; import org.junit.Test; @@ -230,7 +228,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb public static final String REENCRYPTION_SUSPEND = "suspend_reencryption"; /** */ - protected static File defaultDiagnosticDir; + protected static File dfltDiagnosticDir; /** */ protected static File customDiagnosticDir; @@ -238,9 +236,6 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb /** */ protected ListeningTestLogger listeningLog; - /** */ - protected @Nullable PluginProvider pluginProvider; - /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); @@ -264,9 +259,6 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb if (listeningLog != null) cfg.setGridLogger(listeningLog); - if (pluginProvider != null) - cfg.setPluginProviders(pluginProvider); - return cfg; } @@ -274,7 +266,7 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb * @throws IgniteCheckedException If failed. */ protected void initDiagnosticDir() throws IgniteCheckedException { - defaultDiagnosticDir = new File(U.defaultWorkDirectory() + dfltDiagnosticDir = new File(U.defaultWorkDirectory() + separatorChar + DEFAULT_TARGET_FOLDER + separatorChar); customDiagnosticDir = new File(U.defaultWorkDirectory() @@ -285,7 +277,7 @@ protected void initDiagnosticDir() throws IgniteCheckedException { * Clean diagnostic directories. */ protected void cleanDiagnosticDir() { - U.delete(defaultDiagnosticDir); + U.delete(dfltDiagnosticDir); U.delete(customDiagnosticDir); } @@ -2378,7 +2370,7 @@ public void testDiagnosticPageLocksTracker() throws Exception { ); // Check file dump in default path. - checkNumberFiles(defaultDiagnosticDir, 1); + checkNumberFiles(dfltDiagnosticDir, 1); assertEquals( EXIT_CODE_OK, @@ -2401,7 +2393,7 @@ public void testDiagnosticPageLocksTracker() throws Exception { ); // Current cluster 4 nodes -> 4 files + 1 from previous operation. - checkNumberFiles(defaultDiagnosticDir, 5); + checkNumberFiles(dfltDiagnosticDir, 5); assertEquals( EXIT_CODE_OK, @@ -2425,7 +2417,7 @@ public void testDiagnosticPageLocksTracker() throws Exception { ); // Dump locks only for 2 nodes -> 2 files + 5 from previous operation. - checkNumberFiles(defaultDiagnosticDir, 7); + checkNumberFiles(dfltDiagnosticDir, 7); // Dump locks only for 2 nodes use constIds as arg. assertEquals( From c01455d122fc54c6bd44a2b5ed2139a3abf33e48 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Wed, 28 Oct 2026 16:46:02 +0300 Subject: [PATCH 31/44] to idtos --- .../snapshot/SnapshotStatusCommand.java | 21 +- .../snapshot/SnapshotStatusTask.java | 185 +++++++++++++----- ....SnapshotStatusTask$SnapshotOperation.json | 8 + ...hotStatusTask$SnapshotOperationStatus.json | 11 ++ 4 files changed, 173 insertions(+), 52 deletions(-) create mode 100644 modules/wire-compatibility-check/wire/enums/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation.json create mode 100644 modules/wire-compatibility-check/wire/idto/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperationStatus.json diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 33a61e9e300df..c460655e9224a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -70,17 +70,26 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand : "Unsupported job result: " + rawResult.getClass().getSimpleName(); - SnapshotStatus singleStatus = rawResult instanceof SnapshotStatus ? (SnapshotStatus)rawResult : null; - Collection statuses = rawResult instanceof Collection ? (Collection)rawResult : null; + SnapshotStatus singleOldStatus = rawResult instanceof SnapshotStatus ? (SnapshotStatus)rawResult : null; + Collection statuses = rawResult instanceof Collection + ? (Collection)rawResult + : null; - assert statuses == null || (!statuses.isEmpty() || (F.first(statuses) instanceof SnapshotStatus)) : - "Statuses collection has unsupported result type"; + assert statuses == null || statuses.isEmpty() || F.first(statuses) instanceof SnapshotStatusTask.SnapshotOperationStatus : + "Statuses collection has an unsupported result type"; - statuses = statuses == null ? Collections.singleton(singleStatus) : statuses; + // Empty old-style result is null. Enpty new-style result is an empty collection. + if (statuses != null && statuses.isEmpty()) { + printer.accept("There is no any snapshot operation in progress."); + + return; + } + + statuses = statuses == null ? Collections.singleton(new SnapshotStatusTask.SnapshotOperationStatus(singleOldStatus)) : statuses; boolean first = true; - for (SnapshotStatus s0 : statuses) { + for (SnapshotStatusTask.SnapshotOperationStatus s0 : statuses) { if (!first) { printer.accept(U.nl()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 30693dfe98390..e37fa55937cf3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -28,6 +28,8 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; import org.apache.ignite.internal.management.api.NoArg; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; @@ -60,9 +62,9 @@ * Task to get the statuses of the current snapshot operations in the cluster. *

Before {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}, returns single {@link SnapshotStatus} for single * snapshot creation or snapshot restoration operation. - *

With {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}, returns collection of {@link SnapshotStatus} for - * possible several snapshot checks operations or parallel creation/restoration and checks operations. Might also contain - * a single status. + *

With {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}, returns collection of {@link SnapshotOperationStatus} + * for possible several snapshot checks operations or parallel creation/restoration and checks operations. May still + * contain a single status. */ @GridInternal public class SnapshotStatusTask extends VisorMultiNodeTask { @@ -98,10 +100,10 @@ public class SnapshotStatusTask extends VisorMultiNodeTask allStatuses = new ArrayList<>(results.size()); + // All the jobs' statuses combined to one flat collection. + Collection allStatuses = new ArrayList<>(results.size()); for (ComputeJobResult jobRes : results) { if (jobRes.getData() == null) @@ -112,33 +114,32 @@ public class SnapshotStatusTask extends VisorMultiNodeTask : "Unsupported job result type: " + rawResult.getClass().getSimpleName(); - - assert !(rawResult instanceof Collection cll) || cll.isEmpty() || (F.first(cll) instanceof SnapshotStatus) : + assert !(rawResult instanceof Collection cll) || cll.isEmpty() || (F.first(cll) instanceof SnapshotOperationStatus) : "Statuses collection is empty or its first status is of unsupported type"; if (rawResult instanceof SnapshotStatus singleJobStatus) { assert !featureEnabled : "Single snapshot status result is allowed only if the snapshot check status feature is not enabled"; assert singleJobStatus.operation() != SnapshotOperation.CHECK : - "Snapshot check status result is allowed only within the result list, not as a single result"; + "Status of the check operation is allowed only within the new result list, not as a single result"; - allStatuses.add(singleJobStatus); + allStatuses.add(new SnapshotOperationStatus(singleJobStatus)); } else { assert featureEnabled : "Many snapshot status results are allowed only if the snapshot check status feature is enabled"; - Collection manyStatuses = (Collection)rawResult; + Collection jobMultipleStatuses = (Collection)rawResult; // Skip the check status if cluster supports, but not a thin client of previous version. - if (!initiatorSupport) { - manyStatuses = manyStatuses.stream().filter(s -> s.operation() != SnapshotOperation.CHECK).toList(); + if (!initiatorSupports) { + jobMultipleStatuses = jobMultipleStatuses.stream().filter(s -> s.operation() != SnapshotOperation.CHECK).toList(); - assert manyStatuses.isEmpty() || manyStatuses.size() == 1 : + assert jobMultipleStatuses.isEmpty() || jobMultipleStatuses.size() == 1 : "Snapshot creation or restoration are mutually exclusive, only one of them allowed at the same time."; } - allStatuses.addAll(manyStatuses); + allStatuses.addAll(jobMultipleStatuses); } } @@ -146,68 +147,70 @@ public class SnapshotStatusTask extends VisorMultiNodeTask tastResultStatuses = null; + ArrayList taskResult = null; // First, find create or restore operation. - Collection filtered = F.viewReadOnly(allStatuses, s -> s, s -> s.operation() != SnapshotOperation.CHECK); + Collection filtered = F.viewReadOnly(allStatuses, s -> s, s -> s.operation() != SnapshotOperation.CHECK); // Found create or restore statuses. if (!filtered.isEmpty()) { // Merged create or restore. - SnapshotStatus merged = F.first(filtered); + SnapshotOperationStatus merged = F.first(filtered); - String mergedRqId = merged.requestId; + String mergedRqId = merged.reqId; // Filter out differing requests due to concurrent updates on nodes. - filtered = F.view(filtered, s -> s.requestId.equals(mergedRqId)); + filtered = F.view(filtered, s -> s.reqId.equals(mergedRqId)); // Merge nodes progress. Map> progress = new HashMap<>(); filtered.forEach(s -> progress.putAll(s.progress)); - merged = new SnapshotStatus(merged.op, merged.name, merged.incIdx, merged.requestId, merged.startTime, progress); + merged = new SnapshotOperationStatus(merged.op, merged.name, merged.incIdx, merged.reqId, merged.startTime, progress); - // Return the old style single result. - if (!initiatorSupport || !featureEnabled) - return merged; + // If the initiator like thin client of 'control.sh' doesn't support the feature yet, we have to rewrap + // the first single result back to the old format. + if (!initiatorSupports) + return new SnapshotStatus(merged.op, merged.name, merged.incIdx, merged.reqId, merged.startTime, progress); - if (tastResultStatuses == null) - tastResultStatuses = new ArrayList<>(); + if (taskResult == null) + taskResult = new ArrayList<>(); - tastResultStatuses.add(merged); + taskResult.add(merged); } - // Check operation statuses. + // Check operation statuses go last. filtered = F.viewReadOnly(allStatuses, s -> s, s -> s.operation() == SnapshotOperation.CHECK); // Found check statuses. if (!filtered.isEmpty()) { - // Merged check statuses per snapshot name. - Map mergedCheckStatuses = U.newHashMap(filtered.size()); + // Merged check statuses per snapshot. + Map mergedCheckStatuses = U.newHashMap(filtered.size()); filtered.forEach(chkStatus -> { - SnapshotStatus chkStatus0 = mergedCheckStatuses.computeIfAbsent(chkStatus.name, (snpName -> chkStatus)); + SnapshotOperationStatus chkStatus0 = mergedCheckStatuses.computeIfAbsent(chkStatus.name, (snpName -> chkStatus)); if (chkStatus0 != chkStatus) chkStatus0.progress.putAll(chkStatus.progress); }); - if (tastResultStatuses == null) - tastResultStatuses = new ArrayList<>(); + if (taskResult == null) + taskResult = new ArrayList<>(); - tastResultStatuses.addAll(mergedCheckStatuses.values()); + taskResult.addAll(mergedCheckStatuses.values()); } - return tastResultStatuses; + // Empty list notifies that this is the new result but is empty. Old empty is always null. + return taskResult == null ? (Serializable)Collections.emptyList() : taskResult; } /** * A job to get the statuses of the current snapshot operations on current node. *

Before {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}, returns single {@link SnapshotStatus} for single * snapshot creation or snapshot restoration operation. - *

With {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}, returns collection of {@link SnapshotStatus} for - * possible several snapshot checks operations or parallel creation/restoration and checks operations. Might also + *

With {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}, returns collection of {@link SnapshotOperationStatus} + * for possible several snapshot checks operations or parallel creation/restoration and checks operations. May still * contain a single status. */ private static class SnapshotStatusJob extends SnapshotJob { @@ -294,14 +297,14 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { return createOrRestoreStatus; } - // Many results. - ArrayList severalStatuses = null; + // New nany results. + ArrayList snpOpStatuses = null; if (createOrRestoreStatus != null) { - severalStatuses = new ArrayList<>(); + snpOpStatuses = new ArrayList<>(); // Status of creation or restoration operation always goes firts. This is important for old command initiators (control.sh). - severalStatuses.add(createOrRestoreStatus); + snpOpStatuses.add(new SnapshotOperationStatus(createOrRestoreStatus)); } for (ReadOnlyMetricRegistry snpCheckMReg : ignite.context().metric()) { @@ -335,7 +338,7 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { ); } - SnapshotStatus status = new SnapshotStatus( + SnapshotOperationStatus snpOpStatus = new SnapshotOperationStatus( SnapshotOperation.CHECK, MetricUtils.fromFullName(snpCheckMReg.name()).get2(), incIdx, @@ -344,13 +347,13 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { F.asMap(ignite.localNode().id(), metrics) ); - if (severalStatuses == null) - severalStatuses = new ArrayList<>(); + if (snpOpStatuses == null) + snpOpStatuses = new ArrayList<>(); - severalStatuses.add(status); + snpOpStatuses.add(snpOpStatus); } - return severalStatuses; + return snpOpStatuses; } } @@ -425,6 +428,96 @@ public Map> progress() { } } + /** + * Snapshot operation status after {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}. + * Is now a {@linkIgniteDataTransferObject}. + */ + public static class SnapshotOperationStatus extends IgniteDataTransferObject { + /** */ + private static final long serialVersionUID = 0L; + + /** Operation type. */ + @Order(0) + SnapshotOperation op; + + /** Snapshot name. */ + @Order(1) + String name; + + /** Incremental snapshot index. */ + @Order(2) + int incIdx; + + /** Request ID. */ + @Order(3) + String reqId; + + /** Start time. */ + @Order(4) + long startTime; + + /** Progress of operation on nodes. */ + @Order(5) + Map> progress; + + /** Empty constructor for serialization purposes. */ + public SnapshotOperationStatus() { + // No-op. + } + + /** */ + private SnapshotOperationStatus( + SnapshotOperation op, + String name, + int incIdx, + String reqId, + long startTime, + Map> progress + ) { + this.op = op; + this.name = name; + this.incIdx = incIdx; + this.reqId = reqId; + this.startTime = startTime; + this.progress = progress; + } + + /** */ + public SnapshotOperationStatus(SnapshotStatus s) { + this(s.op, s.name, s.incIdx, s.requestId, s.startTime, s.progress); + } + + /** @return Operation type. */ + public SnapshotOperation operation() { + return op; + } + + /** @return Snapshot name. */ + public String name() { + return name; + } + + /** @return Incremental snapshot index. */ + public int incrementIndex() { + return incIdx; + } + + /** @return Request ID. */ + public String requestId() { + return reqId; + } + + /** @return Start time. */ + public long startTime() { + return startTime; + } + + /** @return Progress of operation on nodes. */ + public Map> progress() { + return Collections.unmodifiableMap(progress); + } + } + /** Snapshot operation type. */ public enum SnapshotOperation { /** Create snapshot. */ diff --git a/modules/wire-compatibility-check/wire/enums/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation.json b/modules/wire-compatibility-check/wire/enums/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation.json new file mode 100644 index 0000000000000..e17d08090b831 --- /dev/null +++ b/modules/wire-compatibility-check/wire/enums/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation.json @@ -0,0 +1,8 @@ +{ + "class": "org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation", + "fields": [ + {"order": 0, "type": "org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation", "name": "CREATE"}, + {"order": 1, "type": "org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation", "name": "RESTORE"}, + {"order": 2, "type": "org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation", "name": "CHECK"} + ] +} diff --git a/modules/wire-compatibility-check/wire/idto/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperationStatus.json b/modules/wire-compatibility-check/wire/idto/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperationStatus.json new file mode 100644 index 0000000000000..0d052889397b2 --- /dev/null +++ b/modules/wire-compatibility-check/wire/idto/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperationStatus.json @@ -0,0 +1,11 @@ +{ + "class": "org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperationStatus", + "fields": [ + {"order": 0, "type": "org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation", "name": "op"}, + {"order": 1, "type": "java.lang.String", "name": "name"}, + {"order": 2, "type": "int", "name": "incIdx"}, + {"order": 3, "type": "java.lang.String", "name": "reqId"}, + {"order": 4, "type": "long", "name": "startTime"}, + {"order": 5, "type": "java.util.Map>", "name": "progress"} + ] +} From d466ca3875a0ac34a4f63058058211937b093852 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Thu, 29 Oct 2026 13:37:10 +0300 Subject: [PATCH 32/44] mxbean test fix --- .../snapshot/SnapshotMXBeanImpl.java | 61 +++---------------- 1 file changed, 7 insertions(+), 54 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java index bedd6243724ec..5cf956155c8b5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java @@ -23,14 +23,11 @@ import java.util.stream.Collectors; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.processors.metric.GridMetricManager; -import org.apache.ignite.internal.processors.metric.impl.MetricUtils; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.metric.MetricRegistry; import org.apache.ignite.mxbean.SnapshotMXBean; import org.apache.ignite.spi.metric.IntMetric; -import org.apache.ignite.spi.metric.Metric; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.DFLT_CHECK_ON_RESTORE; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotRestoreProcess.SNAPSHOT_RESTORE_METRICS; @@ -119,17 +116,16 @@ public SnapshotMXBeanImpl(GridKernalContext ctx) { /** {@inheritDoc} */ @Override public String status() { - StringBuilder res = null; - SnapshotOperationRequest req = mgr.currentCreateRequest(); if (req != null) { - res = new StringBuilder("Create snapshot operation is in progress [name=" + req.snapshotName() + + return "Create snapshot operation is in progress [name=" + req.snapshotName() + ", incremental=" + req.incremental() + (req.incremental() ? (", incrementIndex=" + req.incrementIndex()) : "") + - ", id=" + req.requestId() + ']'); + ", id=" + req.requestId() + ']'; } - else if (mgr.isRestoring()) { + + if (mgr.isRestoring()) { MetricRegistry mreg = metricMgr.registry(SNAPSHOT_RESTORE_METRICS); String name = mreg.findMetric("snapshotName").getAsString(); @@ -138,53 +134,10 @@ else if (mgr.isRestoring()) { boolean incremental = incIdx > 0; - res = new StringBuilder("Restore snapshot operation is in progress [name=" + name + ", incremental=" + incremental + - (incremental ? ", incrementIndex=" + incIdx : "") + ", id=" + id + ']'); - } - - boolean one = true; - - for (var mreg : metricMgr) { - if (!mreg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) - continue; - - Metric rqIdMetric = mreg.findMetric("requestId"); - - // The requestId metric is registered last. - if (rqIdMetric == null) - continue; - - String name = MetricUtils.fromFullName(mreg.name()).get2(); - int incIdx = mreg.findMetric("incrementIndex").value(); - String id = rqIdMetric.getAsString(); - - if (one) { - if (res == null) - res = new StringBuilder(); - else - res.append(U.nl()).append(U.nl()); - } - - res.append("Check snapshot operations are in progress ["); - - StringBuilder sb0 = new StringBuilder(); - - if (!one) - sb0.append(", "); - - sb0.append("[name=").append(name).append(", incremental=").append(incIdx > 0) - .append(incIdx > 0 ? ", incrementIndex=" + incIdx : "").append(", id=").append(id).append(']'); - - res.append(sb0); - - one = false; + return "Restore snapshot operation is in progress [name=" + name + ", incremental=" + incremental + + (incremental ? ", incrementIndex=" + incIdx : "") + ", id=" + id + ']'; } - if (res == null) - return "No snapshot operation is in progress."; - - res.append(']'); - - return res.toString(); + return "There is no create or restore snapshot operation in progress."; } } From 3c73806bf412548addba79afcf87d4556427c97f Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Thu, 29 Oct 2026 14:37:20 +0300 Subject: [PATCH 33/44] fixing --- .../feature/CoreFeatureRegistry.java | 2 +- ...SnapshotCheckStatusRollingUpgradeTest.java | 12 ++++----- .../AbstractRollingUpgradeTest.java | 1 - .../TestIgniteReleaseFeatures_2_19_4.java | 27 ------------------- 4 files changed, 7 insertions(+), 35 deletions(-) delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_4.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java index e431b8f07f8a8..13f5a11785a2d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java @@ -112,6 +112,6 @@ public class CoreFeatureRegistry { /** Feature that introduces the snapshot list command. */ public static final IgniteFeature SNAPSHOT_LIST_FEATURE = new IgniteCoreFeature(4); - /** Future that introduces the snaphost check status. */ + /** Future that introduces status of the snaphost check operation. */ public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = new IgniteCoreFeature(5); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java index a5ee0b55f5cee..e5d50deeebeff 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java @@ -118,7 +118,7 @@ public class IgniteClusterSnapshotCheckStatusRollingUpgradeTest extends Abstract @Test public void testClientOfThePreviousVersion() throws Exception { for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) - startGrid(i, "2.19.4", false); + startGrid(i, "master", false); grid(0).cluster().state(ClusterState.ACTIVE); @@ -151,7 +151,7 @@ public void testParallelRollingUpgradeInProgress() throws Exception { for (int i = 0; i < ALL_GRIDS; i++) { assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); - upgradeNodeVersion(i, "2.18.0", "2.19.4"); + upgradeNodeVersion(i, "2.18.0", "master"); } spi(grid(testNodeIx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && @@ -195,7 +195,7 @@ public void testSnapshotListFeature() throws Exception { for (int i = 0; i < ALL_GRIDS; i++) { assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); - upgradeNodeVersion(i, "2.18.0", "2.19.4"); + upgradeNodeVersion(i, "2.18.0", "master"); } ensureSnapshotCheckStatus(false); @@ -235,7 +235,7 @@ private void createSnapshot(int gridIdx) { } /** */ - private void ensureSnapshotCheckStatus(boolean positive, @Nullable VisorTaskArgument clientArg) throws Exception { + private void ensureSnapshotCheckStatus(boolean positive, boolean useCommand, @Nullable VisorTaskArgument clientArg) throws Exception { int blockNodeIdx = 1; spi(grid(blockNodeIdx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && @@ -245,7 +245,7 @@ private void ensureSnapshotCheckStatus(boolean positive, @Nullable VisorTaskArgu assertTrue(waitForCondition(() -> spi(grid(blockNodeIdx)).hasBlockedMessages(), getTestTimeout())); - Collection snpStatuses; + Collection snpStatuses; if (clientArg != null) { snpStatuses = snapshotStatusCli(clientArg); @@ -299,7 +299,7 @@ private static T snapshotStatusTask(Ignite grid) throws Exception { } /** */ - private T snapshotStatusCli(VisorTaskArgument vtArg) throws Exception { + private T snapshotStatusCli(VisorTaskArgument vtArg, boolean useCommand) throws Exception { String taskName = SnapshotStatusTask.class.getName(); try (IgniteClient cli = Ignition.startClient(new ClientConfiguration().setAddresses(Config.SERVER))) { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java index 56972144e6ca4..ab612e766c57d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java @@ -89,7 +89,6 @@ * 2.19.1 0-1 * 2.19.2 0-2 * 2.19.3 0-2,6 - * 2.19.4 0-1 * 2.20.0 2-5 * 2.20.1 2-6 * 2.21.0 6 diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_4.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_4.java deleted file mode 100644 index 1448c0adb263b..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_4.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.rollingupgrade.feature; - -/** */ -public class TestIgniteReleaseFeatures_2_19_4 { - /** */ - public static final IgniteFeature ROLLING_UPGRADE_FEATURE = TestIgniteReleaseFeatures_2_18_0.ROLLING_UPGRADE_FEATURE; - - /** */ - public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = CoreFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE; -} From 60e2efe6a349671325fc682611357541ae539159 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Thu, 29 Oct 2026 17:10:50 +0300 Subject: [PATCH 34/44] fixing --- docs/_docs/snapshots/snapshots.adoc | 7 ++++--- .../management/snapshot/SnapshotStatusCommand.java | 6 ------ .../main/java/org/apache/ignite/mxbean/SnapshotMXBean.java | 7 +++---- ...IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java | 2 +- .../apache/ignite/testsuites/IgniteSnapshotTestSuite9.java | 2 ++ 5 files changed, 10 insertions(+), 14 deletions(-) diff --git a/docs/_docs/snapshots/snapshots.adoc b/docs/_docs/snapshots/snapshots.adoc index effba8fa05236..85aac419a5e8e 100644 --- a/docs/_docs/snapshots/snapshots.adoc +++ b/docs/_docs/snapshots/snapshots.adoc @@ -371,7 +371,7 @@ If the metadata isn't found or cannot be read, snapshot isn't deleted. == Getting Snapshot Operation Status -The status of the create/restore/check snapshot operations in the cluster can be obtained using the `control.sh|bat` script. │ +The status of the create/restore/check snapshot operations in the cluster can be obtained using the `control.sh|bat` script. [tabs] -- @@ -390,11 +390,12 @@ control.bat --snapshot status ---- -- - [NOTE] │ -==== │ +[NOTE] +==== Parallel and multiple snapshot check operations can be running concurrently with any other snapshot operation. But only one check for one snapshot. Create and restore operations cannot run in parallel - only one of each type can be active at a time. The status is printed for each running snapshot operation (if multiple) separately, one after another. +==== == Cancelling Snapshot Operation diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index c460655e9224a..2505dfdb6597c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -67,17 +67,11 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand : - "Unsupported job result: " + rawResult.getClass().getSimpleName(); - SnapshotStatus singleOldStatus = rawResult instanceof SnapshotStatus ? (SnapshotStatus)rawResult : null; Collection statuses = rawResult instanceof Collection ? (Collection)rawResult : null; - assert statuses == null || statuses.isEmpty() || F.first(statuses) instanceof SnapshotStatusTask.SnapshotOperationStatus : - "Statuses collection has an unsupported result type"; - // Empty old-style result is null. Enpty new-style result is an empty collection. if (statuses != null && statuses.isEmpty()) { printer.accept("There is no any snapshot operation in progress."); diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java index dcd3192d3d353..dee7c59a42e7f 100644 --- a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java +++ b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java @@ -129,11 +129,10 @@ public void restoreSnapshot( public void cancelSnapshotRestore(@MXBeanParameter(name = "snpName", description = "Snapshot name.") String name); /** - * Get the status of the current snapshot check or restore operation in the cluster. Doesn't provide status of - * snapshots check operations. + * Get the status of the current snapshot operation in the cluster. * - * @return The status of a current snapshot create or restore operation in the cluster. + * @return The status of a current snapshot operation in the cluster. */ - @MXBeanDescription("The status of a current snapshot create or restore operation in the cluster.") + @MXBeanDescription("The status of a current snapshot operation in the cluster.") public String status(); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java index ec1bdb5507099..61346952b0065 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java @@ -181,7 +181,7 @@ public void testParallelRollingUpgradeInProgress() throws Exception { /** */ @Test - public void testSnapshotListFeature() throws Exception { + public void testSnapshotStatusFeature() throws Exception { for (int i = 0; i < ALL_GRIDS; i++) startGrid(i, "2.18.0", i >= ALL_GRIDS - CLIENTS); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java index 86802121999c7..6b2ccfa4ed443 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotCheckStatusRollingUpgradeTest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteRollingUpgradeTest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteTest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotListRollingUpgradeTest; @@ -47,6 +48,7 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotDeleteTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotListTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotListRollingUpgradeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotCheckStatusRollingUpgradeTest.class, ignoredTests); return suite; } From 1cadd2a22a4b82376814b15e4e929b97801b17f7 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Thu, 29 Oct 2026 17:10:50 +0300 Subject: [PATCH 35/44] fixing --- docs/_docs/snapshots/snapshots.adoc | 14 ++++++++------ .../ignite/util/GridCommandHandlerTest.java | 2 +- .../snapshot/SnapshotStatusCommand.java | 15 ++++----------- .../management/snapshot/SnapshotStatusTask.java | 12 +++--------- .../feature/CoreFeatureRegistry.java | 2 +- .../org/apache/ignite/mxbean/SnapshotMXBean.java | 7 +++---- ...sterSnapshotCheckStatusRollingUpgradeTest.java | 2 +- .../testsuites/IgniteSnapshotTestSuite9.java | 2 ++ 8 files changed, 23 insertions(+), 33 deletions(-) diff --git a/docs/_docs/snapshots/snapshots.adoc b/docs/_docs/snapshots/snapshots.adoc index effba8fa05236..0f9378e4cfd61 100644 --- a/docs/_docs/snapshots/snapshots.adoc +++ b/docs/_docs/snapshots/snapshots.adoc @@ -371,7 +371,7 @@ If the metadata isn't found or cannot be read, snapshot isn't deleted. == Getting Snapshot Operation Status -The status of the create/restore/check snapshot operations in the cluster can be obtained using the `control.sh|bat` script. │ +The status of the create/restore/check snapshot operations in the cluster can be obtained using the `control.sh|bat` script. [tabs] -- @@ -390,11 +390,13 @@ control.bat --snapshot status ---- -- - [NOTE] │ -==== │ -Parallel and multiple snapshot check operations can be running concurrently with any other snapshot operation. -But only one check for one snapshot. Create and restore operations cannot run in parallel - only one of each type can -be active at a time. The status is printed for each running snapshot operation (if multiple) separately, one after another. +[NOTE] +==== +Multiple snapshot check operations can run concurrently with any other snapshot operation of another snapshot. +Only one check can be running for the same snapshot at a time. Create and restore operations cannot run in parallel — +only one of them can be active at a time. The status is printed for each running snapshot operation (if multiple) separately, +one after another. +==== == Cancelling Snapshot Operation diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 19025204efe92..abecc331cef64 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3800,7 +3800,7 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment assertEquals("Unexpected exit code", EXIT_CODE_OK, execute("--snapshot", "status")); - var out = testOut.toString(); + String out = testOut.toString(); if (log.isInfoEnabled()) log.info("Test out:" + U.nl() + out); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index c460655e9224a..5165baca0712b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -67,20 +67,14 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand : - "Unsupported job result: " + rawResult.getClass().getSimpleName(); - SnapshotStatus singleOldStatus = rawResult instanceof SnapshotStatus ? (SnapshotStatus)rawResult : null; Collection statuses = rawResult instanceof Collection ? (Collection)rawResult : null; - assert statuses == null || statuses.isEmpty() || F.first(statuses) instanceof SnapshotStatusTask.SnapshotOperationStatus : - "Statuses collection has an unsupported result type"; - - // Empty old-style result is null. Enpty new-style result is an empty collection. + // Empty old-style result is null. No-statuses result of the new-style result is always an empty collection. if (statuses != null && statuses.isEmpty()) { - printer.accept("There is no any snapshot operation in progress."); + printer.accept("There is no snapshot operation in progress."); return; } @@ -90,11 +84,10 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand(); - taskResult.add(merged); } @@ -196,9 +193,6 @@ public class SnapshotStatusTask extends VisorMultiNodeTask(); - taskResult.addAll(mergedCheckStatuses.values()); } @@ -297,13 +291,13 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { return createOrRestoreStatus; } - // New nany results. + // New many results. ArrayList snpOpStatuses = null; if (createOrRestoreStatus != null) { snpOpStatuses = new ArrayList<>(); - // Status of creation or restoration operation always goes firts. This is important for old command initiators (control.sh). + // Status of creation or restoration operation always goes first. This is important for old command initiators (control.sh). snpOpStatuses.add(new SnapshotOperationStatus(createOrRestoreStatus)); } @@ -430,7 +424,7 @@ public Map> progress() { /** * Snapshot operation status after {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}. - * Is now a {@linkIgniteDataTransferObject}. + * Is now a {@link IgniteDataTransferObject}. */ public static class SnapshotOperationStatus extends IgniteDataTransferObject { /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java index 13f5a11785a2d..b35210c2cf78f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java @@ -112,6 +112,6 @@ public class CoreFeatureRegistry { /** Feature that introduces the snapshot list command. */ public static final IgniteFeature SNAPSHOT_LIST_FEATURE = new IgniteCoreFeature(4); - /** Future that introduces status of the snaphost check operation. */ + /** Future that introduces statuses of the snapshot check operations. */ public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = new IgniteCoreFeature(5); } diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java index dcd3192d3d353..dee7c59a42e7f 100644 --- a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java +++ b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java @@ -129,11 +129,10 @@ public void restoreSnapshot( public void cancelSnapshotRestore(@MXBeanParameter(name = "snpName", description = "Snapshot name.") String name); /** - * Get the status of the current snapshot check or restore operation in the cluster. Doesn't provide status of - * snapshots check operations. + * Get the status of the current snapshot operation in the cluster. * - * @return The status of a current snapshot create or restore operation in the cluster. + * @return The status of a current snapshot operation in the cluster. */ - @MXBeanDescription("The status of a current snapshot create or restore operation in the cluster.") + @MXBeanDescription("The status of a current snapshot operation in the cluster.") public String status(); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java index ec1bdb5507099..61346952b0065 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java @@ -181,7 +181,7 @@ public void testParallelRollingUpgradeInProgress() throws Exception { /** */ @Test - public void testSnapshotListFeature() throws Exception { + public void testSnapshotStatusFeature() throws Exception { for (int i = 0; i < ALL_GRIDS; i++) startGrid(i, "2.18.0", i >= ALL_GRIDS - CLIENTS); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java index 86802121999c7..6b2ccfa4ed443 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotCheckStatusRollingUpgradeTest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteRollingUpgradeTest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteTest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotListRollingUpgradeTest; @@ -47,6 +48,7 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotDeleteTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotListTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotListRollingUpgradeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotCheckStatusRollingUpgradeTest.class, ignoredTests); return suite; } From 1cbfe217071b122f14cc85961a7c7d9a2cbaa7ef Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Thu, 29 Oct 2026 18:21:18 +0300 Subject: [PATCH 36/44] LLM review fixes --- .../ignite/util/GridCommandHandlerTest.java | 12 +- .../snapshot/SnapshotStatusCommand.java | 16 ++- .../snapshot/SnapshotStatusTask.java | 21 +-- .../snapshot/SnapshotCheckProcess.java | 4 +- .../snapshot/SnapshotMXBeanImpl.java | 6 +- .../feature/CoreFeatureRegistry.java | 2 +- ...SnapshotCheckStatusRollingUpgradeTest.java | 127 ++++++++++++++---- .../snapshot/IgniteSnapshotMXBeanTest.java | 2 +- 8 files changed, 134 insertions(+), 56 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index abecc331cef64..0902154f7d252 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -85,6 +85,7 @@ import org.apache.ignite.internal.management.cache.FindAndDeleteGarbageInPersistenceTaskResult; import org.apache.ignite.internal.management.cache.IdleVerifyDumpTask; import org.apache.ignite.internal.management.cache.VerifyBackupPartitionsTask; +import org.apache.ignite.internal.management.snapshot.SnapshotStatusCommand; import org.apache.ignite.internal.management.tx.TxInfo; import org.apache.ignite.internal.management.tx.TxTaskResult; import org.apache.ignite.internal.managers.communication.GridIoMessage; @@ -245,6 +246,13 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb cleanDiagnosticDir(); } + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + listeningLog = null; + } + /** {@inheritDoc} */ @Override protected void cleanPersistenceDir() throws Exception { super.cleanPersistenceDir(); @@ -3649,7 +3657,7 @@ public void testSnapshotRestoreCancelAndStatus() throws Exception { assertTrue(ctxDisposed); assertEquals(EXIT_CODE_OK, execute("--snapshot", "status")); - assertContains(log, testOut.toString(), "There is no create or restore snapshot operation in progress."); + assertContains(log, testOut.toString(), SnapshotStatusCommand.NO_SNP_OP_IN_PROGRESS_MSG); assertNull(ig.cache(DEFAULT_CACHE_NAME)); } @@ -4052,7 +4060,7 @@ private void checkSnapshotStatus(boolean isCreating, boolean isRestoring, boolea assertEquals(out, EXIT_CODE_OK, status); if (!isCreating && !isRestoring) { - assertContains(log, out, "There is no create or restore snapshot operation in progress."); + assertContains(log, out, SnapshotStatusCommand.NO_SNP_OP_IN_PROGRESS_MSG); return; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 5165baca0712b..bc2a2e3b916f7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -44,6 +44,9 @@ /** */ public class SnapshotStatusCommand extends AbstractSnapshotCommand { + /** */ + public static final String NO_SNP_OP_IN_PROGRESS_MSG = "There is no snapshot operation in progress."; + /** {@inheritDoc} */ @Override public String description() { return "Get the status of the current snapshot operation"; @@ -61,6 +64,7 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand printer) { + // An old-style empty result is null. if (rawResult == null) { printer.accept("There is no create or restore snapshot operation in progress."); @@ -72,22 +76,20 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand)rawResult : null; - // Empty old-style result is null. No-statuses result of the new-style result is always an empty collection. + // A new-style result with no statuses is an empty collection, never null. if (statuses != null && statuses.isEmpty()) { - printer.accept("There is no snapshot operation in progress."); + printer.accept(NO_SNP_OP_IN_PROGRESS_MSG); return; } statuses = statuses == null ? Collections.singleton(new SnapshotStatusTask.SnapshotOperationStatus(singleOldStatus)) : statuses; - boolean first = true; + boolean separator = false; for (SnapshotStatusTask.SnapshotOperationStatus s0 : statuses) { - if (!first) + if (separator) printer.accept(U.nl()); - else - first = false; if (s0.operation() == SnapshotStatusTask.SnapshotOperation.CREATE) printer.accept("Create snapshot operation is in progress."); @@ -130,6 +132,8 @@ else if (s0.operation() == SnapshotStatusTask.SnapshotOperation.RESTORE) .collect(Collectors.toList()); SystemViewCommand.printTable(desc.titles(), desc.types(), rows, printer); + + separator = true; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 3e11a463a2ec7..bc631d6159b33 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -40,7 +40,6 @@ import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T5; -import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorJob; @@ -113,31 +112,17 @@ public class SnapshotStatusTask extends VisorMultiNodeTask : - "Unsupported job result type: " + rawResult.getClass().getSimpleName(); - assert !(rawResult instanceof Collection cll) || cll.isEmpty() || (F.first(cll) instanceof SnapshotOperationStatus) : - "Statuses collection is empty or its first status is of unsupported type"; - - if (rawResult instanceof SnapshotStatus singleJobStatus) { - assert !featureEnabled : - "Single snapshot status result is allowed only if the snapshot check status feature is not enabled"; - assert singleJobStatus.operation() != SnapshotOperation.CHECK : - "Status of the check operation is allowed only within the new result list, not as a single result"; - + if (rawResult instanceof SnapshotStatus singleJobStatus) allStatuses.add(new SnapshotOperationStatus(singleJobStatus)); - } else { - assert featureEnabled : - "Many snapshot status results are allowed only if the snapshot check status feature is enabled"; - Collection jobMultipleStatuses = (Collection)rawResult; - // Skip the check status if cluster supports, but not a thin client of previous version. + // Skip the check status for an old command initiator that doesn't know about it. if (!initiatorSupports) { jobMultipleStatuses = jobMultipleStatuses.stream().filter(s -> s.operation() != SnapshotOperation.CHECK).toList(); assert jobMultipleStatuses.isEmpty() || jobMultipleStatuses.size() == 1 : - "Snapshot creation or restoration are mutually exclusive, only one of them allowed at the same time."; + "Snapshot creation and restoration are mutually exclusive; only one of them can run at a time."; } allStatuses.addAll(jobMultipleStatuses); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index 447c8da639645..ebe9ac1020a4d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -760,12 +760,12 @@ private void registerMetrics(SnapshotCheckContext ctx) { "Number of checked snapshot parts (nodes data) on current node."); } - // Presence of this metrics also marks that the registry is complete. + // Presence of this metric marks that the registry is complete. mreg.register( "requestId", () -> ctx.req.requestId().toString(), String.class, - "The request ID of the last running cluster snapshot check operation on this node." + "The request ID of the current cluster snapshot check operation on this node." ); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java index 5cf956155c8b5..fac5b21c9c39a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java @@ -36,6 +36,10 @@ * Snapshot MBean features. */ public class SnapshotMXBeanImpl implements SnapshotMXBean { + /** */ + public static final String NO_CREATE_OR_RESTORE_IN_PROGRESS_MSG = + "There is no create or restore snapshot operation in progress."; + /** Instance of snapshot cache shared manager. */ private final IgniteSnapshotManager mgr; @@ -138,6 +142,6 @@ public SnapshotMXBeanImpl(GridKernalContext ctx) { (incremental ? ", incrementIndex=" + incIdx : "") + ", id=" + id + ']'; } - return "There is no create or restore snapshot operation in progress."; + return NO_CREATE_OR_RESTORE_IN_PROGRESS_MSG; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java index b35210c2cf78f..e4e186fffad72 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java @@ -112,6 +112,6 @@ public class CoreFeatureRegistry { /** Feature that introduces the snapshot list command. */ public static final IgniteFeature SNAPSHOT_LIST_FEATURE = new IgniteCoreFeature(4); - /** Future that introduces statuses of the snapshot check operations. */ + /** Feature that introduces statuses of the snapshot check operations. */ public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = new IgniteCoreFeature(5); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java index 61346952b0065..543cb07846ae0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java @@ -43,6 +43,7 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.management.api.NoArg; import org.apache.ignite.internal.management.snapshot.SnapshotStatusTask; +import org.apache.ignite.internal.management.snapshot.SnapshotStatusTask.SnapshotOperation; import org.apache.ignite.internal.processors.rollingupgrade.AbstractRollingUpgradeTest; import org.apache.ignite.internal.util.distributed.SingleNodeMessage; import org.apache.ignite.internal.util.typedef.F; @@ -50,6 +51,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorTaskArgument; import org.apache.ignite.internal.visor.VisorTaskResult; +import org.apache.ignite.lang.IgniteFuture; import org.jetbrains.annotations.Nullable; import org.junit.Test; @@ -57,6 +59,7 @@ import static org.apache.ignite.internal.TestRecordingCommunicationSpi.spi; import static org.apache.ignite.internal.processors.rollingupgrade.feature.ManagementApiVersionValidationTest.withCoreVersion; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_METAS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.END_SNAPSHOT; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RU_PREPARE_VERSION_FINALIZATION; import static org.apache.ignite.testframework.GridTestUtils.runAsync; import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; @@ -118,8 +121,68 @@ public class IgniteClusterSnapshotCheckStatusRollingUpgradeTest extends Abstract /** */ @Test public void testClientOfThePreviousVersion() throws Exception { + doTestClientOfVersion(MASTER_VER_REF, "2.18.0"); + } + + /** */ + @Test + public void testClientOfTheNextVersion() throws Exception { + doTestClientOfVersion("2.18.0", MASTER_VER_REF); + } + + /** */ + @Test + public void testClientOfTheNextVersionWithCreateInProgress() throws Exception { for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) - startGrid(i, "master", false); + startGrid(i, "2.18.0", false); + + grid(0).cluster().state(ClusterState.ACTIVE); + + createCache(0); + + // Blocks the finalization of the create operation to keep it in progress while the status is requested. + for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) { + spi(grid(i)).blockMessages((node, msg) -> + msg instanceof SingleNodeMessage snm && snm.type() == END_SNAPSHOT.ordinal()); + } + + IgniteFuture createFut = snp(0).createSnapshot(SNP_NAME); + + try { + assertTrue("Snapshot create operation has not started.", waitForCondition(() -> { + for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) { + if (snp(i).currentCreateRequest() == null) + return false; + } + + return true; + }, 10_000L)); + + Collection nodes = Collections.singleton(nodeId(0)); + + Collection statuses = + snapshotStatusTaskWithClient(new VisorTaskArgument<>(nodes, new NoArg(), false)); + + assertEquals(1, statuses.size()); + + SnapshotStatusTask.SnapshotOperationStatus status = F.first(statuses); + + assertEquals(SnapshotOperation.CREATE, status.operation()); + assertEquals(SNP_NAME, status.name()); + assertEquals(ALL_GRIDS - CLIENTS, status.progress().size()); + } + finally { + for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) + spi(grid(i)).stopBlock(); + } + + createFut.get(10_000L); + } + + /** */ + private void doTestClientOfVersion(String clusterVer, String clientVer) throws Exception { + for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) + startGrid(i, clusterVer, false); grid(0).cluster().state(ClusterState.ACTIVE); @@ -127,10 +190,9 @@ public void testClientOfThePreviousVersion() throws Exception { Collection nodes = Collections.singleton(nodeId(0)); - // Client not supporting the snapshot check status. - VisorTaskArgument vtArg = withCoreVersion("2.18.0", () -> new VisorTaskArgument<>(nodes, new NoArg(), false)); + VisorTaskArgument vtArg = withCoreVersion(clientVer, () -> new VisorTaskArgument<>(nodes, new NoArg(), false)); - ensureSnapshotCheckStatus(false, vtArg); + validateSnapshotCheckStatusFeature(false, vtArg); } /** */ @@ -145,14 +207,14 @@ public void testParallelRollingUpgradeInProgress() throws Exception { createCacheAndSnapshot(testNodeIx); - ensureSnapshotCheckStatus(false); + validateSnapshotCheckStatusFeature(false); ru(grid(testNodeIx)).enableVersionUpgrade(); for (int i = 0; i < ALL_GRIDS; i++) { assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); - upgradeNodeVersion(i, "2.18.0", "master"); + upgradeNodeVersion(i, "2.18.0", MASTER_VER_REF); } spi(grid(testNodeIx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && @@ -162,7 +224,7 @@ public void testParallelRollingUpgradeInProgress() throws Exception { assertTrue(spi(grid(testNodeIx)).waitForBlocked(1, getTestTimeout())); - ensureSnapshotCheckStatus(false); + validateSnapshotCheckStatusFeature(false); spi(grid(testNodeIx)).stopBlock(); @@ -176,7 +238,7 @@ public void testParallelRollingUpgradeInProgress() throws Exception { assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout())); } - ensureSnapshotCheckStatus(true); + validateSnapshotCheckStatusFeature(true); } /** */ @@ -189,17 +251,17 @@ public void testSnapshotStatusFeature() throws Exception { createCacheAndSnapshot(1); - ensureSnapshotCheckStatus(false); + validateSnapshotCheckStatusFeature(false); ru(grid(0)).enableVersionUpgrade(); for (int i = 0; i < ALL_GRIDS; i++) { assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); - upgradeNodeVersion(i, "2.18.0", "master"); + upgradeNodeVersion(i, "2.18.0", MASTER_VER_REF); } - ensureSnapshotCheckStatus(false); + validateSnapshotCheckStatusFeature(false); ru(grid(1)).finalizeClusterVersion(); @@ -209,11 +271,18 @@ public void testSnapshotStatusFeature() throws Exception { assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout())); } - ensureSnapshotCheckStatus(true); + validateSnapshotCheckStatusFeature(true); } /** */ private void createCacheAndSnapshot(int gridIdx) { + createCache(gridIdx); + + createSnapshot(gridIdx); + } + + /** Creates a replicated cache and fills it with data. */ + private void createCache(int gridIdx) { int partsCnt = 4; int keysCnt = partsCnt * 5; @@ -226,8 +295,6 @@ private void createCacheAndSnapshot(int gridIdx) { for (int i = 0; i < keysCnt; i++) ds.addData(i, i); } - - createSnapshot(gridIdx); } /** */ @@ -236,7 +303,7 @@ private void createSnapshot(int gridIdx) { } /** */ - private void ensureSnapshotCheckStatus(boolean supports, @Nullable VisorTaskArgument clientArg) throws Exception { + private void validateSnapshotCheckStatusFeature(boolean supports, @Nullable VisorTaskArgument clientArg) throws Exception { int blockNodeIdx = 1; spi(grid(blockNodeIdx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && @@ -246,8 +313,8 @@ private void ensureSnapshotCheckStatus(boolean supports, @Nullable VisorTaskArgu assertTrue(waitForCondition(() -> spi(grid(blockNodeIdx)).hasBlockedMessages(), getTestTimeout())); - for (Ignite ig : G.allGrids()) { - Serializable rawRes = snapshotStatusTask(ig, clientArg); + if (clientArg != null) { + Serializable rawRes = snapshotStatusTaskWithClient(clientArg); if (supports) { assertTrue(rawRes instanceof Collection cll && !cll.isEmpty() && @@ -256,6 +323,18 @@ private void ensureSnapshotCheckStatus(boolean supports, @Nullable VisorTaskArgu else assertTrue(rawRes == null); } + else { + for (Ignite ig : G.allGrids()) { + Serializable rawRes = snapshotStatusTaskWithGrid(ig); + + if (supports) { + assertTrue(rawRes instanceof Collection cll && !cll.isEmpty() && + F.first(cll) instanceof SnapshotStatusTask.SnapshotOperationStatus); + } + else + assertTrue(rawRes == null); + } + } spi(grid(blockNodeIdx)).stopBlock(); @@ -271,8 +350,8 @@ private void ensureSnapshotCheckStatus(boolean supports, @Nullable VisorTaskArgu } /** */ - private void ensureSnapshotCheckStatus(boolean supports) throws Exception { - ensureSnapshotCheckStatus(supports, null); + private void validateSnapshotCheckStatusFeature(boolean supports) throws Exception { + validateSnapshotCheckStatusFeature(supports, null); } /** */ @@ -281,18 +360,16 @@ private IgniteSnapshotManager snp(int gridIdx) { } /** */ - private static T snapshotStatusTask(Ignite grid, @Nullable VisorTaskArgument vtArg) throws Exception { - if (vtArg == null) { - Collection nodes = grid.cluster().nodes().stream().map(ClusterNode::id).toList(); + private static T snapshotStatusTaskWithGrid(Ignite grid) throws Exception { + Collection nodes = grid.cluster().nodes().stream().map(ClusterNode::id).toList(); - vtArg = new VisorTaskArgument<>(nodes, new NoArg(), false); - } + VisorTaskArgument vtArg = new VisorTaskArgument<>(nodes, new NoArg(), false); return (T)grid.compute().execute(SnapshotStatusTask.class, vtArg).result(); } /** */ - private T snapshotStatusCli(VisorTaskArgument vtArg) throws Exception { + private T snapshotStatusTaskWithClient(VisorTaskArgument vtArg) throws Exception { String taskName = SnapshotStatusTask.class.getName(); try (IgniteClient cli = Ignition.startClient(new ClientConfiguration().setAddresses(Config.SERVER))) { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java index cdb30ae79b828..a6538f646afcf 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java @@ -328,7 +328,7 @@ private void checkSnapshotStatus(boolean isCreating, boolean isRestoring, boolea String status = bean.status(); if (!isCreating && !isRestoring) { - assertContains(log, status, "There is no create or restore snapshot operation in progress"); + assertContains(log, status, SnapshotMXBeanImpl.NO_CREATE_OR_RESTORE_IN_PROGRESS_MSG); continue; } From 153ef53059c90e46e3f256ed7e10bc490544cf4b Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Thu, 29 Oct 2026 20:28:59 +0300 Subject: [PATCH 37/44] merge dev branch --- docs/_docs/clustering/multi-data-center.adoc | 1 + docs/_docs/data-rebalancing.adoc | 119 ++ docs/_docs/snapshots/snapshots.adoc | 14 +- docs/_docs/tools/control-script.adoc | 6 + .../ignite/ssl/DelegatingSSLContextSpi.java | 4 +- .../ignite/util/GridCommandHandlerTest.java | 22 +- .../apache/ignite/util/MetricCommandTest.java | 12 +- .../internal/direct/DirectMessageWriter.java | 68 +- .../cache/VerifyBackupPartitionsTask.java | 200 +- .../MetricConfigureHistogramCommandArg.java | 4 - .../snapshot/SnapshotStatusCommand.java | 32 +- .../snapshot/SnapshotStatusTask.java | 204 +- .../communication/CompressedMessage.java | 132 +- .../cache/IgniteCacheOffheapManager.java | 15 + .../cache/IgniteCacheOffheapManagerImpl.java | 10 + .../persistence/pagemem/PageMemoryImpl.java | 30 +- .../snapshot/SnapshotCheckProcess.java | 4 +- .../snapshot/SnapshotMXBeanImpl.java | 65 +- .../SnapshotPartitionsVerifyHandler.java | 8 +- .../cache/verify/IdleVerifyUtility.java | 23 +- .../metric/ConfigurableHistogramMetric.java | 3 + .../processors/metric/GridMetricManager.java | 5 +- .../processors/metric/MetricRegistryImpl.java | 4 +- .../metric/impl/HistogramMetricImpl.java | 9 + .../impl/PeriodicHistogramMetricImpl.java | 11 +- .../offheap/GridOffHeapProcessor.java | 417 ---- .../processors/pool/PoolProcessor.java | 7 +- .../feature/CoreFeatureRegistry.java | 2 +- .../offheap/GridOffHeapEvictListener.java | 38 - .../internal/util/offheap/GridOffHeapMap.java | 219 -- .../util/offheap/GridOffHeapMapFactory.java | 226 -- .../offheap/GridOffHeapPartitionedMap.java | 249 --- .../util/offheap/unsafe/GridUnsafeLru.java | 650 ------ .../util/offheap/unsafe/GridUnsafeMap.java | 1814 ----------------- .../unsafe/GridUnsafePartitionedMap.java | 428 ---- .../apache/ignite/mxbean/SnapshotMXBean.java | 7 +- .../resources/META-INF/classnames.properties | 8 - .../direct/DirectMarshallingMessagesTest.java | 32 + .../communication/CompressedMessageTest.java | 103 +- .../metric/MetricsConfigurationTest.java | 18 + .../rebalancing/MultiDcRebalancingTest.java | 587 +++++- .../MultiDcSplitBrainRebalanceTest.java | 446 ++++ ...deOrCacheStoppingDuringIdleVerifyTest.java | 274 +++ ...SnapshotCheckStatusRollingUpgradeTest.java | 130 +- .../snapshot/IgniteSnapshotMXBeanTest.java | 2 +- .../AbstractRollingUpgradeTest.java | 1 - .../TestIgniteReleaseFeatures_2_19_4.java | 27 - .../util/offheap/GridByteArrayWrapper.java | 63 - .../GridOffHeapMapAbstractSelfTest.java | 843 -------- ...GridOffHeapMapPerformanceAbstractTest.java | 295 --- ...OffHeapPartitionedMapAbstractSelfTest.java | 1178 ----------- ...PartitionedMapPerformanceAbstractTest.java | 436 ---- .../unsafe/GridUnsafeMapPerformanceTest.java | 32 - .../offheap/unsafe/GridUnsafeMapSelfTest.java | 32 - ...idUnsafePartitionedMapPerformanceTest.java | 34 - .../GridUnsafePartitionedMapSelfTest.java | 32 - .../unsafe/GridUnsafeStripedLruSefTest.java | 237 --- .../ignite/ssl/SSLContextWrapperTest.java | 57 + .../junits/common/GridCommonAbstractTest.java | 30 +- .../testsuites/IgniteCacheTestSuite19.java | 2 + .../testsuites/IgnitePdsTestSuite3.java | 2 + .../IgnitePerformanceTestSuite.java | 4 - .../testsuites/IgniteSnapshotTestSuite9.java | 2 + .../testsuites/IgniteUtilSelfTestSuite.java | 6 - .../ignite/testsuites/SecurityTestSuite.java | 2 + modules/ducktests/README.md | 3 + .../disk_full/DiskFullLoadApplication.java | 88 + .../disk_full/StdoutFloodApplication.java | 51 + .../tests/disk_full/StdoutFloodService.java} | 14 +- .../disk_full/StdoutFloodServiceImpl.java | 62 + .../tests/checks/utils/check_ignite_spec.py | 20 +- .../tests/checks/utils/check_parametrized.py | 30 +- .../ignitetest/services/mdc/mdc_cluster.py | 18 +- .../ignitetest/services/utils/ignite_aware.py | 31 + .../ignite_configuration/data_storage.py | 1 + .../ignitetest/services/utils/ignite_spec.py | 24 +- .../tests/ignitetest/tests/disk_full_test.py | 458 +++++ .../mdc/mdc_historical_rebalance_test.py | 142 ++ .../tests/ignitetest/utils/__init__.py | 4 +- .../ducktests/tests/ignitetest/utils/_mark.py | 34 + .../cache/index/ResumeCreateIndexTest.java | 29 +- .../util/offheap/GridOffHeapEvent.java | 47 - .../offheap/GridOffHeapEventListener.java | 26 - .../util/offheap/unsafe/GridUnsafeMemory.java | 153 -- ....SnapshotStatusTask$SnapshotOperation.json | 8 + ...hotStatusTask$SnapshotOperationStatus.json | 11 + 86 files changed, 3310 insertions(+), 7921 deletions(-) delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/offheap/GridOffHeapProcessor.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapEvictListener.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapMap.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapMapFactory.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapPartitionedMap.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLru.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMap.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafePartitionedMap.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/MultiDcSplitBrainRebalanceTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/NodeOrCacheStoppingDuringIdleVerifyTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_4.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridByteArrayWrapper.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapMapAbstractSelfTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapMapPerformanceAbstractTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapPartitionedMapAbstractSelfTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapPartitionedMapPerformanceAbstractTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMapPerformanceTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMapSelfTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafePartitionedMapPerformanceTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafePartitionedMapSelfTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeStripedLruSefTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/ssl/SSLContextWrapperTest.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java rename modules/{core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLruPoller.java => ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodService.java} (75%) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java create mode 100644 modules/ducktests/tests/ignitetest/tests/disk_full_test.py create mode 100644 modules/ducktests/tests/ignitetest/tests/mdc/mdc_historical_rebalance_test.py delete mode 100644 modules/unsafe/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapEvent.java delete mode 100644 modules/unsafe/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapEventListener.java create mode 100644 modules/wire-compatibility-check/wire/enums/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation.json create mode 100644 modules/wire-compatibility-check/wire/idto/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperationStatus.json diff --git a/docs/_docs/clustering/multi-data-center.adoc b/docs/_docs/clustering/multi-data-center.adoc index a5a37e25b1ff3..1eb10bd8f51a1 100644 --- a/docs/_docs/clustering/multi-data-center.adoc +++ b/docs/_docs/clustering/multi-data-center.adoc @@ -246,6 +246,7 @@ During rebalancing, Ignite prefers nodes from the same data center when such sup If the required partition data cannot be obtained from the same data center, Ignite can rebalance from a remote data center. This behavior applies to both full and historical rebalance. +See link:data-rebalancing#rebalance-strategy[Rebalance Strategy] for the conditions that determine which strategy is used. == Monitoring diff --git a/docs/_docs/data-rebalancing.adoc b/docs/_docs/data-rebalancing.adoc index 2a603f8f224a6..49a43ca76e77d 100644 --- a/docs/_docs/data-rebalancing.adoc +++ b/docs/_docs/data-rebalancing.adoc @@ -28,6 +28,83 @@ In pure in-memory clusters, the default behavior is to start rebalancing immedia In clusters with persistence, the baseline topology has to be changed manually (default behavior), or can be changed automatically when link:clustering/baseline-topology#baseline-topology-autoadjustment[automatic baseline adjustment] is enabled. ==== +== Rebalancing Strategy [[rebalance-strategy]] + +When a node joins or rejoins the cluster, each partition is restored using one of two strategies: + +- *Full rebalance* — the node downloads the entire partition content from another server node that holds a copy of the partition. +- *Historical rebalance* — the node restores its own data from the local disk and downloads only the delta (WAL entries written while it was absent) from the source node that holds a copy of the partition. + +The strategy is selected per partition based on the following rules: + +. A full rebalance is used if the node has no local data for the partition (update counter is zero). ++ +This applies both to a *new node joining for the first time* (it has no data on disk at all) and to a node that was *evicted from the baseline* and rejoined (its local partition data was invalidated). In both cases the full partition content must be downloaded from another node. + +. A full rebalance is used if the delta exceeds the total number of entries in the partition — transferring the full content is more efficient in that case. ++ +Example: a partition contains 100 entries. While the node was absent, 500 writes hit this partition. Downloading 500 delta entries is more expensive than just downloading all 100 entries → full rebalance. + +. A full rebalance is used if the partition size is below the `historical.rebalance.threshold` (DMS property, default 500 entries), unless the `IGNITE_PREFER_WAL_REBALANCE` system property is set to `true`. ++ +Example: a partition contains 10 entries. While the node was absent, 3 writes hit this partition. The delta (3) is smaller than the partition (10), but 10 < 500 — the partition is too small to justify WAL replay overhead → full rebalance. With `IGNITE_PREFER_WAL_REBALANCE=true` this threshold check is bypassed, and a historical rebalance is used regardless of the partition size (provided the other rules above are not triggered). + +. A historical rebalance is used when all of the above conditions are not met: the node has local data, the delta is smaller than the partition, and the partition exceeds the threshold. ++ +Example: a partition contains 10,000 entries. While the node was absent, 30 writes hit this partition. Downloading 30 delta entries is far cheaper than all 10,000 → historical rebalance. + +[NOTE] +==== +If both nodes report the same update counter for a partition, no rebalance is performed for that partition. + +A node reports counters only for partitions it has written to. If the dataset is small relative to the partition count, most partitions will have a zero counter and will not be included in the rebalance. +==== + +[NOTE] +==== +*Historical rebalance requires the delta to be present in the WAL of the source node.* If the WAL segments that contain the delta were already archived or deleted by the time the node rejoins — for example, the node was absent for a long time, or WAL history and archiving are configured with a small retention — the source node cannot supply the delta. In that case Ignite does not lose data: the supplier detects the failure to iterate over the WAL, marks the last checkpoint as not applicable for WAL rebalance, marks the affected partitions as missed, and the node falls back to a *full rebalance*. +==== + +[CAUTION] +==== +The `rebalanceDelay` (deprecated) setting also prevents a historical rebalance: when a delay is configured, the rebalance is scheduled outside of the partition map exchange context, and the historical path (which is tied to that context) is never taken — every partition is restored with a full rebalance regardless of the delta size and regardless of `IGNITE_PREFER_WAL_REBALANCE`. +==== + +In a link:clustering/multi-data-center[multi-data-center] setup, the node requesting the data prefers a source from the same data center that holds a copy of the partition. If no such node is available, it falls back to a node from a remote data center. This behavior applies to both full and historical rebalance. + +== Historical Rebalance: Internal Mechanism + +The following section describes the internal flow of a historical rebalance. This information is useful for understanding why the rebalance is chosen as historical or full, and what happens when the historical path fails. + +=== Source Selection + +When the node requesting data (the *demander*) needs a partition, it selects a source node from the available nodes that hold a copy of that partition. In a link:clustering/multi-data-center[multi-data-center] setup, a node from the same data center is preferred. The selection happens during the partition map exchange phase, and the source nodes can be changed during the rebalance if one of them becomes inapplicable. + +=== The Demand-Supply Protocol + +1. The demander sends a *demand* message to the chosen source node (supplier), specifying which partitions it needs and whether it expects a full or historical transfer. +2. The supplier iterates over the partition data (or WAL entries, for historical) and sends *supply* messages back to the demander, batch by batch. +3. The demander sends the next demand when it is ready for more data. This repeats until all partitions are either fully transferred or marked as *missed*. + +=== Fallback from Historical to Full + +If the supplier fails to read the WAL (for example, the WAL segments were archived or deleted), the historical rebalance for that partition is interrupted. The following happens: + +. The supplier's historical iterator throws an exception. +. The supplier marks the last checkpoint as *not applicable for WAL rebalance* (so that subsequent checkpoints will not try to serve WAL for this partition). +. The supplier marks all remaining partitions as *missed* in the supply message. +. The demander, upon receiving the *missed* marking for a partition that was originally requested as historical, excludes the source node from historical rebalancing and re-requests the partition as *full* from another available source node. + +This mechanism ensures that data is never lost: the node will always receive either the historical delta or the full partition content. + +=== Protection Against Data Loss on Full Rebalance + +When a partition is requested as *full* (not historical), the demander additionally checks whether the source node's copy of the partition is up to date. A source node that is known to be behind (its update counter for the partition is lower than the demander's local counter) is *excluded from full rebalancing* for that partition — accepting a full partition from such a node would overwrite newer local data with older data. The demander then selects a different source or waits for the partition to become available. + +=== Message Ordering (Striped Executor) + +Historical rebalance messages are processed on a *striped* executor: all supply messages from the same source node are handled sequentially in the order they were received. This preserves the ordering of WAL entries, which is required for correct replay. Full rebalance messages are processed on a regular (non-ordered) executor because individual partition transfers are independent. + == Configuring Rebalancing Mode Ignite supports both synchronous and asynchronous rebalancing. @@ -147,6 +224,48 @@ The following table lists the properties of `IgniteConfiguration` related to reb |=== +== Rebalance Risks + +A rebalance transfers a significant volume of data between nodes. Depending on the cluster size and the data volume, it can have a measurable impact on availability and performance. The following risks should be considered before scaling a cluster, particularly across data centers. + +=== Bandwidth saturation + +During a rebalance, one or more source nodes simultaneously transfer partition data to the target node. In a link:clustering/multi-data-center[multi-data-center] setup this traffic traverses the inter-DC (WAN) link, which is typically the narrowest part of the topology. A rebalance can saturate this link, degrading or starving all other traffic between the data centers (client requests, replication, monitoring). + +The impact is proportional to the number of source nodes and the size of the transferred partitions, and inversely proportional to the available link bandwidth. It is most severe for small clusters (few nodes, each holding a large share of the data) over limited WAN links. + +=== Full cluster stall (synchronous mode) + +In the `SYNC` rebalancing mode, all public cache operations are blocked until the rebalance completes. For a large cache over a slow or high-latency link, this can result in a prolonged unavailability of the entire cache. + +=== Target node disk I/O + +The target node must persist all received partitions to disk. This produces a write I/O spike that can saturate the node's storage. If the node serves live traffic concurrently, request latency will increase. + +=== Source node contention + +Each source node transfers its share of the partitions while continuing to serve requests. This reduces the effective request throughput of the existing nodes for the duration of the rebalance. The effect is more pronounced in smaller clusters, where each node holds a larger share of the data. + +=== Rebalance timeout + +Each rebalance message is subject to `rebalanceTimeout` (default 10 seconds). Over a slow or high-latency link, transferring a single batch may exceed this value, causing the rebalance to fail and the target node to remain only partially populated. + +== Recommendations for Cluster Administrators + +The following recommendations apply when scaling a cluster, especially when adding nodes to a remote or a new data center. + +. *Use asynchronous rebalancing.* Configure the cache with the `ASYNC` rebalancing mode so that the cache remains available for operations while the data is transferred in the background. The `SYNC` mode should be avoided for large caches or cross-DC transfers. +. *Throttle the transfer.* Set `rebalanceThrottle` (a delay between batches) and, if necessary, reduce `rebalanceBatchSize`, to limit the bandwidth consumed by the rebalance and avoid saturating the inter-DC link. +. *Schedule during low-load periods.* Start the scaling operation when the cluster is under minimal load, so that the rebalance traffic competes with as little production traffic as possible. +. *Increase the timeout for slow links.* On high-latency or low-bandwidth links, increase `rebalanceTimeout` (and reduce `rebalanceBatchSize`) so that individual batches do not time out. +. *Manage the baseline explicitly.* When the baseline topology is not auto-adjusted, confirm the new node is fully up and then commit the current cluster topology as the new baseline via `ignite.cluster().setBaselineTopology(ignite.cluster().topologyVersion())`, so that the rebalance starts at the intended moment. +. *Monitor the rebalance.* Track the link:monitoring-metrics/new-metrics#caches[rebalancing metrics], the inter-DC link utilization, and the I/O load of the target node during the operation, and be prepared to throttle or abort if the impact exceeds expectations. + +[NOTE] +==== +For a node joining for the first time (no local data), the rebalance is always a full transfer of its partitions, regardless of the `IGNITE_PREFER_WAL_REBALANCE` setting. The bandwidth, disk, and stall considerations above therefore apply in full. +==== + == Monitoring Rebalancing Process You can monitor the link:monitoring-metrics/new-metrics#caches[rebalancing process for specific caches using JMX]. diff --git a/docs/_docs/snapshots/snapshots.adoc b/docs/_docs/snapshots/snapshots.adoc index effba8fa05236..0f9378e4cfd61 100644 --- a/docs/_docs/snapshots/snapshots.adoc +++ b/docs/_docs/snapshots/snapshots.adoc @@ -371,7 +371,7 @@ If the metadata isn't found or cannot be read, snapshot isn't deleted. == Getting Snapshot Operation Status -The status of the create/restore/check snapshot operations in the cluster can be obtained using the `control.sh|bat` script. │ +The status of the create/restore/check snapshot operations in the cluster can be obtained using the `control.sh|bat` script. [tabs] -- @@ -390,11 +390,13 @@ control.bat --snapshot status ---- -- - [NOTE] │ -==== │ -Parallel and multiple snapshot check operations can be running concurrently with any other snapshot operation. -But only one check for one snapshot. Create and restore operations cannot run in parallel - only one of each type can -be active at a time. The status is printed for each running snapshot operation (if multiple) separately, one after another. +[NOTE] +==== +Multiple snapshot check operations can run concurrently with any other snapshot operation of another snapshot. +Only one check can be running for the same snapshot at a time. Create and restore operations cannot run in parallel — +only one of them can be active at a time. The status is printed for each running snapshot operation (if multiple) separately, +one after another. +==== == Cancelling Snapshot Operation diff --git a/docs/_docs/tools/control-script.adoc b/docs/_docs/tools/control-script.adoc index 6f7541afa5427..ac52f55792d8c 100644 --- a/docs/_docs/tools/control-script.adoc +++ b/docs/_docs/tools/control-script.adoc @@ -1118,6 +1118,12 @@ control.bat --metric --configure-max-value max-value-metric-name 60000 NOTE: For metric command use following format as metric name: `.`. For example: `io.datastorage.WalLoggingRate` must be set for `WalLoggingRate` metric. +NOTE: For the `PageTimestampHistogram` metric, the values set +link:monitoring-metrics/system-views#pages_timestamp_histogram[time intervals] rather than bounds: +the interval length in milliseconds is the difference between the first two values, the number of intervals is +the number of values, and the other values are ignored. +For example, `0,600000,0` sets three 10-minute intervals. + == Indexes Management The commands below allow to get a specific information on indexes and to trigger the indexes rebuild process. diff --git a/modules/commons/src/main/java/org/apache/ignite/ssl/DelegatingSSLContextSpi.java b/modules/commons/src/main/java/org/apache/ignite/ssl/DelegatingSSLContextSpi.java index 360e92239c6bb..767f51d6b01c1 100644 --- a/modules/commons/src/main/java/org/apache/ignite/ssl/DelegatingSSLContextSpi.java +++ b/modules/commons/src/main/java/org/apache/ignite/ssl/DelegatingSSLContextSpi.java @@ -70,8 +70,8 @@ class DelegatingSSLContextSpi extends SSLContextSpi { } /** {@inheritDoc} */ - @Override protected SSLEngine engineCreateSSLEngine(String s, int i) { - final SSLEngine engine = delegate.createSSLEngine(); + @Override protected SSLEngine engineCreateSSLEngine(String host, int port) { + final SSLEngine engine = delegate.createSSLEngine(host, port); if (parameters != null) engine.setSSLParameters(parameters); diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 73d6a8ba7a044..0902154f7d252 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -85,6 +85,7 @@ import org.apache.ignite.internal.management.cache.FindAndDeleteGarbageInPersistenceTaskResult; import org.apache.ignite.internal.management.cache.IdleVerifyDumpTask; import org.apache.ignite.internal.management.cache.VerifyBackupPartitionsTask; +import org.apache.ignite.internal.management.snapshot.SnapshotStatusCommand; import org.apache.ignite.internal.management.tx.TxInfo; import org.apache.ignite.internal.management.tx.TxTaskResult; import org.apache.ignite.internal.managers.communication.GridIoMessage; @@ -135,7 +136,6 @@ import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.metric.MetricRegistry; -import org.apache.ignite.plugin.PluginProvider; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.spi.metric.LongMetric; @@ -150,7 +150,6 @@ import org.apache.ignite.transactions.TransactionRollbackException; import org.apache.ignite.transactions.TransactionTimeoutException; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.junit.Assume; import org.junit.Test; @@ -238,9 +237,6 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb /** */ protected ListeningTestLogger listeningLog; - /** */ - protected @Nullable PluginProvider pluginProvider; - /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); @@ -250,6 +246,13 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb cleanDiagnosticDir(); } + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + listeningLog = null; + } + /** {@inheritDoc} */ @Override protected void cleanPersistenceDir() throws Exception { super.cleanPersistenceDir(); @@ -264,9 +267,6 @@ public class GridCommandHandlerTest extends GridCommandHandlerClusterPerMethodAb if (listeningLog != null) cfg.setGridLogger(listeningLog); - if (pluginProvider != null) - cfg.setPluginProviders(pluginProvider); - return cfg; } @@ -3657,7 +3657,7 @@ public void testSnapshotRestoreCancelAndStatus() throws Exception { assertTrue(ctxDisposed); assertEquals(EXIT_CODE_OK, execute("--snapshot", "status")); - assertContains(log, testOut.toString(), "There is no create or restore snapshot operation in progress."); + assertContains(log, testOut.toString(), SnapshotStatusCommand.NO_SNP_OP_IN_PROGRESS_MSG); assertNull(ig.cache(DEFAULT_CACHE_NAME)); } @@ -3808,7 +3808,7 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment assertEquals("Unexpected exit code", EXIT_CODE_OK, execute("--snapshot", "status")); - var out = testOut.toString(); + String out = testOut.toString(); if (log.isInfoEnabled()) log.info("Test out:" + U.nl() + out); @@ -4060,7 +4060,7 @@ private void checkSnapshotStatus(boolean isCreating, boolean isRestoring, boolea assertEquals(out, EXIT_CODE_OK, status); if (!isCreating && !isRestoring) { - assertContains(log, out, "There is no create or restore snapshot operation in progress."); + assertContains(log, out, SnapshotStatusCommand.NO_SNP_OP_IN_PROGRESS_MSG); return; } diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/MetricCommandTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/MetricCommandTest.java index 8af01ebcb5b60..de53379c8a496 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/MetricCommandTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/MetricCommandTest.java @@ -163,12 +163,6 @@ public void testInvalidConfigureMetricParameter() { "Can't parse number 'not_a_number', expected type: java.lang.Long" ); - assertContains( - log, - executeCommand(EXIT_CODE_INVALID_ARGUMENTS, CMD_METRIC, CONFIGURE_HISTOGRAM, "some.metric", "3,2,1"), - "Bounds must be sorted" - ); - assertContains( log, executeCommand( @@ -202,6 +196,12 @@ public void testConfigureHistogram() { assertEquals(50, bounds[0]); assertEquals(500, bounds[1]); + assertContains( + log, + executeCommand(EXIT_CODE_INVALID_ARGUMENTS, CMD_METRIC, CONFIGURE_HISTOGRAM, histogram.name(), "3,2,1"), + "bounds should be sorted" + ); + executeCommand(EXIT_CODE_OK, CMD_METRIC, CONFIGURE_HISTOGRAM, histogram.name(), "1,2,3"); bounds = histogram.bounds(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java index 48c1af526fb31..e57161d26384a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java @@ -19,6 +19,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.BitSet; import java.util.Collection; import java.util.List; @@ -38,6 +39,7 @@ import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteOutClosure; import org.apache.ignite.lang.IgniteProductVersion; import org.apache.ignite.lang.IgniteUuid; @@ -59,6 +61,9 @@ public class DirectMessageWriter implements MessageWriter { /** Temporary buffer capacity. */ private static final int TMP_BUF_CAPACITY = 10 * 1024; + /** Empty array of byte arrays. */ + private static final byte[][] EMPTY_BYTE_ARRAYS = new byte[0][]; + /** State. */ @GridToStringInclude private final DirectMessageState state; @@ -81,7 +86,7 @@ public class DirectMessageWriter implements MessageWriter { /** Reusable writer for compressed payload serialization. Lazily created; thread-confined like the writer itself. */ private DirectMessageWriter tmpWriter; - /** Reusable scratch buffer for compressed payload serialization. Retained at the largest size seen so far. */ + /** Reusable fixed-size scratch buffer for compressed payload serialization. */ private ByteBuffer tmpBuf; /** @param msgFactory Message factory. */ @@ -507,12 +512,12 @@ private void writeCompressedMessage( } if (!stream.serializeFinished()) { - // CompressedMessage consumes the scratch buffer in its constructor (deflates into its own byte array), - // so the buffer never escapes this method and can be reused across fields; heap is cheaper than direct - // here (no native alloc / Cleaner churn). - ByteBuffer buf = serializeToTemporaryBuffer(consumer); + try (CompressedMessage.Compressor compressor = new CompressedMessage.Compressor(compressionLvl)) { + serializeToTemporaryBuffer(consumer, compressor::compress); + + stream.compressedMessage(compressor.message()); + } - stream.compressedMessage(new CompressedMessage(buf, compressionLvl)); stream.serializeFinished(true); } @@ -526,11 +531,12 @@ private void writeCompressedMessage( /** {@inheritDoc} */ @Override public void addPendingRawField(int tag, Consumer valueWriter) { - ByteBuffer buf = serializeToTemporaryBuffer(valueWriter::accept); + List pieces = new ArrayList<>(1); - byte[] bytes = new byte[buf.remaining()]; + serializeToTemporaryBuffer(valueWriter::accept, piece -> + pieces.add(Arrays.copyOfRange(piece.array(), piece.position(), piece.limit()))); - buf.get(bytes); + byte[] bytes = pieces.size() == 1 ? pieces.get(0) : U.join(pieces.toArray(EMPTY_BYTE_ARRAYS)); state.item().pendingRawFields().add(new RawField(tag, bytes)); } @@ -544,12 +550,14 @@ private void writeCompressedMessage( return stream.lastFinished(); } - /** */ - private ByteBuffer serializeToTemporaryBuffer(Consumer consumer) { + /** + * Serializes through the temporary buffer as through a socket buffer: each time the buffer fills up or the + * serialization finishes, its content is passed to the piece consumer and the buffer is reused. + */ + private void serializeToTemporaryBuffer(Consumer consumer, Consumer pieceConsumer) { + // Heap is cheaper than direct here (no native alloc / Cleaner churn). if (tmpBuf == null) tmpBuf = ByteBuffer.allocate(TMP_BUF_CAPACITY); - else - tmpBuf.clear(); // Reuse the temp writer across fields/messages instead of allocating a fresh state stack each time. if (tmpWriter == null) @@ -559,29 +567,33 @@ private ByteBuffer serializeToTemporaryBuffer(Consumer cons tmpWriter.setBuffer(tmpBuf); - boolean finished; + boolean ok = false; - do { - if (tmpBuf.remaining() <= tmpBuf.capacity() / 10) { - ByteBuffer newBuf = ByteBuffer.allocate(tmpBuf.capacity() * 2); + try { + boolean finished; - tmpBuf.flip(); - newBuf.put(tmpBuf); + do { + tmpBuf.clear(); - tmpBuf = newBuf; + consumer.accept(tmpWriter); - tmpWriter.setBuffer(tmpBuf); - } + finished = tmpWriter.state.item().stream.lastFinished(); - consumer.accept(tmpWriter); + assert finished || tmpBuf.position() > 0 : "No progress in the temporary buffer"; - finished = tmpWriter.state.item().stream.lastFinished(); - } - while (!finished); + tmpBuf.flip(); - tmpBuf.flip(); + pieceConsumer.accept(tmpBuf); + } + while (!finished); - return tmpBuf; + ok = true; + } + finally { + // Don't reuse a writer with half-written state. + if (!ok) + tmpWriter = null; + } } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsTask.java index 2330c00334ea4..4913a0690e293 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsTask.java @@ -85,12 +85,21 @@ public class VerifyBackupPartitionsTask extends ComputeTaskAdapter grpIds = getGroupIds(); @@ -217,81 +226,102 @@ public VerifyBackupPartitionsJob(CacheIdleVerifyCommandArg arg) { completionCntr.set(0); - List>> partHashCalcFuts = - calcPartitionHashAsync(grpIds); + IgniteException jobFailure = null; + List exceptions = null; + boolean interrupted = false; Map res = new HashMap<>(); - List exceptions = new ArrayList<>(); - long lastProgressLogTs = U.currentTimeMillis(); - for (int i = 0; i < partHashCalcFuts.size(); ) { - if (isCancelled()) { - cancelFuts(i, partHashCalcFuts); - - throw new IgniteException(getClass().getSimpleName() + " was cancelled."); - } + List>> partHashCalcFuts = calcPartitionHashAsync(grpIds); + for (int i = 0; i < partHashCalcFuts.size(); i++) { Future> fut = partHashCalcFuts.get(i); try { - Map partHash = fut.get(100, TimeUnit.MILLISECONDS); + Map partHash = fut.get(PARTITION_FUTURE_TIMEOUT, TimeUnit.MILLISECONDS); - res.putAll(partHash); - - i++; + if (jobFailure == null) + res.putAll(partHash); } - catch (InterruptedException | ExecutionException e) { + catch (ExecutionException e) { if (e.getCause() instanceof IgniteException && !(e.getCause() instanceof GridNotIdleException)) { - exceptions.add((IgniteException)e.getCause()); + if (exceptions == null) + exceptions = new ArrayList<>(); - i++; + exceptions.add((IgniteException)e.getCause()); + } + else if (jobFailure == null) { + jobFailure = new IgniteException("Failed to execute a partition hash calculation [grpIds=" + + grpIds + ", reason=" + e.getCause().getMessage() + ']', e.getCause()); - continue; + cancel(); } + else + jobFailure.addSuppressed(e.getCause()); + } + catch (InterruptedException e) { + interrupted = true; - cancelFuts(i, partHashCalcFuts); + if (jobFailure == null) { + jobFailure = new IgniteInterruptedException("Partition hash calculation has been interrupted " + + "[grpIds=" + grpIds + "].", e); - if (e instanceof InterruptedException) - throw new IgniteInterruptedException((InterruptedException)e); + cancel(); + } else - throw new IgniteException(e.getCause()); + jobFailure.addSuppressed(e); } - catch (TimeoutException ignored) { - if (U.currentTimeMillis() - lastProgressLogTs > 3 * 60 * 1000L) { + catch (TimeoutException e) { + if (U.currentTimeMillis() - lastProgressLogTs > LOG_PROGRESS_TIMEOUT) { lastProgressLogTs = U.currentTimeMillis(); log.warning("idle_verify is still running, processed " + completionCntr.get() + " of " + partHashCalcFuts.size() + " local partitions"); } + + --i; } } - if (log.isInfoEnabled()) - log.info("Idle verify procedure has finished."); + if (interrupted) + Thread.currentThread().interrupt(); + + if (jobFailure != null) + throw jobFailure; + + if (isCancelled()) + throw new IgniteException(CALC_CANCELLED_ERR_MSG + " [grpIds=" + grpIds + "]."); + + if (!F.isEmpty(exceptions)) { + if (ignite.context().isStopping()) { + throw new IgniteException(CALC_CANCELLED_ERR_MSG + " because the local node is stopping [grpIds=" + + grpIds + "]."); + } + + Integer stoppingGrpId = F.find(grpIds, null, this::isGroupStopping); + + if (stoppingGrpId != null) { + throw new IgniteException(CALC_CANCELLED_ERR_MSG + " because the cache group is stopping [grpId=" + + stoppingGrpId + "]."); + } - if (!F.isEmpty(exceptions)) throw new IdleVerifyException(exceptions); + } - return res; - } + if (log.isInfoEnabled()) { + log.info("Idle verify procedure has finished [skipZeros=" + arg.skipZeros() + ", checkCrc=" + + arg.checkCrc() + ", grpIds=" + grpIds + "]."); + } - /** - * Cancels futures from given index. - * - * @param start Index to start from. - * @param partHashCalcFuts Partitions hash calculation futures to cancel. - */ - private void cancelFuts(int start, List>> partHashCalcFuts) { - for (int i = start; i < partHashCalcFuts.size(); i++) - partHashCalcFuts.get(i).cancel(false); + return res; } /** * Class that processes cache filtering chain. */ - private class CachesFiltering { + private static class CachesFiltering { /** * Initially all cache descriptors are included. */ @@ -489,6 +519,9 @@ private Future> calculatePartitionHashAsy final BooleanSupplier cancelled ) { return pool.submit(() -> { + if (isCancelledOrStopped(cancelled, gctx)) + throw cancellationException(gctx, part); + Map res = emptyMap(); if (!part.reserve()) @@ -497,30 +530,42 @@ private Future> calculatePartitionHashAsy try { PartitionUpdateCounter updCntr = part.dataStore().partUpdateCounter(); PartitionUpdateCounter updateCntrBefore = updCntr == null ? null : updCntr.copy(); - - if (arg.checkCrc() && gctx.persistenceEnabled()) { - FilePageStoreManager pageStoreMgr = - (FilePageStoreManager)ignite.context().cache().context().pageStore(); - - checkPartitionsPageCrcSum( - () -> (FilePageStore)pageStoreMgr.getStore(gctx.groupId(), part.id()), - part.id(), - FLAG_DATA, - cancelled + PartitionKey key = new PartitionKey(gctx.groupId(), part.id(), gctx.cacheOrGroupName()); + PartitionHashRecord hash; + + // The cache group can be destroyed concurrently. + if (!gctx.offheap().enterBusy()) + throw cancellationException(gctx, part); + + try { + // Also hold the lock during the read to avoid a concurrent deletion of the partition + // file when the cache is destroyed. + if (arg.checkCrc() && gctx.persistenceEnabled()) { + FilePageStoreManager pageStoreMgr = + (FilePageStoreManager)ignite.context().cache().context().pageStore(); + + checkPartitionsPageCrcSum( + () -> (FilePageStore)pageStoreMgr.getStore(gctx.groupId(), part.id()), + part.id(), + FLAG_DATA, + () -> isCancelledOrStopped(cancelled, gctx) + ); + } + + hash = calculatePartitionHash( + key, + updateCntrBefore == null ? 0 : updateCntrBefore.comparableState(), + ignite.context().discovery().localNode().consistentId(), + part.state(), + part.primary(gctx.topology().readyTopologyVersion()), + part.dataStore().fullSize(), + gctx.offheap().partitionIterator(part.id()), + () -> isCancelledOrStopped(cancelled, gctx) ); } - - PartitionKey key = new PartitionKey(gctx.groupId(), part.id(), gctx.cacheOrGroupName()); - - PartitionHashRecord hash = calculatePartitionHash(key, - updateCntrBefore == null ? 0 : updateCntrBefore.comparableState(), - ignite.context().discovery().localNode().consistentId(), - part.state(), - part.primary(gctx.topology().readyTopologyVersion()), - part.dataStore().fullSize(), - gctx.offheap().partitionIterator(part.id()), - cancelled - ); + finally { + gctx.offheap().leaveBusy(); + } if (hash != null) res = Collections.singletonMap(key, hash); @@ -534,7 +579,8 @@ private Future> calculatePartitionHashAsy } } catch (IgniteCheckedException e) { - String msg = CACL_PART_HASH_ERR_MSG + " [grpId=" + gctx.groupId() + ", partId=" + part.id() + "]"; + String msg = CALC_PART_HASH_ERR_MSG + " [grpId=" + gctx.groupId() + ", partId=" + part.id() + + ", reason=" + e.getMessage() + "]"; U.error(log, msg, e); @@ -549,5 +595,31 @@ private Future> calculatePartitionHashAsy return res; }); } + + /** + * Checks whether the work should be stopped: the task was cancelled, the local node is stopping, + * or the cache group is stopping or being destroyed. + * + * @param cancelled Supplier that returns {@code true} if the task cancellation was requested. + * @param gctx Cache group context. + * @return {@code true} if the work should be stopped. + */ + private boolean isCancelledOrStopped(BooleanSupplier cancelled, CacheGroupContext gctx) { + return cancelled.getAsBoolean() || gctx.isPreparedToStop() || ignite.context().isStopping(); + } + + /** */ + private boolean isGroupStopping(int grpId) { + CacheGroupContext grp = ignite.context().cache().cacheGroup(grpId); + + return grp == null || grp.isPreparedToStop(); + } + + /** */ + private static IgniteException cancellationException(CacheGroupContext gctx, GridDhtLocalPartition part) { + return new IgniteException(CALC_CANCELLED_ERR_MSG + ". The task has been cancelled, or the local node is stopping, " + + "or the cache group is stopping [groupId=" + gctx.groupId() + + ", groupName=" + gctx.cacheOrGroupName() + ", partId=" + part.id() + "]."); + } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/metric/MetricConfigureHistogramCommandArg.java b/modules/core/src/main/java/org/apache/ignite/internal/management/metric/MetricConfigureHistogramCommandArg.java index 7a5a1274f84b8..33301d6990c43 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/metric/MetricConfigureHistogramCommandArg.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/metric/MetricConfigureHistogramCommandArg.java @@ -20,7 +20,6 @@ import org.apache.ignite.internal.Order; import org.apache.ignite.internal.management.api.Argument; import org.apache.ignite.internal.management.api.Positional; -import org.apache.ignite.internal.util.typedef.F; /** */ public class MetricConfigureHistogramCommandArg extends MetricCommandArg { @@ -40,9 +39,6 @@ public long[] newBounds() { /** */ public void newBounds(long[] bounds) { - if (!F.isSorted(bounds)) - throw new IllegalArgumentException("Bounds must be sorted"); - this.newBounds = bounds; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index 33a61e9e300df..bc2a2e3b916f7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -44,6 +44,9 @@ /** */ public class SnapshotStatusCommand extends AbstractSnapshotCommand { + /** */ + public static final String NO_SNP_OP_IN_PROGRESS_MSG = "There is no snapshot operation in progress."; + /** {@inheritDoc} */ @Override public String description() { return "Get the status of the current snapshot operation"; @@ -61,32 +64,33 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand printer) { + // An old-style empty result is null. if (rawResult == null) { printer.accept("There is no create or restore snapshot operation in progress."); return; } - assert rawResult instanceof SnapshotStatus || rawResult instanceof Collection : - "Unsupported job result: " + rawResult.getClass().getSimpleName(); + SnapshotStatus singleOldStatus = rawResult instanceof SnapshotStatus ? (SnapshotStatus)rawResult : null; + Collection statuses = rawResult instanceof Collection + ? (Collection)rawResult + : null; - SnapshotStatus singleStatus = rawResult instanceof SnapshotStatus ? (SnapshotStatus)rawResult : null; - Collection statuses = rawResult instanceof Collection ? (Collection)rawResult : null; + // A new-style result with no statuses is an empty collection, never null. + if (statuses != null && statuses.isEmpty()) { + printer.accept(NO_SNP_OP_IN_PROGRESS_MSG); - assert statuses == null || (!statuses.isEmpty() || (F.first(statuses) instanceof SnapshotStatus)) : - "Statuses collection has unsupported result type"; + return; + } - statuses = statuses == null ? Collections.singleton(singleStatus) : statuses; + statuses = statuses == null ? Collections.singleton(new SnapshotStatusTask.SnapshotOperationStatus(singleOldStatus)) : statuses; - boolean first = true; + boolean separator = false; - for (SnapshotStatus s0 : statuses) { - if (!first) { + for (SnapshotStatusTask.SnapshotOperationStatus s0 : statuses) { + if (separator) printer.accept(U.nl()); - first = false; - } - if (s0.operation() == SnapshotStatusTask.SnapshotOperation.CREATE) printer.accept("Create snapshot operation is in progress."); else if (s0.operation() == SnapshotStatusTask.SnapshotOperation.RESTORE) @@ -128,6 +132,8 @@ else if (s0.operation() == SnapshotStatusTask.SnapshotOperation.RESTORE) .collect(Collectors.toList()); SystemViewCommand.printTable(desc.titles(), desc.types(), rows, printer); + + separator = true; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index 30693dfe98390..bc631d6159b33 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -28,6 +28,8 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; import org.apache.ignite.internal.management.api.NoArg; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcess; @@ -60,9 +62,9 @@ * Task to get the statuses of the current snapshot operations in the cluster. *

Before {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}, returns single {@link SnapshotStatus} for single * snapshot creation or snapshot restoration operation. - *

With {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}, returns collection of {@link SnapshotStatus} for - * possible several snapshot checks operations or parallel creation/restoration and checks operations. Might also contain - * a single status. + *

With {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}, returns collection of {@link SnapshotOperationStatus} + * for possible several snapshot checks operations or parallel creation/restoration and checks operations. May still + * contain a single status. */ @GridInternal public class SnapshotStatusTask extends VisorMultiNodeTask { @@ -98,10 +100,10 @@ public class SnapshotStatusTask extends VisorMultiNodeTask allStatuses = new ArrayList<>(results.size()); + // All the jobs' statuses combined to one flat collection. + Collection allStatuses = new ArrayList<>(results.size()); for (ComputeJobResult jobRes : results) { if (jobRes.getData() == null) @@ -110,104 +112,84 @@ public class SnapshotStatusTask extends VisorMultiNodeTask : - "Unsupported job result type: " + rawResult.getClass().getSimpleName(); - - assert !(rawResult instanceof Collection cll) || cll.isEmpty() || (F.first(cll) instanceof SnapshotStatus) : - "Statuses collection is empty or its first status is of unsupported type"; - - if (rawResult instanceof SnapshotStatus singleJobStatus) { - assert !featureEnabled : - "Single snapshot status result is allowed only if the snapshot check status feature is not enabled"; - assert singleJobStatus.operation() != SnapshotOperation.CHECK : - "Snapshot check status result is allowed only within the result list, not as a single result"; - - allStatuses.add(singleJobStatus); - } + if (rawResult instanceof SnapshotStatus singleJobStatus) + allStatuses.add(new SnapshotOperationStatus(singleJobStatus)); else { - assert featureEnabled : - "Many snapshot status results are allowed only if the snapshot check status feature is enabled"; - - Collection manyStatuses = (Collection)rawResult; + Collection jobMultipleStatuses = (Collection)rawResult; - // Skip the check status if cluster supports, but not a thin client of previous version. - if (!initiatorSupport) { - manyStatuses = manyStatuses.stream().filter(s -> s.operation() != SnapshotOperation.CHECK).toList(); + // Skip the check status for an old command initiator that doesn't know about it. + if (!initiatorSupports) { + jobMultipleStatuses = jobMultipleStatuses.stream().filter(s -> s.operation() != SnapshotOperation.CHECK).toList(); - assert manyStatuses.isEmpty() || manyStatuses.size() == 1 : - "Snapshot creation or restoration are mutually exclusive, only one of them allowed at the same time."; + assert jobMultipleStatuses.isEmpty() || jobMultipleStatuses.size() == 1 : + "Snapshot creation and restoration are mutually exclusive; only one of them can run at a time."; } - allStatuses.addAll(manyStatuses); + allStatuses.addAll(jobMultipleStatuses); } } - // There is no snapshot operation or the check operation in not supported yet in the result. + // Empty list notifies that this is the new result but is empty. Old empty is always null. if (allStatuses.isEmpty()) - return null; + return initiatorSupports && featureEnabled ? (Serializable)Collections.emptyList() : null; - ArrayList tastResultStatuses = null; + ArrayList taskResult = new ArrayList<>(); // First, find create or restore operation. - Collection filtered = F.viewReadOnly(allStatuses, s -> s, s -> s.operation() != SnapshotOperation.CHECK); + Collection filtered = F.viewReadOnly(allStatuses, s -> s, s -> s.operation() != SnapshotOperation.CHECK); // Found create or restore statuses. if (!filtered.isEmpty()) { // Merged create or restore. - SnapshotStatus merged = F.first(filtered); + SnapshotOperationStatus merged = F.first(filtered); - String mergedRqId = merged.requestId; + String mergedRqId = merged.reqId; // Filter out differing requests due to concurrent updates on nodes. - filtered = F.view(filtered, s -> s.requestId.equals(mergedRqId)); + filtered = F.view(filtered, s -> s.reqId.equals(mergedRqId)); // Merge nodes progress. Map> progress = new HashMap<>(); filtered.forEach(s -> progress.putAll(s.progress)); - merged = new SnapshotStatus(merged.op, merged.name, merged.incIdx, merged.requestId, merged.startTime, progress); + merged = new SnapshotOperationStatus(merged.op, merged.name, merged.incIdx, merged.reqId, merged.startTime, progress); - // Return the old style single result. - if (!initiatorSupport || !featureEnabled) - return merged; + // If the initiator like thin client of 'control.sh' doesn't support the feature yet, we have to rewrap + // the first single result back to the old format. + if (!initiatorSupports) + return new SnapshotStatus(merged.op, merged.name, merged.incIdx, merged.reqId, merged.startTime, progress); - if (tastResultStatuses == null) - tastResultStatuses = new ArrayList<>(); - - tastResultStatuses.add(merged); + taskResult.add(merged); } - // Check operation statuses. + // Check operation statuses go last. filtered = F.viewReadOnly(allStatuses, s -> s, s -> s.operation() == SnapshotOperation.CHECK); // Found check statuses. if (!filtered.isEmpty()) { - // Merged check statuses per snapshot name. - Map mergedCheckStatuses = U.newHashMap(filtered.size()); + // Merged check statuses per snapshot. + Map mergedCheckStatuses = U.newHashMap(filtered.size()); filtered.forEach(chkStatus -> { - SnapshotStatus chkStatus0 = mergedCheckStatuses.computeIfAbsent(chkStatus.name, (snpName -> chkStatus)); + SnapshotOperationStatus chkStatus0 = mergedCheckStatuses.computeIfAbsent(chkStatus.name, (snpName -> chkStatus)); if (chkStatus0 != chkStatus) chkStatus0.progress.putAll(chkStatus.progress); }); - if (tastResultStatuses == null) - tastResultStatuses = new ArrayList<>(); - - tastResultStatuses.addAll(mergedCheckStatuses.values()); + taskResult.addAll(mergedCheckStatuses.values()); } - return tastResultStatuses; + return taskResult; } /** * A job to get the statuses of the current snapshot operations on current node. *

Before {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}, returns single {@link SnapshotStatus} for single * snapshot creation or snapshot restoration operation. - *

With {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}, returns collection of {@link SnapshotStatus} for - * possible several snapshot checks operations or parallel creation/restoration and checks operations. Might also + *

With {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}, returns collection of {@link SnapshotOperationStatus} + * for possible several snapshot checks operations or parallel creation/restoration and checks operations. May still * contain a single status. */ private static class SnapshotStatusJob extends SnapshotJob { @@ -294,14 +276,14 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { return createOrRestoreStatus; } - // Many results. - ArrayList severalStatuses = null; + // New many results. + ArrayList snpOpStatuses = null; if (createOrRestoreStatus != null) { - severalStatuses = new ArrayList<>(); + snpOpStatuses = new ArrayList<>(); - // Status of creation or restoration operation always goes firts. This is important for old command initiators (control.sh). - severalStatuses.add(createOrRestoreStatus); + // Status of creation or restoration operation always goes first. This is important for old command initiators (control.sh). + snpOpStatuses.add(new SnapshotOperationStatus(createOrRestoreStatus)); } for (ReadOnlyMetricRegistry snpCheckMReg : ignite.context().metric()) { @@ -335,7 +317,7 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { ); } - SnapshotStatus status = new SnapshotStatus( + SnapshotOperationStatus snpOpStatus = new SnapshotOperationStatus( SnapshotOperation.CHECK, MetricUtils.fromFullName(snpCheckMReg.name()).get2(), incIdx, @@ -344,13 +326,13 @@ protected SnapshotStatusJob(@Nullable NoArg arg, boolean debug) { F.asMap(ignite.localNode().id(), metrics) ); - if (severalStatuses == null) - severalStatuses = new ArrayList<>(); + if (snpOpStatuses == null) + snpOpStatuses = new ArrayList<>(); - severalStatuses.add(status); + snpOpStatuses.add(snpOpStatus); } - return severalStatuses; + return snpOpStatuses; } } @@ -425,6 +407,96 @@ public Map> progress() { } } + /** + * Snapshot operation status after {@link CoreFeatureRegistry#SNAPSHOT_CHECK_STATUS_FEATURE}. + * Is now a {@link IgniteDataTransferObject}. + */ + public static class SnapshotOperationStatus extends IgniteDataTransferObject { + /** */ + private static final long serialVersionUID = 0L; + + /** Operation type. */ + @Order(0) + SnapshotOperation op; + + /** Snapshot name. */ + @Order(1) + String name; + + /** Incremental snapshot index. */ + @Order(2) + int incIdx; + + /** Request ID. */ + @Order(3) + String reqId; + + /** Start time. */ + @Order(4) + long startTime; + + /** Progress of operation on nodes. */ + @Order(5) + Map> progress; + + /** Empty constructor for serialization purposes. */ + public SnapshotOperationStatus() { + // No-op. + } + + /** */ + private SnapshotOperationStatus( + SnapshotOperation op, + String name, + int incIdx, + String reqId, + long startTime, + Map> progress + ) { + this.op = op; + this.name = name; + this.incIdx = incIdx; + this.reqId = reqId; + this.startTime = startTime; + this.progress = progress; + } + + /** */ + public SnapshotOperationStatus(SnapshotStatus s) { + this(s.op, s.name, s.incIdx, s.requestId, s.startTime, s.progress); + } + + /** @return Operation type. */ + public SnapshotOperation operation() { + return op; + } + + /** @return Snapshot name. */ + public String name() { + return name; + } + + /** @return Incremental snapshot index. */ + public int incrementIndex() { + return incIdx; + } + + /** @return Request ID. */ + public String requestId() { + return reqId; + } + + /** @return Start time. */ + public long startTime() { + return startTime; + } + + /** @return Progress of operation on nodes. */ + public Map> progress() { + return Collections.unmodifiableMap(progress); + } + } + /** Snapshot operation type. */ public enum SnapshotOperation { /** Create snapshot. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/CompressedMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/CompressedMessage.java index 0988452f6a506..3c99af130f746 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/CompressedMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/CompressedMessage.java @@ -50,7 +50,7 @@ public class CompressedMessage extends AbstractMessage implements NonMarshallabl */ private static final int MAX_DEFLATE_BLOCK_OVERHEAD = 64; - /** Compressed data chunks: filled by {@link #compress(ByteBuffer)} on send, by the serializer on receive. */ + /** Compressed data chunks: filled by {@link Compressor} on send, by the serializer on receive. */ List chunks; /** Index of the next chunk to send. */ @@ -65,26 +65,11 @@ public class CompressedMessage extends AbstractMessage implements NonMarshallabl /** Flag indicating whether this is the last chunk. */ boolean finalChunk; - /** Compression level. */ - private int compressionLvl; - /** Constructor. */ public CompressedMessage() { // No-op. } - /** - * @param buf Source buffer with serialized data. - * @param compressionLvl Compression level. - */ - public CompressedMessage(ByteBuffer buf, int compressionLvl) { - dataSize = buf.remaining(); - this.compressionLvl = compressionLvl; - - if (dataSize > 0) - compress(buf); - } - /** @return Raw data size. */ public int dataSize() { return dataSize; @@ -102,40 +87,6 @@ public byte[] nextChunk() { return chunkIdx < chunks.size() ? chunks.get(chunkIdx++) : null; } - /** @param buf Buffer. */ - private void compress(ByteBuffer buf) { - Deflater deflater = new Deflater(compressionLvl, true); - - try { - deflater.setInput(buf); - deflater.finish(); - - chunks = new ArrayList<>(dataSize / CHUNK_SIZE + 1); - - // Incompressible data may expand: raw-deflate worst case below CHUNK_SIZE is one stored block (~11 bytes overhead). - byte[] chunk0 = new byte[dataSize >= CHUNK_SIZE ? CHUNK_SIZE : dataSize + 16]; - - int len = 0; - - while (!deflater.finished()) { - len += deflater.deflate(chunk0, len, chunk0.length - len); - - if (len == chunk0.length && !deflater.finished()) { - chunks.add(chunk0); - - chunk0 = new byte[CHUNK_SIZE]; - len = 0; - } - } - - if (len > 0) - chunks.add(len == chunk0.length ? chunk0 : Arrays.copyOf(chunk0, len)); - } - finally { - deflater.end(); - } - } - /** @return Uncompressed data. */ private byte[] uncompress() { if (chunks == null) @@ -201,4 +152,85 @@ private byte[] uncompress() { @Override public String toString() { return S.toString(CompressedMessage.class, this); } + + /** Compresses serialized data piece by piece, so that the data never has to be held in one buffer. */ + public static class Compressor implements AutoCloseable { + /** */ + private final Deflater deflater; + + /** */ + private final List chunks = new ArrayList<>(); + + /** Chunk being filled. */ + private byte[] chunk; + + /** Number of bytes in {@link #chunk}. */ + private int chunkLen; + + /** Raw data size. */ + private int dataSize; + + /** @param compressionLvl Compression level. */ + public Compressor(int compressionLvl) { + deflater = new Deflater(compressionLvl, true); + } + + /** + * Compresses the next piece of serialized data. The piece is consumed fully, so its buffer can be reused for + * the next piece. + * + * @param piece Piece of serialized data. + */ + public void compress(ByteBuffer piece) { + // Incompressible data may expand: raw-deflate worst case below CHUNK_SIZE is one stored block (~11 bytes overhead). + if (chunk == null) + chunk = new byte[Math.min(CHUNK_SIZE, piece.remaining() + 16)]; + + dataSize += piece.remaining(); + + deflater.setInput(piece); + + deflate(false); + } + + /** @return Message with the compressed data. */ + public CompressedMessage message() { + CompressedMessage msg = new CompressedMessage(); + + msg.dataSize = dataSize; + + if (dataSize > 0) { + deflater.finish(); + + deflate(true); + + if (chunkLen > 0) + chunks.add(chunkLen == chunk.length ? chunk : Arrays.copyOf(chunk, chunkLen)); + + msg.chunks = chunks; + } + + return msg; + } + + /** @param finish Whether to deflate until the stream is finished rather than until the input is consumed. */ + private void deflate(boolean finish) { + do { + if (chunkLen == chunk.length) { + chunks.add(chunk); + + chunk = new byte[CHUNK_SIZE]; + chunkLen = 0; + } + + chunkLen += deflater.deflate(chunk, chunkLen, chunk.length - chunkLen); + } + while (finish ? !deflater.finished() : !deflater.needsInput()); + } + + /** {@inheritDoc} */ + @Override public void close() { + deflater.end(); + } + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index 54884f20bb755..2ddd55889e616 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -408,6 +408,21 @@ public void storeEntries(GridDhtLocalPartition part, Iterator ctx.req.requestId().toString(), String.class, - "The request ID of the last running cluster snapshot check operation on this node." + "The request ID of the current cluster snapshot check operation on this node." ); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java index bedd6243724ec..fac5b21c9c39a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java @@ -23,14 +23,11 @@ import java.util.stream.Collectors; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.processors.metric.GridMetricManager; -import org.apache.ignite.internal.processors.metric.impl.MetricUtils; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.metric.MetricRegistry; import org.apache.ignite.mxbean.SnapshotMXBean; import org.apache.ignite.spi.metric.IntMetric; -import org.apache.ignite.spi.metric.Metric; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.DFLT_CHECK_ON_RESTORE; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotRestoreProcess.SNAPSHOT_RESTORE_METRICS; @@ -39,6 +36,10 @@ * Snapshot MBean features. */ public class SnapshotMXBeanImpl implements SnapshotMXBean { + /** */ + public static final String NO_CREATE_OR_RESTORE_IN_PROGRESS_MSG = + "There is no create or restore snapshot operation in progress."; + /** Instance of snapshot cache shared manager. */ private final IgniteSnapshotManager mgr; @@ -119,17 +120,16 @@ public SnapshotMXBeanImpl(GridKernalContext ctx) { /** {@inheritDoc} */ @Override public String status() { - StringBuilder res = null; - SnapshotOperationRequest req = mgr.currentCreateRequest(); if (req != null) { - res = new StringBuilder("Create snapshot operation is in progress [name=" + req.snapshotName() + + return "Create snapshot operation is in progress [name=" + req.snapshotName() + ", incremental=" + req.incremental() + (req.incremental() ? (", incrementIndex=" + req.incrementIndex()) : "") + - ", id=" + req.requestId() + ']'); + ", id=" + req.requestId() + ']'; } - else if (mgr.isRestoring()) { + + if (mgr.isRestoring()) { MetricRegistry mreg = metricMgr.registry(SNAPSHOT_RESTORE_METRICS); String name = mreg.findMetric("snapshotName").getAsString(); @@ -138,53 +138,10 @@ else if (mgr.isRestoring()) { boolean incremental = incIdx > 0; - res = new StringBuilder("Restore snapshot operation is in progress [name=" + name + ", incremental=" + incremental + - (incremental ? ", incrementIndex=" + incIdx : "") + ", id=" + id + ']'); + return "Restore snapshot operation is in progress [name=" + name + ", incremental=" + incremental + + (incremental ? ", incrementIndex=" + incIdx : "") + ", id=" + id + ']'; } - boolean one = true; - - for (var mreg : metricMgr) { - if (!mreg.name().startsWith(SnapshotCheckProcess.SNAPSHOT_CHECK_METRIC)) - continue; - - Metric rqIdMetric = mreg.findMetric("requestId"); - - // The requestId metric is registered last. - if (rqIdMetric == null) - continue; - - String name = MetricUtils.fromFullName(mreg.name()).get2(); - int incIdx = mreg.findMetric("incrementIndex").value(); - String id = rqIdMetric.getAsString(); - - if (one) { - if (res == null) - res = new StringBuilder(); - else - res.append(U.nl()).append(U.nl()); - } - - res.append("Check snapshot operations are in progress ["); - - StringBuilder sb0 = new StringBuilder(); - - if (!one) - sb0.append(", "); - - sb0.append("[name=").append(name).append(", incremental=").append(incIdx > 0) - .append(incIdx > 0 ? ", incrementIndex=" + incIdx : "").append(", id=").append(id).append(']'); - - res.append(sb0); - - one = false; - } - - if (res == null) - return "No snapshot operation is in progress."; - - res.append(']'); - - return res.toString(); + return NO_CREATE_OR_RESTORE_IN_PROGRESS_MSG; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java index 84c79b88cc6a6..16562996ef954 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java @@ -304,15 +304,17 @@ private Map checkSnapshotFiles( // There is no `primary` partitions for snapshot. PartitionKey key = new PartitionKey(grpId, partId, grpName); - PartitionHashRecord hash = calculatePartitionHash(key, + PartitionHashRecord hash = calculatePartitionHash( + key, updateCntr, meta.consistentId(), GridDhtPartitionState.OWNING, false, size, - skipHash() ? F.emptyIterator() + skipHash() + ? F.emptyIterator() : snpMgr.partitionRowIterator(snpCtx, grpName, partId, pageStore), - null + () -> cctx.kernalContext().isStopping() ); assert hash != null : "OWNING must have hash: " + key; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java index 3cccbb3e72a3a..6a42f7138b2e5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java @@ -122,7 +122,7 @@ public static void checkPartitionsPageCrcSum( for (int pageNo = 0; pageNo < pageStore.pages(); pageId++, pageNo++) { if (cancelled != null && cancelled.getAsBoolean()) - throw new IgniteException("Checking of partitions page CRC sum has been cancelled."); + throw new IgniteException("Checking of partitions page CRC sum has been cancelled [partId=" + partId + ']'); buf.clear(); @@ -288,17 +288,18 @@ public static List compareUpdateCounters( boolean isPrimary, long partSize, GridIterator it, - @Nullable BooleanSupplier cancelled + BooleanSupplier cancelled ) throws IgniteCheckedException { if (state == GridDhtPartitionState.MOVING || state == GridDhtPartitionState.LOST) { - return new PartitionHashRecord(partKey, + return new PartitionHashRecord( + partKey, isPrimary, consId, updCntr, - state == GridDhtPartitionState.MOVING ? - PartitionHashRecord.MOVING_PARTITION_SIZE : 0, - state == GridDhtPartitionState.MOVING ? - PartitionHashRecord.PartitionState.MOVING : PartitionHashRecord.PartitionState.LOST, + state == GridDhtPartitionState.MOVING ? PartitionHashRecord.MOVING_PARTITION_SIZE : 0, + state == GridDhtPartitionState.MOVING + ? PartitionHashRecord.PartitionState.MOVING + : PartitionHashRecord.PartitionState.LOST, new VerifyPartitionContext() ); } @@ -308,10 +309,9 @@ public static List compareUpdateCounters( VerifyPartitionContext ctx = new VerifyPartitionContext(); - while (it.hasNextX()) { - if (cancelled != null && cancelled.getAsBoolean()) - throw new IgniteCheckedException("Partition hash calculation has been cancelled."); + boolean cancelled0; + while (!(cancelled0 = cancelled.getAsBoolean()) && it.hasNextX()) { CacheDataRow row = it.nextX(); if (row.expireTime() > 0) @@ -320,6 +320,9 @@ public static List compareUpdateCounters( ctx.update(row.key(), row.value(), row.version()); } + if (cancelled0) + throw new IgniteException("Partition hash calculation has been cancelled [part=" + partKey + ']'); + return new PartitionHashRecord( partKey, isPrimary, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/ConfigurableHistogramMetric.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/ConfigurableHistogramMetric.java index 2a3d15ca16e6b..62252010c54a6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/ConfigurableHistogramMetric.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/ConfigurableHistogramMetric.java @@ -25,4 +25,7 @@ public interface ConfigurableHistogramMetric extends HistogramMetric { /** Sets bounds for this histogram. */ public void bounds(long[] bounds); + + /** Checks that the bounds can be set for this histogram, throws {@link IllegalArgumentException} otherwise. */ + public void checkBounds(long[] bounds); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/GridMetricManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/GridMetricManager.java index c116f17a70f43..dd7d7759d3b4e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/GridMetricManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/GridMetricManager.java @@ -511,13 +511,14 @@ private void configureIntervalMetric( */ public void configureHistogram(String name, long[] bounds) throws IgniteCheckedException { A.notNullOrEmpty(name, "name"); - A.notEmpty(bounds, "bounds"); A.notNull(metastorage, "Metastorage not ready. Node not started?"); if (ctx.isStopping()) throw new NodeStoppingException("Operation has been cancelled (node is stopping)"); - ensureMetricRegistered(name, HistogramMetric.class); + ensureMetricRegistered(name, ConfigurableHistogramMetric.class); + + find(name, ConfigurableHistogramMetric.class).checkBounds(bounds); metastorage.write(metricName(HISTOGRAM_CFG_PREFIX, name), bounds); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/MetricRegistryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/MetricRegistryImpl.java index d4fe5729ef2bf..94cb8a22c67bd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/MetricRegistryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/MetricRegistryImpl.java @@ -319,11 +319,11 @@ private T addMetric(String name, T metric) throws IgniteExcep private void configureMetrics(Metric metric) { assert !customMetric(name()) : "Custom metrics cannot be configured yet"; - if (metric instanceof HistogramMetricImpl) { + if (metric instanceof ConfigurableHistogramMetric) { long[] cfgBounds = histogramCfgProvider.apply(metric.name()); if (cfgBounds != null) - ((HistogramMetricImpl)metric).reset(cfgBounds); + ((ConfigurableHistogramMetric)metric).bounds(cfgBounds); } else if (metric instanceof HitRateMetric) { Long cfgRateTimeInterval = hitRateCfgProvider.apply(metric.name()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/HistogramMetricImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/HistogramMetricImpl.java index 2526300613951..c85499090c664 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/HistogramMetricImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/HistogramMetricImpl.java @@ -22,6 +22,7 @@ import org.apache.ignite.internal.processors.metric.AbstractMetric; import org.apache.ignite.internal.processors.metric.ConfigurableHistogramMetric; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.A; import org.jetbrains.annotations.Nullable; /** @@ -83,9 +84,17 @@ public void reset(long[] bounds) { /** {@inheritDoc} */ @Override public void bounds(long[] bounds) { + checkBounds(bounds); + reset(bounds); } + /** {@inheritDoc} */ + @Override public void checkBounds(long[] bounds) { + A.notEmpty(bounds, "bounds"); + A.ensure(F.isSorted(bounds), "bounds should be sorted"); + } + /** {@inheritDoc} */ @Override public void reset() { reset(holder.bounds); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/PeriodicHistogramMetricImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/PeriodicHistogramMetricImpl.java index 008173c0b94ce..d935bb4fda78c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/PeriodicHistogramMetricImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/PeriodicHistogramMetricImpl.java @@ -110,14 +110,19 @@ private PeriodicHistogramMetricImpl(long startTs, String name, @Nullable String /** {@inheritDoc} */ @Override public void bounds(long[] bounds) { - A.notNull(bounds, "bounds"); - A.ensure(bounds.length > 1, "bounds.length > 1"); - A.ensure(bounds[0] < bounds[1], "bounds[0] < bounds[1]"); + checkBounds(bounds); // We need only interval between bounds and count of buckets, skip all values except first 2. reinit(bounds[1] - bounds[0], bounds.length); } + /** {@inheritDoc} */ + @Override public void checkBounds(long[] bounds) { + A.notNull(bounds, "bounds"); + A.ensure(bounds.length > 1, "bounds.length > 1"); + A.ensure(bounds[0] < bounds[1], "bounds[0] < bounds[1]"); + } + /** {@inheritDoc} */ @Override public long[] value() { return histogram().get2(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/offheap/GridOffHeapProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/offheap/GridOffHeapProcessor.java deleted file mode 100644 index aa0109f24e1ba..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/offheap/GridOffHeapProcessor.java +++ /dev/null @@ -1,417 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.offheap; - -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.internal.GridKernalContext; -import org.apache.ignite.internal.processors.GridProcessorAdapter; -import org.apache.ignite.internal.processors.cache.KeyCacheObject; -import org.apache.ignite.internal.util.GridEmptyCloseableIterator; -import org.apache.ignite.internal.util.lang.GridCloseableIterator; -import org.apache.ignite.internal.util.offheap.GridOffHeapEvictListener; -import org.apache.ignite.internal.util.offheap.GridOffHeapMapFactory; -import org.apache.ignite.internal.util.offheap.GridOffHeapPartitionedMap; -import org.apache.ignite.internal.util.typedef.CX2; -import org.apache.ignite.internal.util.typedef.T2; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.lang.IgniteBiPredicate; -import org.apache.ignite.lang.IgniteBiTuple; -import org.apache.ignite.marshaller.Marshaller; -import org.jetbrains.annotations.Nullable; - -/** - * Manages offheap memory caches. - */ -public class GridOffHeapProcessor extends GridProcessorAdapter { - /** */ - private final ConcurrentHashMap offheap = - new ConcurrentHashMap<>(); - - /** */ - private final Marshaller marsh; - - /** - * @param ctx Kernal context. - */ - public GridOffHeapProcessor(GridKernalContext ctx) { - super(ctx); - - marsh = ctx.marshaller(); - } - - /** - * Creates offheap map for given space name. Previous one will be destructed if it exists. - * - * @param spaceName Space name. - * @param parts Partitions number. - * @param init Initial size. - * @param max Maximum size. - * @param lsnr Eviction listener. - */ - public void create(@Nullable String spaceName, int parts, long init, long max, - @Nullable GridOffHeapEvictListener lsnr) { - spaceName = maskNull(spaceName); - - GridOffHeapPartitionedMap m = GridOffHeapMapFactory.unsafePartitionedMap(parts, 1024, 0.75f, init, max, - (short)512, lsnr); - - GridOffHeapPartitionedMap old = offheap.put(spaceName, m); - - if (old != null) - old.destruct(); - } - - /** - * Destructs offheap map for given space name. - * - * @param spaceName Space name. - * */ - public void destruct(@Nullable String spaceName) { - spaceName = maskNull(spaceName); - - GridOffHeapPartitionedMap map = offheap.remove(spaceName); - - if (map != null) - map.destruct(); - } - - /** {@inheritDoc} */ - @Override public void stop(boolean cancel) throws IgniteCheckedException { - super.stop(cancel); - - for (GridOffHeapPartitionedMap m : offheap.values()) - m.destruct(); - } - - /** - * Gets offheap swap space for given space name. - * - * @param spaceName Space name. - * @return Offheap swap space. - */ - @Nullable private GridOffHeapPartitionedMap offheap(@Nullable String spaceName) { - return offheap.get(maskNull(spaceName)); - } - - /** - * Ensures that we have {@code keyBytes}. - * - * @param key Key. - * @param keyBytes Optional key bytes. - * @return Key bytes - * @throws IgniteCheckedException If failed. - */ - private byte[] keyBytes(KeyCacheObject key, @Nullable byte[] keyBytes) throws IgniteCheckedException { - assert key != null; - - return keyBytes != null ? keyBytes : U.marshal(marsh, key); - } - - /** - * Masks {@code null} space name. - * - * @param spaceName Space name. - * @return Masked space name. - */ - private String maskNull(@Nullable String spaceName) { - if (spaceName == null) - return "gg-dflt-offheap-swap"; - - return spaceName; - } - - /** - * Checks if offheap space contains value for the given key. - * - * @param spaceName Space name. - * @param part Partition. - * @param key Key. - * @param keyBytes Key bytes. - * @return {@code true} If offheap space contains value for the given key. - * @throws IgniteCheckedException If failed. - */ - public boolean contains(@Nullable String spaceName, int part, KeyCacheObject key, byte[] keyBytes) - throws IgniteCheckedException { - GridOffHeapPartitionedMap m = offheap(spaceName); - - return m != null && m.contains(part, U.hash(key), keyBytes(key, keyBytes)); - } - - /** - * Gets value bytes from offheap space for the given key. - * - * @param spaceName Space name. - * @param part Partition. - * @param key Key. - * @param keyBytes Key bytes. - * @return Value bytes. - * @throws IgniteCheckedException If failed. - */ - @Nullable public byte[] get(@Nullable String spaceName, int part, KeyCacheObject key, byte[] keyBytes) - throws IgniteCheckedException { - GridOffHeapPartitionedMap m = offheap(spaceName); - - return m == null ? null : m.get(part, U.hash(key), keyBytes(key, keyBytes)); - } - - /** - * Gets value pointer from offheap space for the given key. While pointer is in use eviction is - * disabled for corresponding entry. Eviction for entry is enabled when {@link #put} or - * {@link #enableEviction} is called. - * - * @param spaceName Space name. - * @param part Partition. - * @param key Key. - * @param keyBytes Key bytes. - * @return Tuple where first value is pointer and second is value size. - * @throws IgniteCheckedException If failed. - */ - @Nullable public IgniteBiTuple valuePointer(@Nullable String spaceName, int part, KeyCacheObject key, - byte[] keyBytes) throws IgniteCheckedException { - GridOffHeapPartitionedMap m = offheap(spaceName); - - return m == null ? null : m.valuePointer(part, U.hash(key), keyBytes(key, keyBytes)); - } - - /** - * Enables eviction for entry after {@link #valuePointer} was called. - * - * @param spaceName Space name. - * @param part Partition. - * @param key Key. - * @param keyBytes Key bytes. - * @throws IgniteCheckedException If failed. - */ - public void enableEviction(@Nullable String spaceName, int part, KeyCacheObject key, byte[] keyBytes) - throws IgniteCheckedException { - GridOffHeapPartitionedMap m = offheap(spaceName); - - if (m != null) - m.enableEviction(part, U.hash(key), keyBytes(key, keyBytes)); - } - - /** - * Gets value from offheap space for the given key. - * - * @param spaceName Space name. - * @param part Partition. - * @param key Key. - * @param keyBytes Key bytes. - * @param ldr Class loader. - * @return Value bytes. - * @throws IgniteCheckedException If failed. - */ - @Nullable public T getValue(@Nullable String spaceName, int part, KeyCacheObject key, byte[] keyBytes, - @Nullable ClassLoader ldr) throws IgniteCheckedException { - byte[] valBytes = get(spaceName, part, key, keyBytes); - - if (valBytes == null) - return null; - - return U.unmarshal(marsh, valBytes, U.resolveClassLoader(ldr, ctx.config())); - } - - /** - * Removes value from offheap space for the given key. - * - * @param spaceName Space name. - * @param part Partition. - * @param key Key. - * @param keyBytes Key bytes. - * @return Value bytes. - * @throws IgniteCheckedException If failed. - */ - @Nullable public byte[] remove( - @Nullable String spaceName, - int part, - KeyCacheObject key, - byte[] keyBytes - ) throws IgniteCheckedException { - GridOffHeapPartitionedMap m = offheap(spaceName); - - if (log.isTraceEnabled()) - log.trace("offheap remove [key=" + key + ']'); - - return m == null ? null : m.remove(part, U.hash(key), keyBytes(key, keyBytes)); - } - - /** - * Puts the given value to offheap space for the given key. - * - * @param spaceName Space name. - * @param part Partition. - * @param key Key. - * @param keyBytes Key bytes. - * @param valBytes Value bytes. - * @throws IgniteCheckedException If failed. - */ - public void put(@Nullable String spaceName, int part, KeyCacheObject key, byte[] keyBytes, byte[] valBytes) - throws IgniteCheckedException { - GridOffHeapPartitionedMap m = offheap(spaceName); - - if (m == null) - throw new IgniteCheckedException("Failed to write data to off-heap space, no space registered for name: " + - spaceName); - - if (log.isTraceEnabled()) - log.trace("offheap put [key=" + key + ']'); - - m.put(part, U.hash(key), keyBytes(key, keyBytes), valBytes); - } - - /** - * Removes value from offheap space for the given key. - * - * @param spaceName Space name. - * @param part Partition. - * @param key Key. - * @param keyBytes Key bytes. - * @return {@code true} If succeeded. - * @throws IgniteCheckedException If failed. - */ - public boolean removex(@Nullable String spaceName, int part, KeyCacheObject key, byte[] keyBytes) - throws IgniteCheckedException { - GridOffHeapPartitionedMap m = offheap(spaceName); - - if (log.isTraceEnabled()) - log.trace("offheap removex [key=" + key + ']'); - - return m != null && m.removex(part, U.hash(key), keyBytes(key, keyBytes)); - } - - /** - * Removes value from offheap space for the given key. - * - * @param spaceName Space name. - * @param part Partition. - * @param key Key. - * @param keyBytes Key bytes. - * @param p Value predicate (arguments are value address and value length). - * @return {@code true} If succeeded. - * @throws IgniteCheckedException If failed. - */ - public boolean removex(@Nullable String spaceName, - int part, - KeyCacheObject key, - byte[] keyBytes, - IgniteBiPredicate p) throws IgniteCheckedException { - GridOffHeapPartitionedMap m = offheap(spaceName); - - if (log.isTraceEnabled()) - log.trace("offheap removex [key=" + key + ']'); - - return m != null && m.removex(part, U.hash(key), keyBytes(key, keyBytes), p); - } - - /** - * Gets iterator over contents of the given space. - * - * @param spaceName Space name. - * @return Iterator. - */ - public GridCloseableIterator> iterator(@Nullable String spaceName) { - GridOffHeapPartitionedMap m = offheap(spaceName); - - return m == null ? new GridEmptyCloseableIterator>() : m.iterator(); - } - - /** - * Gets iterator over contents of the given space. - * - * @param spaceName Space name. - * @param c Key/value closure. - * @return Iterator. - */ - public GridCloseableIterator iterator(@Nullable String spaceName, - CX2, T2, T> c) { - assert c != null; - - GridOffHeapPartitionedMap m = offheap(spaceName); - - return m == null ? new GridEmptyCloseableIterator() : m.iterator(c); - } - - /** - * Gets iterator over contents of the given space. - * - * @param spaceName Space name. - * @param c Key/value closure. - * @param part Partition. - * @return Iterator. - */ - public GridCloseableIterator iterator(@Nullable String spaceName, - CX2, T2, T> c, int part) { - assert c != null; - - GridOffHeapPartitionedMap m = offheap(spaceName); - - return m == null ? new GridEmptyCloseableIterator() : m.iterator(c, part); - } - - /** - * Gets number of elements in the given space. - * - * @param spaceName Space name. Optional. - * @return Number of elements or {@code -1} if no space with the given name has been found. - */ - public long entriesCount(@Nullable String spaceName) { - GridOffHeapPartitionedMap m = offheap(spaceName); - - return m == null ? -1 : m.size(); - } - - /** - * Gets number of elements in the given space. - * - * @param spaceName Space name. Optional. - * @param parts Partitions. - * @return Number of elements or {@code -1} if no space with the given name has been found. - */ - public long entriesCount(@Nullable String spaceName, Set parts) { - GridOffHeapPartitionedMap m = offheap(spaceName); - - return m == null ? -1 : m.size(parts); - } - - /** - * Gets size of a memory allocated for the entries of the given space. - * - * @param spaceName Space name. Optional. - * @return Allocated memory size or {@code -1} if no space with the given name has been found. - */ - public long allocatedSize(@Nullable String spaceName) { - GridOffHeapPartitionedMap m = offheap(spaceName); - - return m == null ? -1 : m.allocatedSize(); - } - - /** - * Gets iterator over contents of partition. - * - * @param spaceName Space name. - * @param part Partition. - * @return Iterator. - */ - public GridCloseableIterator> iterator(@Nullable String spaceName, int part) { - GridOffHeapPartitionedMap m = offheap(spaceName); - - return m == null ? new GridEmptyCloseableIterator>() : m.iterator(part); - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java index bedbb96bf66c0..5f264a7ec816a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/pool/PoolProcessor.java @@ -145,10 +145,13 @@ public class PoolProcessor extends GridProcessorAdapter { */ public static final long DFLT_PERIODIC_STARVATION_CHECK_FREQ = 30 * 1000L; + /** */ + public static final String POOL_NAME_IDLE_VERIFY = "idleVerify"; + /** */ @SystemProperty(value = "Idle verify utility thread pool size.", type = Integer.class, defaults = "Total available CPUs minus 2, but at least 4 threads.") - public static final String IDLE_VERIFY_POOL_SIZE_PROPERTY = "IGNITE_IDLE_VERIFY_POOL_SIZE"; + private static final String IDLE_VERIFY_POOL_SIZE_PROPERTY = "IGNITE_IDLE_VERIFY_POOL_SIZE"; /** */ private static final int DFLT_IDLE_VERIFY_POOL_SIZE = Math.max(4, IgniteConfiguration.AVAILABLE_PROC_CNT - 2); @@ -395,7 +398,7 @@ public PoolProcessor(GridKernalContext ctx) { validateThreadPoolSize(idleVerifyPoolSz, "idle verify"); idleVerifyExecSvc = new IgniteThreadPoolExecutor( - "idleVerify", + POOL_NAME_IDLE_VERIFY, cfg.getIgniteInstanceName(), idleVerifyPoolSz, idleVerifyPoolSz, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java index e431b8f07f8a8..e4e186fffad72 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/CoreFeatureRegistry.java @@ -112,6 +112,6 @@ public class CoreFeatureRegistry { /** Feature that introduces the snapshot list command. */ public static final IgniteFeature SNAPSHOT_LIST_FEATURE = new IgniteCoreFeature(4); - /** Future that introduces the snaphost check status. */ + /** Feature that introduces statuses of the snapshot check operations. */ public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = new IgniteCoreFeature(5); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapEvictListener.java b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapEvictListener.java deleted file mode 100644 index e7f0962a97593..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapEvictListener.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap; - -/** - * Callback for whenever entries get evicted from off-heap store. - */ -public interface GridOffHeapEvictListener { - /** - * Eviction callback. - * - * @param part Entry partition - * @param hash Hash. - * @param keyBytes Key bytes. - * @param valBytes Value bytes. - */ - public void onEvict(int part, int hash, byte[] keyBytes, byte[] valBytes); - - /** - * @return {@code True} if entry selected for eviction should be immediately removed. - */ - public boolean removeEvicted(); -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapMap.java b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapMap.java deleted file mode 100644 index 837bcc87313b1..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapMap.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap; - -import org.apache.ignite.internal.util.lang.GridCloseableIterator; -import org.apache.ignite.internal.util.typedef.CX2; -import org.apache.ignite.internal.util.typedef.T2; -import org.apache.ignite.lang.IgniteBiPredicate; -import org.apache.ignite.lang.IgniteBiTuple; -import org.jetbrains.annotations.Nullable; - -/** - * Off-heap map. - */ -public interface GridOffHeapMap { - /** - * Gets partition this map belongs to. - * - * @return Partition this map belongs to. - */ - public int partition(); - - /** - * Gets load factor of this map. - * - * @return Load factor. - */ - public float loadFactor(); - - /** - * Gets concurrency level for this map. - * - * @return Concurrency level. - */ - public int concurrency(); - - /** - * Checks if given key is contained in the map. - * - * @param hash Hash. - * @param keyBytes Key bytes. - * @return {@code True} if key is contained in the map. - */ - public boolean contains(int hash, byte[] keyBytes); - - /** - * Gets value bytes for given key. - * - * @param hash Hash. - * @param keyBytes Key bytes. - * @return Value bytes. - */ - @Nullable public byte[] get(int hash, byte[] keyBytes); - - /** - * Gets value pointer for given key. - * - * @param hash Hash. - * @param keyBytes Key bytes. - * @return Value pointer. - */ - @Nullable public IgniteBiTuple valuePointer(int hash, byte[] keyBytes); - - /** - * Enables eviction for given key. - * - * @param hash Hash. - * @param keyBytes Key bytes. - */ - public void enableEviction(int hash, byte[] keyBytes); - - /** - * Removes value from off-heap map. - * - * @param hash Hash. - * @param keyBytes Key bytes. - * @return Removed value bytes. - */ - @Nullable public byte[] remove(int hash, byte[] keyBytes); - - /** - * Removes value from off-heap map without returning it. - * - * @param hash Hash. - * @param keyBytes Key bytes. - * @return {@code True} if value was removed. - */ - public boolean removex(int hash, byte[] keyBytes); - - /** - * Removes value from off-heap map without returning it. - * - * @param hash Hash. - * @param keyBytes Key bytes. - * @param p Value predicate (arguments are value address and value length). - * @return {@code True} if value was removed. - */ - public boolean removex(int hash, byte[] keyBytes, IgniteBiPredicate p); - - /** - * Puts key and value bytes into the map potentially replacing - * existing entry. - * - * @param hash Hash. - * @param keyBytes Key bytes. - * @param valBytes Value bytes. - * @return {@code True} if new entry was created, {@code false} if existing value was updated. - */ - public boolean put(int hash, byte[] keyBytes, byte[] valBytes); - - /** - * Inserts new entry into the map without comparing if there is - * a mapping for given key already stored in map. - *

- * Use with caution whenever certain that inserting a new value - * without current mapping. - * - * @param hash Hash. - * @param keyBytes Key bytes. - * @param valBytes Value bytes. - */ - public void insert(int hash, byte[] keyBytes, byte[] valBytes); - - /** - * Gets number of elements in the map. - * - * @return Number of elements in the map. - */ - public long totalSize(); - - /** - * Gets number of elements in the map. - * - * @return Number of elements in the map. - */ - public long size(); - - /** - * Gets total available memory size. - * - * @return Memory size. - */ - public long memorySize(); - - /** - * Gets memory allocated for map entries so far. - * - * @return Allocated memory. - */ - public long allocatedSize(); - - /** - * Gets memory allocated for map internal structure so far. - * - * @return Allocated memory. - */ - public long systemAllocatedSize(); - - /** - * Gets available memory. - * - * @return Available memory. - */ - public long freeSize(); - - /** - * Destructs this map and deallocates all memory. - */ - public void destruct(); - - /** - * Gets iterator over map. - * - * @return Iterator over map. - */ - public GridCloseableIterator> iterator(); - - /** - * Gets iterator over map. - * - * @param c Key/value closure. - * @return Iterator over map. - */ - public GridCloseableIterator iterator(@Nullable CX2, T2, T> c); - - /** - * Sets callback for when entries are evicted due to memory constraints. - * The parameter into closure is key bytes. - * - * @param lsnr Evict listener. - * @return {@code True} if evict listener was added, {@code false} if - * another listener already exists or {@code LRU} is disabled. - */ - public boolean evictListener(GridOffHeapEvictListener lsnr); - - /** - * Adds off-heap event listener. - * - * @param lsnr Listener. - * @return {@code True} if event listener was added, {@code false} if - * another listener already exists. - */ - public boolean eventListener(GridOffHeapEventListener lsnr); -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapMapFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapMapFactory.java deleted file mode 100644 index aae4705c1dd37..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapMapFactory.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap; - -import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMap; -import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafePartitionedMap; -import org.jetbrains.annotations.Nullable; - -/** - * Factory for off-heap maps. - */ -public class GridOffHeapMapFactory { - /** - * Creates off-heap map based on {@code Unsafe} implementation with - * unlimited memory. - * - * @param initCap Initial capacity. - * @return Off-heap map. - */ - public static GridOffHeapMap unsafeMap(long initCap) { - return new GridUnsafeMap(128, 0.75f, initCap, 0, (short)0, null); - } - - /** - * Creates off-heap map based on {@code Unsafe} implementation. - * - * @param concurrency Concurrency. - * @param initCap Initial capacity. - * @return Off-heap map. - */ - public static GridOffHeapMap unsafeMap(int concurrency, long initCap) { - return new GridUnsafeMap(concurrency, 0.75f, initCap, 0, (short)0, null); - } - - /** - * Creates off-heap map based on {@code Unsafe} implementation. - * - * @param concurrency Concurrency. - * @param load Load factor. - * @param initCap Initial capacity. - * @return Off-heap map. - */ - public static GridOffHeapMap unsafeMap(int concurrency, float load, long initCap) { - return new GridUnsafeMap(concurrency, load, initCap, 0, (short)0, null); - } - - /** - * Creates off-heap map based on {@code Unsafe} implementation with limited - * memory and LRU-based eviction. - * - * @param initCap Initial capacity. - * @param totalMem Total memory. - * @param lruStripes Number of LRU stripes. - * @return Off-heap map. - */ - public static GridOffHeapMap unsafeMap(long initCap, long totalMem, short lruStripes) { - return new GridUnsafeMap(128, 0.75f, initCap, totalMem, lruStripes, null); - } - - /** - * Creates off-heap map based on {@code Unsafe} implementation with limited - * memory and LRU-based eviction. - * - * @param initCap Initial capacity. - * @param totalMem Total memory. - * @param lruStripes Number of LRU stripes. - * @param lsnr Optional eviction listener which gets notified every time an entry is evicted. - * @return Off-heap map. - */ - public static GridOffHeapMap unsafeMap(long initCap, long totalMem, short lruStripes, - @Nullable GridOffHeapEvictListener lsnr) { - return new GridUnsafeMap(128, 0.75f, initCap, totalMem, lruStripes, lsnr); - } - - /** - * Creates off-heap map based on {@code Unsafe} implementation with limited - * memory and LRU-based eviction. - * - * @param concurrency Concurrency. - * @param initCap Initial capacity. - * @param totalMem Total memory. - * @param lruStripes Number of LRU stripes. - * @param lsnr Optional eviction listener which gets notified every time an entry is evicted. - * @return Off-heap map. - */ - public static GridOffHeapMap unsafeMap(int concurrency, long initCap, long totalMem, short lruStripes, - @Nullable GridOffHeapEvictListener lsnr) { - return new GridUnsafeMap(concurrency, 0.75f, initCap, totalMem, lruStripes, lsnr); - } - - /** - * Creates off-heap map based on {@code Unsafe} implementation with limited - * memory and LRU-based eviction. - * - * @param concurrency Concurrency. - * @param load Load factor. - * @param initCap Initial capacity. - * @param totalMem Total memory. - * @param lruStripes Number of LRU stripes. - * @param lsnr Optional eviction listener which gets notified every time an entry is evicted. - * @return Off-heap map. - */ - public static GridOffHeapMap unsafeMap(int concurrency, float load, long initCap, long totalMem, - short lruStripes, @Nullable GridOffHeapEvictListener lsnr) { - return new GridUnsafeMap(concurrency, load, initCap, totalMem, lruStripes, lsnr); - } - - /** - * Creates off-heap map based on {@code Unsafe} implementation with - * unlimited memory. - * - * @param parts Partitions. - * @param initCap Initial capacity. - * @return Off-heap map. - */ - public static GridOffHeapPartitionedMap unsafePartitionedMap(int parts, long initCap) { - return new GridUnsafePartitionedMap(parts, 128, 0.75f, initCap, 0, (short)0, null); - } - - /** - * Creates off-heap map based on {@code Unsafe} implementation. - * - * @param parts Partitions. - * @param concurrency Concurrency. - * @param initCap Initial capacity. - * @return Off-heap map. - */ - public static GridOffHeapPartitionedMap unsafePartitionedMap(int parts, int concurrency, long initCap) { - return new GridUnsafePartitionedMap(parts, concurrency, 0.75f, initCap, 0, (short)0, null); - } - - /** - * Creates off-heap map based on {@code Unsafe} implementation. - * - * @param parts Partitions. - * @param concurrency Concurrency. - * @param load Load factor. - * @param initCap Initial capacity. - * @return Off-heap map. - */ - public static GridOffHeapPartitionedMap unsafePartitionedMap(int parts, int concurrency, float load, - long initCap) { - return new GridUnsafePartitionedMap(parts, concurrency, load, initCap, 0, (short)0, null); - } - - /** - * Creates off-heap map based on {@code Unsafe} implementation with limited - * memory and LRU-based eviction. - * - * @param parts Partitions. - * @param initCap Initial capacity. - * @param totalMem Total memory. - * @param lruStripes Number of LRU stripes. - * @return Off-heap map. - */ - public static GridOffHeapPartitionedMap unsafePartitionedMap(int parts, long initCap, long totalMem, - short lruStripes) { - return new GridUnsafePartitionedMap(parts, 128, 0.75f, initCap, totalMem, lruStripes, null); - } - - /** - * Creates off-heap map based on {@code Unsafe} implementation with limited - * memory and LRU-based eviction. - * - * @param parts Partitions. - * @param initCap Initial capacity. - * @param totalMem Total memory. - * @param lruStripes Number of LRU stripes. - * @param lsnr Optional eviction listener which gets notified every time an entry is evicted. - * @return Off-heap map. - */ - public static GridOffHeapPartitionedMap unsafePartitionedMap(int parts, long initCap, long totalMem, - short lruStripes, @Nullable GridOffHeapEvictListener lsnr) { - return new GridUnsafePartitionedMap(parts, 128, 0.75f, initCap, totalMem, lruStripes, lsnr); - } - - /** - * Creates off-heap map based on {@code Unsafe} implementation with limited - * memory and LRU-based eviction. - * - * @param parts Partitions. - * @param concurrency Concurrency. - * @param initCap Initial capacity. - * @param totalMem Total memory. - * @param lruStripes Number of LRU stripes. - * @param lsnr Optional eviction listener which gets notified every time an entry is evicted. - * @return Off-heap map. - */ - public static GridOffHeapPartitionedMap unsafePartitionedMap(int parts, int concurrency, long initCap, - long totalMem, short lruStripes, @Nullable GridOffHeapEvictListener lsnr) { - return new GridUnsafePartitionedMap(parts, concurrency, 0.75f, initCap, totalMem, lruStripes, lsnr); - } - - /** - * Creates off-heap map based on {@code Unsafe} implementation with limited - * memory and LRU-based eviction. - * - * @param parts Partitions. - * @param concurrency Concurrency. - * @param load Load factor. - * @param initCap Initial capacity. - * @param totalMem Total memory. - * @param lruStripes Number of LRU stripes. - * @param lsnr Optional eviction listener which gets notified every time an entry is evicted. - * @return Off-heap map. - */ - public static GridOffHeapPartitionedMap unsafePartitionedMap(int parts, int concurrency, float load, - long initCap, long totalMem, short lruStripes, @Nullable GridOffHeapEvictListener lsnr) { - return new GridUnsafePartitionedMap(parts, concurrency, load, initCap, totalMem, lruStripes, lsnr); - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapPartitionedMap.java b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapPartitionedMap.java deleted file mode 100644 index f54910e1e14e2..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapPartitionedMap.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap; - -import java.util.Set; -import org.apache.ignite.internal.util.lang.GridCloseableIterator; -import org.apache.ignite.internal.util.typedef.CX2; -import org.apache.ignite.internal.util.typedef.T2; -import org.apache.ignite.lang.IgniteBiPredicate; -import org.apache.ignite.lang.IgniteBiTuple; -import org.jetbrains.annotations.Nullable; - -/** - * Off-heap map. - */ -public interface GridOffHeapPartitionedMap { - /** - * Gets load factor of this map. - * - * @return Load factor. - */ - public float loadFactor(); - - /** - * Gets concurrency level for this map. - * - * @return Concurrency level. - */ - public int concurrency(); - - /** - * Gets number of partitions. - * - * @return Number of partitions. - */ - public int partitions(); - - /** - * Checks if given key is contained in the map. - * - * @param p Partition. - * @param hash Hash. - * @param keyBytes Key bytes. - * @return {@code True} if key is contained in the map. - */ - public boolean contains(int p, int hash, byte[] keyBytes); - - /** - * Gets value bytes for given key. - * - * @param p Partition. - * @param hash Hash. - * @param keyBytes Key bytes. - * @return Value bytes. - */ - @Nullable public byte[] get(int p, int hash, byte[] keyBytes); - - /** - * Gets value pointer for given key. While pointer is in use eviction is - * disabled for corresponding entry. Eviction for entry is enabled when {@link #put} or - * {@link #enableEviction} is called. - * - * @param p Partition. - * @param hash Hash. - * @param keyBytes Key bytes. - * @return Value pointer. - */ - @Nullable public IgniteBiTuple valuePointer(int p, int hash, byte[] keyBytes); - - /** - * Enables eviction for entry. - * - * @param p Partition. - * @param hash Hash. - * @param keyBytes Key bytes. - */ - public void enableEviction(int p, int hash, byte[] keyBytes); - - /** - * Removes value from off-heap map. - * - * @param p Partition. - * @param hash Hash. - * @param keyBytes Key bytes. - * @return Removed value bytes. - */ - @Nullable public byte[] remove(int p, int hash, byte[] keyBytes); - - /** - * Removes value from off-heap map without returning it. - * - * @param p Partition. - * @param hash Hash. - * @param keyBytes Key bytes. - * @return {@code True} if value was removed. - */ - public boolean removex(int p, int hash, byte[] keyBytes); - - /** - * Removes value from off-heap map without returning it. - * - * @param part Partition. - * @param hash Hash. - * @param keyBytes Key bytes. - * @param p Value predicate (arguments are value address and value length). - * @return {@code True} if value was removed. - */ - public boolean removex(int part, int hash, byte[] keyBytes, IgniteBiPredicate p); - - /** - * Puts key and value bytes into the map potentially replacing - * existing entry. - * - * @param p Partition. - * @param hash Hash. - * @param keyBytes Key bytes. - * @param valBytes Value bytes. - * @return {@code True} if new entry was created, {@code false} if existing value was updated. - */ - public boolean put(int p, int hash, byte[] keyBytes, byte[] valBytes); - - /** - * Inserts new entry into the map without comparing if there is - * a mapping for given key already stored in map. - *

- * Use with caution whenever certain that inserting a new value - * without current mapping. - * - * @param p Partition. - * @param hash Hash. - * @param keyBytes Key bytes. - * @param valBytes Value bytes. - */ - public void insert(int p, int hash, byte[] keyBytes, byte[] valBytes); - - /** - * Gets number of elements in the map. - * - * @return Number of elements in the map. - */ - public long size(); - - /** - * Gets number of elements in the map. - * - * @param parts Partitions. - * @return Number of elements in the map. - */ - public long size(Set parts); - - /** - * Gets total available memory size. - * - * @return Memory size. - */ - public long memorySize(); - - /** - * Gets size of a memory allocated for map entries so far. - * - * @return Allocated memory size. - */ - public long allocatedSize(); - - /** - * Gets memory allocated for map internal structure so far. - * - * @return Allocated memory. - */ - public long systemAllocatedSize(); - - /** - * Gets available memory. - * - * @return Available memory. - */ - public long freeSize(); - - /** - * Destructs this map and deallocates all memory. - */ - public void destruct(); - - /** - * Gets iterator over the whole map. - * - * @return Iterator over the whole map. - */ - public GridCloseableIterator> iterator(); - - /** - * Gets iterator over the whole map. - * - * @param c Key/value closure. - * @return Iterator over the whole map. - */ - public GridCloseableIterator iterator(CX2, T2, T> c); - - /** - * Gets iterator over the partition. - * - * @param c Key/value closure. - * @param part Partition. - * @return Iterator over the partition. - */ - public GridCloseableIterator iterator(CX2, T2, T> c, int part); - - /** - * Gets iterator over certain partition. - * - * @param p Partition. - * @return Iterator over certain partition. - */ - public GridCloseableIterator> iterator(int p); - - /** - * Sets callback for when entries are evicted due to memory constraints. - * The parameter into closure is key bytes. - * - * @param lsnr Evict listener. - * @return {@code True} if evict listener was added, {@code false} if - * another listener already exists or {@code LRU} is disabled. - */ - public boolean evictListener(GridOffHeapEvictListener lsnr); - - /** - * Adds off-heap event listener. - * - * @param lsnr Listener. - * @return {@code True} if event listener was added, {@code false} if - * another listener already exists. - */ - public boolean eventListener(GridOffHeapEventListener lsnr); -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLru.java b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLru.java deleted file mode 100644 index 7998421e5216a..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLru.java +++ /dev/null @@ -1,650 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap.unsafe; - -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import org.apache.ignite.internal.util.offheap.GridOffHeapOutOfMemoryException; -import org.apache.ignite.internal.util.tostring.GridToStringExclude; -import org.apache.ignite.internal.util.typedef.internal.S; - -/** - * Striped LRU queue. - */ -@SuppressWarnings("ForLoopReplaceableByForEach") class GridUnsafeLru { - /** Number of stripes. */ - private final short cnt; - - /** Stripes. */ - @GridToStringExclude - private final LruStripe[] lrus; - - /** Unsafe memory. */ - @GridToStringExclude - private final GridUnsafeMemory mem; - - /** Current round-robin add stripe index. */ - private final AtomicInteger addIdx; - - /** Current round-robin remove stripe index. */ - private final AtomicInteger rmvIdx; - - /** Max stripe index count. */ - private final int maxIdxCnt; - - /** Released flag. */ - private AtomicBoolean released = new AtomicBoolean(false); - - /** - * @param cnt Number of stripes. - * @param mem Unsafe memory. - */ - GridUnsafeLru(short cnt, GridUnsafeMemory mem) { - assert cnt > 0; - assert mem != null; - - lrus = new LruStripe[cnt]; - - this.cnt = cnt; - this.mem = mem; - - for (short i = 0; i < cnt; i++) - lrus[i] = new LruStripe(i, mem); - - addIdx = new AtomicInteger(); - rmvIdx = new AtomicInteger(cnt / 2); - - maxIdxCnt = cnt - 1; - } - - /** - * Gets number of stripes. - * - * @return Number of stripes. - */ - short concurrency() { - return cnt; - } - - /** - * @return Total number of entries in all stripes. - */ - long size() { - long sum = 0; - - for (int i = 0; i < lrus.length; i++) - sum += lrus[i].size(); - - return sum; - } - - /** - * @return Total memory consumed by all stripes. - */ - long memorySize() { - long sum = 0; - - for (int i = 0; i < lrus.length; i++) - sum += lrus[i].memorySize(); - - return sum; - } - - /** - * Reads order of LRU stripe for queue node. - * - * @param qAddr Queue node address. - * @return Order of LRU stripe. - */ - short order(long qAddr) { - return LruStripe.order(qAddr, mem); - } - - /** - * Reads partition of entry at queue node address. - * - * @param order Order of LRU stripe. - * @param qAddr Queue node address. - * @return Entry partition. - */ - int partition(short order, long qAddr) { - return lrus[order].partition(qAddr); - } - - /** - * Reads hash code of entry at queue node address. - * - * @param order Order of LRU stripe. - * @param qAddr Queue node address. - * @return Entry hash code. - */ - int hash(short order, long qAddr) { - return lrus[order].hash(qAddr); - } - - /** - * Reads entry address for given queue node. - * - * @param order Order of LRU stripe. - * @param qAddr Queue node address. - * @return Entry address. - */ - long entry(short order, long qAddr) { - return lrus[order].entry(qAddr); - } - - /** - * Adds entry address to LRU queue. - * - * @param part Entry Entry partition. - * @param addr Entry address. - * @param hash Entry hash code. - * @return Queue node address. - * @throws GridOffHeapOutOfMemoryException If failed. - */ - long offer(int part, long addr, int hash) throws GridOffHeapOutOfMemoryException { - return lrus[incrementAndGet(addIdx, maxIdxCnt)].offer(part, addr, hash); - } - - /** - * Marks oldest node from the queue as {@code polling}. - * - * @return Queue node address. - */ - long prePoll() { - int idx = incrementAndGet(rmvIdx, maxIdxCnt); - - // Must try to poll from each LRU. - for (int i = 0; i < lrus.length; i++) { - long qAddr = lrus[(idx + i) % cnt].prePoll(); - - if (qAddr != 0) - return qAddr; - } - - return 0; - } - - /** - * Removes polling node from the queue. - * - * @param qAddr Queue node address. - */ - void poll(long qAddr) { - lrus[LruStripe.order(qAddr, mem)].poll(qAddr); - } - - /** - * Updates entry address at specified queue node address. - * - * @param qAddr Queue address. - * @param addr Entry address. - */ - void touch(long qAddr, long addr) { - lrus[LruStripe.order(qAddr, mem)].touch(qAddr, addr); - } - - /** - * Removes queue node from queue. - * - * @param qAddr Address of queue node. - */ - void remove(long qAddr) { - lrus[LruStripe.order(qAddr, mem)].remove(qAddr); - } - - /** - * Releases memory allocated for this queue. - */ - void destruct() { - if (released.compareAndSet(false, true)) { - for (int i = 0; i < cnt; i++) - lrus[i].destruct(); - } - } - - /** - * Atomically increments the given value by one, re-starting from 0 when the specified maximum is reached. - * - * @param value Value to increment. - * @param max Maximum after reaching which the value is reset to 0. - * @return Incremented value. - */ - private int incrementAndGet(AtomicInteger value, int max) { - while (true) { - int cur = value.get(); - int next = cur == max ? 0 : cur + 1; - - if (value.compareAndSet(cur, next)) - return next; - } - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(GridUnsafeLru.class, this); - } - - /** - * Single LRU stripe. - */ - private static class LruStripe { - /** Size of a queue node. */ - private static final int NODE = 2/*queue-index*/ + 4 /*part*/ + 4 /*hash*/ + 1/*poll-flag*/ + 8 /*previous*/ + - 8/*next*/ + 8/*entry-address*/; - - /** Unsafe memory. */ - private final GridUnsafeMemory mem; - - /** Stripe order. */ - private final short order; - - /** Queue head. */ - private long head; - - /** Queue tail */ - private long tail; - - /** Number of elements in the queue. */ - private volatile long size; - - /** Mutex. */ - private final Lock lock = new ReentrantLock(); - - /** - * @param order Stripe order. - * @param mem Unsafe memory. - */ - private LruStripe(short order, GridUnsafeMemory mem) { - assert order >= 0; - assert mem != null; - - this.order = order; - this.mem = mem; - } - - /** - * Gets stripe order for queue node. - * - * @param qAddr Queue node address. - * @param mem Unsafe memory. - * @return Stripe order. - */ - static short order(long qAddr, GridUnsafeMemory mem) { - return mem.readShort(qAddr); - } - - /** - * @return Stripe order. - */ - int order() { - return order; - } - - /** - * @return Number if entries in queue. - */ - long size() { - return size; - } - - /** - * @return Memory size. - */ - long memorySize() { - return size * NODE; - } - - /** - * Releases memory allocated for this queue stripe. - */ - void destruct() { - lock.lock(); - - try { - for (long n = head, prev = 0; n != 0; prev = n, n = next(n)) - mem.releaseSystem(prev, NODE); - } - finally { - lock.unlock(); - } - } - - /** - * Adds entry address to LRU queue. - * - * @param part Entry partition. - * @param addr Entry address. - * @param hash Entry hash code. - * @return Queue node address. - * @throws GridOffHeapOutOfMemoryException If failed. - */ - long offer(int part, long addr, int hash) throws GridOffHeapOutOfMemoryException { - lock.lock(); - - try { - long qAddr = mem.allocateSystem(NODE, false); - - if (head == 0) - head = qAddr; - - long prev = tail; - - tail = qAddr; - - if (prev != 0) - next(prev, qAddr); - - order(qAddr); - partition(qAddr, part); - polling(qAddr, false); - hash(qAddr, hash); - entry(qAddr, addr); - previous(qAddr, prev); - next(qAddr, 0L); - - size++; - - return qAddr; - } - finally { - lock.unlock(); - } - } - - /** - * Polls oldest entry from the queue. - * - * @return Queue node address. - */ - long prePoll() { - lock.lock(); - - try { - long n = head; - - while (n != 0) { - if (!polling(n)) { - // Mark as polling, but do not remove. - // Node will be removed by explicitly calling remove. - polling(n, true); - - break; - } - - n = next(n); - } - - return n; - } - finally { - lock.unlock(); - } - } - - /** - * Updates entry address at specified queue node address. - * - * @param qAddr Queue address. - * @param addr Entry address. - */ - void touch(long qAddr, long addr) { - lock.lock(); - - try { - entry(qAddr, addr); - - if (qAddr != tail) { - long prev = previous(qAddr); - long next = next(qAddr); - - if (prev != 0) - next(prev, next); - else { - assert qAddr == head; - - head = next; - } - - if (next != 0) - previous(next, prev); - - next(tail, qAddr); - next(qAddr, 0); - previous(qAddr, tail); - - tail = qAddr; - } - } - finally { - lock.unlock(); - } - } - - /** - * Removes queue node from queue. - * - * @param qAddr Address of queue node. - */ - void poll(long qAddr) { - lock.lock(); - - try { - assert polling(qAddr); - - unlink(qAddr); - } - finally { - lock.unlock(); - } - } - - /** - * Removes queue node from queue. - * - * @param qAddr Address of queue node. - */ - void remove(long qAddr) { - lock.lock(); - - try { - // Don't remove polling entries (poll operation will remove them). - if (!polling(qAddr)) - unlink(qAddr); - else - // Update entry address in node to 0. - entry(qAddr, 0); - } - finally { - lock.unlock(); - } - } - - /** - * Unlinks and releases queue node. - * - * @param qAddr Queue node address. - */ - private void unlink(long qAddr) { - assert head != 0 && tail != 0; - - long prev = 0; - long next = next(qAddr); - - if (head == qAddr) - head = next; - else { - prev = previous(qAddr); - - assert prev != 0 : "Invalid previous link for stripe: " + order; - - next(prev, next); - } - - if (next != 0) - previous(next, prev); - else { - assert qAddr == tail; - - tail = prev; - } - - mem.releaseSystem(qAddr, NODE); - - size--; - - assert head != 0 || (tail == 0 && size == 0); - } - - /** - * Writes queue order. - * - * @param qAddr Queue node address. - */ - private void order(long qAddr) { - mem.writeShort(qAddr, order); - } - - /** - * Reads partition entry belongs to. - * - * @param qAddr Queue node address. - * @return Entry partition. - */ - private int partition(long qAddr) { - return mem.readInt(qAddr + 2); - } - - /** - * Writes partition entry belongs to. - * - * @param qAddr Queue node address. - * @param part Entry partition. - */ - private void partition(long qAddr, int part) { - mem.writeInt(qAddr + 2, part); - } - - /** - * Reads entry hash code. - * - * @param qAddr Queue node address. - * @return Entry hash code. - */ - private int hash(long qAddr) { - return mem.readInt(qAddr + 6); - } - - /** - * Writes entry hash code. - * - * @param qAddr Queue node address. - * @param hash Entry hash code. - */ - private void hash(long qAddr, int hash) { - mem.writeInt(qAddr + 6, hash); - } - - /** - * Checks if queue node is being polled. - * - * @param qAddr Queue node address. - * @return {@code True} if queue node is being polled. - */ - private boolean polling(long qAddr) { - return mem.readByte(qAddr + 10) == 1; - } - - /** - * Mark entry as being polled or not. - * - * @param qAddr Queue node address. - * @param polling Polling flag. - */ - private void polling(long qAddr, boolean polling) { - mem.writeByte(qAddr + 10, (byte)(polling ? 1 : 0)); - } - - /** - * Reads address of previous queue node. - * - * @param qAddr Queue node address. - * @return Address of previous queue node. - */ - private long previous(long qAddr) { - return mem.readLong(qAddr + 11); - } - - /** - * Writes address of previous queue node. - * - * @param qAddr Queue node address. - * @param prev Address of previous node. - */ - private void previous(long qAddr, long prev) { - mem.writeLong(qAddr + 11, prev); - } - - /** - * Reads address of next queue node. - * - * @param qAddr Queue node address. - * @return Address of next queue node. - */ - private long next(long qAddr) { - return mem.readLong(qAddr + 19); - } - - /** - * Writes address of next queue node. - * - * @param qAddr Queue node address. - * @param next Address of next node. - */ - private void next(long qAddr, long next) { - mem.writeLong(qAddr + 19, next); - } - - /** - * Reads address of entry. - * - * @param qAddr Queue node address. - * @return Address of entry. - */ - private long entry(long qAddr) { - return mem.readLong(qAddr + 27); - } - - /** - * Writes address of entry. - * - * @param qAddr Queue node address. - * @param addr Address of entry. - */ - private void entry(long qAddr, long addr) { - mem.writeLong(qAddr + 27, addr); - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(LruStripe.class, this); - } - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMap.java b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMap.java deleted file mode 100644 index 4c05d360241a8..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMap.java +++ /dev/null @@ -1,1814 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap.unsafe; - -import java.util.LinkedList; -import java.util.NoSuchElementException; -import java.util.Queue; -import java.util.concurrent.atomic.LongAdder; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.internal.util.GridCloseableIteratorAdapter; -import org.apache.ignite.internal.util.lang.GridCloseableIterator; -import org.apache.ignite.internal.util.offheap.GridOffHeapEventListener; -import org.apache.ignite.internal.util.offheap.GridOffHeapEvictListener; -import org.apache.ignite.internal.util.offheap.GridOffHeapMap; -import org.apache.ignite.internal.util.offheap.GridOffHeapOutOfMemoryException; -import org.apache.ignite.internal.util.typedef.CX2; -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.T2; -import org.apache.ignite.internal.util.typedef.X; -import org.apache.ignite.lang.IgniteBiPredicate; -import org.apache.ignite.lang.IgniteBiTuple; -import org.jetbrains.annotations.Nullable; - -import static org.apache.ignite.internal.util.offheap.GridOffHeapEvent.REHASH; - -/** - * Off-heap map based on {@code Unsafe} implementation. - */ -public class GridUnsafeMap implements GridOffHeapMap { - /** Header size. */ - private static final int HEADER_SIZE = 8 /*queue-address*/ + 8 /*next-address*/ + 4 /*hash*/ + 4 /*key-size*/ - + 4 /*value-size*/; - - /** Header queue address offset. */ - private static final long HEADER_QUEUE_ADDR_OFF = 0; - - /** Header next address offset. */ - private static final long HEADER_NEXT_ADDR_OFF = 8; - - /** Header hash offset. */ - private static final long HEADER_HASH_OFF = 16; - - /** Header key size offset. */ - private static final long HEADER_KEY_SIZE_OFF = 20; - - /** Header value size. */ - private static final long HEADER_VALUE_SIZE = 24; - - /** Debug flag. */ - private static final boolean DEBUG = false; - - /** */ - private static final int MAX_CONCURRENCY = 512; - - /** */ - private static final int MIN_SIZE = 16; - - /** - * The maximum capacity, used if a higher value is implicitly - * specified by either of the constructors with arguments. MUST - * be a power of two <= 1<<30 to ensure that entries are indexable - * using ints. - */ - private static final int MAXIMUM_CAPACITY = 1 << 30; - - /** Empty byte array. */ - private static final byte[] EMPTY_BYTES = new byte[0]; - - /** Partition this map belongs to. */ - private final int part; - - /** Concurrency. */ - private final int concurrency; - - /** Load factor. */ - private final float load; - - /** Segments. */ - private final Segment[] segs; - - /** Total memory. */ - private final GridUnsafeMemory mem; - - /** - * Mask value for indexing into segments. The upper bits of a - * key's hash code are used to choose the segment. - */ - private final int segmentMask; - - /** - * Shift value for indexing within segments. - */ - private final int segmentShift; - - /** Evict closure. */ - private GridOffHeapEvictListener evictLsnr; - - /** Striped LRU policy. */ - private final GridUnsafeLru lru; - - /** Total entry count. */ - private final LongAdder totalCnt; - - /** Event listener. */ - private GridOffHeapEventListener evtLsnr; - - /** Flag indicating whether this class owns LRU. */ - private final boolean lruRelease; - - /** LRU poller. */ - private final GridUnsafeLruPoller lruPoller; - - /** */ - private final boolean rmvEvicted; - - /** - * @param concurrency Concurrency. - * @param load Load factor. - * @param initCap Initial capacity. - * @param totalMem Total memory. - * @param lruStripes Number of LRU stripes. - * @param evictLsnr Eviction listener. - */ - public GridUnsafeMap(int concurrency, float load, long initCap, long totalMem, short lruStripes, - @Nullable GridOffHeapEvictListener evictLsnr) { - this.concurrency = concurrency; - this.load = load; - - part = 0; - - mem = new GridUnsafeMemory(totalMem); - - lru = totalMem > 0 ? new GridUnsafeLru(lruStripes, mem) : null; - - lruRelease = true; - - if (lru != null) - this.evictLsnr = evictLsnr; - - totalCnt = new LongAdder(); - - // Find power-of-two sizes best matching arguments - int shift = 0; - int size = 1; - - while (size < concurrency) { - ++shift; - - size <<= 1; - } - - segmentShift = 32 - shift; - segmentMask = size - 1; - - segs = new Segment[size]; - - init(initCap, size); - - lruPoller = new GridUnsafeLruPoller() { - @Override public void lruPoll(int size) { - if (lru == null) - return; - - int left = size; - - while (left > 0) { - // Pre-poll outside of lock. - long qAddr = lru.prePoll(); - - if (qAddr == 0) - return; // LRU is empty. - - short order = lru.order(qAddr); - - int released = freeSpace(order, qAddr); - - if (released == 0) - return; - - left -= released; - } - } - }; - - rmvEvicted = evictLsnr == null || evictLsnr.removeEvicted(); - } - - /** - * @param part Partition number. - * @param concurrency Concurrency. - * @param load Load factor. - * @param initCap Initial capacity. - * @param totalCnt Total count. - * @param mem Memory. - * @param lru LRU. - * @param evictLsnr Eviction closure. - * @param lruPoller LRU poller. - */ - GridUnsafeMap(int part, int concurrency, float load, long initCap, LongAdder totalCnt, GridUnsafeMemory mem, - GridUnsafeLru lru, @Nullable GridOffHeapEvictListener evictLsnr, GridUnsafeLruPoller lruPoller) { - this.part = part; - this.concurrency = concurrency > MAX_CONCURRENCY ? MAX_CONCURRENCY : concurrency; - this.load = load; - this.totalCnt = totalCnt; - this.mem = mem; - this.lru = lru; - this.lruPoller = lruPoller; - - if (lru != null) - this.evictLsnr = evictLsnr; - - lruRelease = false; - - // Find power-of-two sizes best matching arguments - int shift = 0; - int size = 1; - - while (size < this.concurrency) { - ++shift; - - size <<= 1; - } - - segmentShift = 32 - shift; - segmentMask = size - 1; - - segs = new Segment[size]; - - init(initCap, size); - - rmvEvicted = evictLsnr == null || evictLsnr.removeEvicted(); - } - - /** - * @param initCap Initial capacity. - * @param size Size. - */ - private void init(long initCap, int size) { - long c = initCap / size; - - if (c < MIN_SIZE) - c = MIN_SIZE; - - if (c * size < initCap) - ++c; - - int cap = 1; - - while (cap < c) - cap <<= 1; - - for (int i = 0; i < size; i++) { - try { - segs[i] = new Segment(i, cap); - } - catch (GridOffHeapOutOfMemoryException e) { - destruct(); - - throw e; - } - } - } - - /** {@inheritDoc} */ - @Override public boolean eventListener(GridOffHeapEventListener evtLsnr) { - if (this.evtLsnr != null) - return false; - - this.evtLsnr = evtLsnr; - - mem.listen(evtLsnr); - - return true; - } - - /** {@inheritDoc} */ - @Override public boolean evictListener(GridOffHeapEvictListener evictLsnr) { - if (this.evictLsnr != null || lru == null) - return false; - - this.evictLsnr = evictLsnr; - - return true; - } - - /** {@inheritDoc} */ - @Override public int partition() { - return part; - } - - /** {@inheritDoc} */ - @Override public float loadFactor() { - return load; - } - - /** {@inheritDoc} */ - @Override public int concurrency() { - return concurrency; - } - - /** {@inheritDoc} */ - @Override public boolean contains(int hash, byte[] keyBytes) { - return segmentFor(hash).contains(hash, keyBytes); - } - - /** {@inheritDoc} */ - @Override public byte[] get(int hash, byte[] keyBytes) { - return segmentFor(hash).get(hash, keyBytes); - } - - /** {@inheritDoc} */ - @Nullable @Override public IgniteBiTuple valuePointer(int hash, byte[] keyBytes) { - return segmentFor(hash).valuePointer(hash, keyBytes); - } - - /** {@inheritDoc} */ - @Override public void enableEviction(int hash, byte[] keyBytes) { - assert lru != null; - - segmentFor(hash).enableEviction(hash, keyBytes); - } - - /** {@inheritDoc} */ - @Override public byte[] remove(int hash, byte[] keyBytes) { - return segmentFor(hash).remove(hash, keyBytes); - } - - /** {@inheritDoc} */ - @Override public boolean removex(int hash, byte[] keyBytes) { - return segmentFor(hash).removex(hash, keyBytes); - } - - /** {@inheritDoc} */ - @Override public boolean removex(int hash, byte[] keyBytes, IgniteBiPredicate p) { - return segmentFor(hash).removex(hash, keyBytes, p); - } - - /** {@inheritDoc} */ - @Override public boolean put(int hash, byte[] keyBytes, byte[] valBytes) { - return segmentFor(hash).put(hash, keyBytes, valBytes); - } - - /** {@inheritDoc} */ - @Override public void insert(int hash, byte[] keyBytes, byte[] valBytes) { - segmentFor(hash).insert(hash, keyBytes, valBytes); - } - - /** {@inheritDoc} */ - @Override public long totalSize() { - return totalCnt.sum(); - } - - /** {@inheritDoc} */ - @Override public long size() { - long size = 0; - - for (int i = 0; i < segs.length; i++) - size += segs[i].count(); - - return size; - } - - /** {@inheritDoc} */ - @Override public long memorySize() { - return mem.totalSize(); - } - - /** {@inheritDoc} */ - @Override public long allocatedSize() { - return mem.allocatedSize(); - } - - /** {@inheritDoc} */ - @Override public long systemAllocatedSize() { - return mem.systemAllocatedSize(); - } - - /** {@inheritDoc} */ - @Override public long freeSize() { - return mem.freeSize(); - } - - /** {@inheritDoc} */ - @Override public void destruct() { - for (Segment seg : segs) { - if (seg != null) - seg.destruct(); - } - - if (lru != null && lruRelease) - lru.destruct(); - } - - /** {@inheritDoc} */ - @Override public GridCloseableIterator> iterator() { - return new GridCloseableIteratorAdapter>() { - private GridCloseableIterator> curIt; - - private int idx; - - { - try { - advance(); - } - catch (IgniteCheckedException e) { - e.printStackTrace(); // Should never happen. - } - } - - private void advance() throws IgniteCheckedException { - curIt = null; - - while (idx < segs.length) { - curIt = segs[idx++].iterator(); - - if (curIt.hasNext()) - return; - else - curIt.close(); - } - - curIt = null; - } - - @Override protected IgniteBiTuple onNext() throws IgniteCheckedException { - if (curIt == null) - throw new NoSuchElementException(); - - IgniteBiTuple t = curIt.next(); - - if (!curIt.hasNext()) { - curIt.close(); - - advance(); - } - - return t; - } - - @Override protected boolean onHasNext() { - return curIt != null; - } - - @Override protected void onRemove() { - throw new UnsupportedOperationException(); - } - - @Override protected void onClose() throws IgniteCheckedException { - if (curIt != null) - curIt.close(); - } - }; - } - - /** {@inheritDoc} */ - @Override public GridCloseableIterator iterator(final CX2, T2, T> c) { - return new GridCloseableIteratorAdapter() { - private GridCloseableIterator curIt; - - private int idx; - - { - try { - advance(); - } - catch (IgniteCheckedException e) { - e.printStackTrace(); // Should never happen. - } - } - - private void advance() throws IgniteCheckedException { - curIt = null; - - while (idx < segs.length) { - curIt = segs[idx++].iterator(c); - - if (curIt.hasNext()) - return; - else - curIt.close(); - } - - curIt = null; - } - - @Override protected T onNext() throws IgniteCheckedException { - if (curIt == null) - throw new NoSuchElementException(); - - T t = curIt.next(); - - if (!curIt.hasNext()) { - curIt.close(); - - advance(); - } - - return t; - } - - @Override protected boolean onHasNext() { - return curIt != null; - } - - @Override protected void onRemove() { - throw new UnsupportedOperationException(); - } - - @Override protected void onClose() throws IgniteCheckedException { - if (curIt != null) - curIt.close(); - } - }; - } - - /** - * Gets number of LRU stripes. - * - * @return Number of LRU stripes. - */ - public short lruStripes() { - return lru.concurrency(); - } - - /** - * Gets memory size occupied by LRU queue. - * - * @return Memory size occupied by LRU queue. - */ - public long lruMemorySize() { - return lru.memorySize(); - } - - /** - * Gets number of elements in LRU queue. - * - * @return Number of elements in LRU queue. - */ - public long lruSize() { - return lru.size(); - } - - /** - * Returns the segment that should be used for key with given hash - * @param hash the hash code for the key - * @return the segment - */ - private Segment segmentFor(int hash) { - return segs[(hash >>> segmentShift) & segmentMask]; - } - - /** - * Frees space by polling entries from LRU queue. - * - * @param order Queue stripe order. - * @param qAddr Queue node address. - * @return Released size. - */ - int freeSpace(short order, long qAddr) { - if (lru == null) - return 0; - - int hash = lru.hash(order, qAddr); - - return segmentFor(hash).freeSpace(hash, order, qAddr); - } - - /** - * Segment. - */ - private class Segment { - /** Lock. */ - private final ReadWriteLock lock = new ReentrantReadWriteLock(); - - /** Segment index. */ - private final int idx; - - /** Capacity. */ - private volatile long cap; - - /** Memory capacity. */ - private volatile long memCap; - - /** Count. */ - private volatile long cnt; - - /** Table pointer. */ - private volatile long tblAddr; - - /** Threshold. */ - private long threshold; - - /** - * @param idx Segment index. - * @param cap Capacity. - */ - private Segment(int idx, long cap) { - this.idx = idx; - this.cap = cap; - - threshold = (long)(cap * load); - - memCap = cap * 8; - - tblAddr = mem.allocateSystem(memCap, true); - } - - /** - * @return Index ID. - */ - int id() { - return idx; - } - - /** - * @return Table pointer. - */ - long tableAddress() { - return tblAddr; - } - - /** - * @return Capacity. - */ - long capacity() { - return cap; - } - - /** - * @return Load factor. - */ - float loadFactor() { - return load; - } - - /** - * @return Count of entries in the segment. - */ - long count() { - return cnt; - } - - /** - * @param hash Hash. - * @param cap Capacity. - * @return Bin index. - */ - long binIndex(int hash, long cap) { - assert Long.bitCount(cap) == 1; - - return hash & (cap - 1); - } - - /** - * @param hash Hash. - * @return Memory address for the bin. - */ - long binAddress(int hash) { - return binAddress(hash, tblAddr, cap); - } - - /** - * @param hash Hash. - * @param tblPtr Table pointer. - * @param cap Capacity. - * @return Bin address. - */ - long binAddress(int hash, long tblPtr, long cap) { - return tblPtr + binIndex(hash, cap) * 8; - } - - /** - * Acquires write lock and returns bin address for given hash code. - * - * @param hash Hash code. - * @return Locked bin address. - */ - @SuppressWarnings("LockAcquiredButNotSafelyReleased") - private long writeLock(int hash) { - lock.writeLock().lock(); - - // Get bin address inside the lock. - return binAddress(hash); - } - - /** - * Unlocks bin address. - */ - private void writeUnlock() { - lock.writeLock().unlock(); - } - - /** - * Acquires read lock abd returns bin address for given hash code. - * - * @param hash Hash code. - * @return Locked bin address. - */ - @SuppressWarnings("LockAcquiredButNotSafelyReleased") - private long readLock(int hash) { - lock.readLock().lock(); - - // Get bin address inside the lock. - return binAddress(hash); - } - - /** - * Unlocks bin address. - */ - private void readUnlock() { - lock.readLock().unlock(); - } - - /** - * Releases allocated table. - */ - void destruct() { - lock.writeLock().lock(); - - try { - if (tblAddr == 0) - return; - - for (long binAddr = tblAddr, tblEnd = (tblAddr + memCap); binAddr < tblEnd; binAddr += 8) { - long entryAddr = Bin.first(binAddr, mem); - - if (entryAddr == 0) - continue; - - while (true) { - long next = Entry.nextAddress(entryAddr, mem); - - mem.release(entryAddr, Entry.size(entryAddr, mem)); - - if (next == 0) - break; - else - entryAddr = next; - } - } - - mem.releaseSystem(tblAddr, memCap); - } - finally { - tblAddr = 0; - - lock.writeLock().unlock(); - } - } - - /** - * @return Iterator. - */ - GridCloseableIterator> iterator() { - return new GridCloseableIteratorAdapter>() { - private final Queue> bin = new LinkedList<>(); - - { - lock.readLock().lock(); - - try { - advance(); - } - finally { - lock.readLock().unlock(); - } - } - - private void advance() { - assert bin.isEmpty(); - - long tblEnd = tblAddr + memCap; - - for (long binAddr = tblAddr; binAddr < tblEnd; binAddr += 8) { - long entryAddr = Bin.first(binAddr, mem); - - if (entryAddr == 0) - continue; - - while (entryAddr != 0) { - // Read key and value bytes. - // TODO: GG-8123: Inlined as a workaround. Revert when 7u60 is released. -// bin.add(F.t(Entry.readKeyBytes(entryAddr, mem), Entry.readValueBytes(entryAddr, mem))); - { - int keyLen = Entry.readKeyLength(entryAddr, mem); - int valLen = Entry.readValueLength(entryAddr, mem); - - byte[] valBytes = mem.readBytes(entryAddr + HEADER_SIZE + keyLen, valLen); - - bin.add(F.t(Entry.readKeyBytes(entryAddr, mem), valBytes)); - } - - entryAddr = Entry.nextAddress(entryAddr, mem); - } - } - } - - @Override protected boolean onHasNext() { - return !bin.isEmpty(); - } - - @Override protected IgniteBiTuple onNext() { - IgniteBiTuple t = bin.poll(); - - if (t == null) - throw new NoSuchElementException(); - - return t; - } - - @Override protected void onRemove() { - throw new UnsupportedOperationException(); - } - - @Override protected void onClose() { - // No-op. - } - }; - } - - /** - * @param c Key/value closure. - * @return Iterator. - */ - GridCloseableIterator iterator(final CX2, T2, T> c) { - return new GridCloseableIteratorAdapter() { - private final Queue bin = new LinkedList<>(); - - { - lock.readLock().lock(); - - try { - advance(); - } - finally { - lock.readLock().unlock(); - } - } - - private void advance() { - assert bin.isEmpty(); - - long tblEnd = tblAddr + memCap; - - for (long binAddr = tblAddr; binAddr < tblEnd; binAddr += 8) { - long entryAddr = Bin.first(binAddr, mem); - - if (entryAddr == 0) - continue; - - while (entryAddr != 0) { - // Read key and value bytes. - // TODO: GG-8123: Inlined as a workaround. Revert when 7u60 is released. -// bin.add(F.t(Entry.readKeyBytes(entryAddr, mem), Entry.readValueBytes(entryAddr, mem))); - { - int keyLen = Entry.readKeyLength(entryAddr, mem); - int valLen = Entry.readValueLength(entryAddr, mem); - - T2 keyPtr = new T2<>(entryAddr + HEADER_SIZE, keyLen); - - T2 valPtr = new T2<>(entryAddr + HEADER_SIZE + keyLen, valLen); - - T res = c.apply(keyPtr, valPtr); - - if (res != null) - bin.add(res); - } - - entryAddr = Entry.nextAddress(entryAddr, mem); - } - } - } - - @Override protected boolean onHasNext() { - return !bin.isEmpty(); - } - - @Override protected T onNext() { - T t = bin.poll(); - - if (t == null) - throw new NoSuchElementException(); - - return t; - } - - @Override protected void onRemove() { - throw new UnsupportedOperationException(); - } - - @Override protected void onClose() { - // No-op. - } - }; - } - - /** - * Rehashes this segment. - */ - @SuppressWarnings("TooBroadScope") - private void rehash() { - if (cnt >= MAXIMUM_CAPACITY || cnt <= threshold) - return; - - boolean release = false; - - long oldTblAddr = -1; - long oldMemCap = -1; - - lock.writeLock().lock(); - - try { - // Read values inside the lock. - long oldCap = cap; - oldMemCap = memCap; - oldTblAddr = tblAddr; - - if (cnt >= MAXIMUM_CAPACITY || cnt <= threshold) - return; - - long newCap = oldCap << 1; - long newMemCap = newCap * 8; - - if (DEBUG) - X.println("Rehashing [size=" + totalCnt.sum() + ", segIdx=" + idx + ", oldCap=" + oldCap + - ", oldMemCap=" + oldMemCap + ", newCap=" + newCap + ", newMemCap=" + newMemCap + ']'); - - // Allocate new memory. - long newTblAddr = mem.allocateSystem(newMemCap, true); - - long oldTblEnd = oldTblAddr + memCap; - - for (long oldBinAddr = oldTblAddr; oldBinAddr < oldTblEnd; oldBinAddr += 8) { - long entryAddr = Bin.first(oldBinAddr, mem); - - if (entryAddr == 0) - continue; - - while (true) { - int hash = Entry.hash(entryAddr, mem); - long next = Entry.nextAddress(entryAddr, mem); - - long newBinAddr = binAddress(hash, newTblAddr, newCap); - - long newFirst = Bin.first(newBinAddr, mem); - - Bin.first(newBinAddr, entryAddr, mem); - - Entry.nextAddress(entryAddr, newFirst, mem); - - if (next == 0) - break; - else - entryAddr = next; - } - } - - tblAddr = newTblAddr; - memCap = newMemCap; - cap = newCap; - threshold = (long)(newCap * load); - release = true; - - if (evtLsnr != null) - evtLsnr.onEvent(REHASH); - } - finally { - lock.writeLock().unlock(); - - // Release allocated memory outside of lock. - if (release) { - assert oldTblAddr != tblAddr; - - assert oldTblAddr != -1; - assert oldMemCap != -1; - - mem.releaseSystem(oldTblAddr, oldMemCap); - } - } - } - - /** - * Frees space by polling entries from LRU queue. - * - * @param hash Hash code. - * @param order Queue stripe order. - * @param qAddr Queue address. - * @return Released size. - */ - @SuppressWarnings({"AssertWithSideEffects"}) - private int freeSpace(int hash, short order, long qAddr) { - assert lru != null; - - byte[] keyBytes = null; - byte[] valBytes = null; - - int relSize = 0; - long relAddr = 0; - - long binAddr = writeLock(hash); - - try { - // Read LRU queue node inside of the lock. - long addr = lru.entry(order, qAddr); - - if (addr != 0) { - long first = Bin.first(binAddr, mem); - - if (first != 0) { - long prev = 0; - long cur = first; - - // Find the address to poll. - while (cur != addr && cur != 0) { - prev = cur; - - cur = Entry.nextAddress(cur, mem); - } - - if (cur != 0) { - long qAddr0 = Entry.queueAddress(cur, mem); - - assert qAddr == qAddr0 : "Queue node address mismatch " + - "[qAddr=" + qAddr + ", entryQueueAddr=" + qAddr + ']'; - - if (evictLsnr != null) { - keyBytes = Entry.readKeyBytes(cur, mem); - - int keyLen = Entry.readKeyLength(cur, mem); - int valLen = Entry.readValueLength(cur, mem); - - valBytes = mem.readBytes(cur + HEADER_SIZE + keyLen, valLen); - } - - if (rmvEvicted) { - long a; - - assert qAddr == (a = Entry.queueAddress(cur, mem)) : "Queue node address mismatch " + - "[qAddr=" + qAddr + ", entryQueueAddr=" + a + ']'; - - long next = Entry.nextAddress(cur, mem); - - if (prev != 0) - Entry.nextAddress(prev, next, mem); // Relink. - else { - if (next == 0) - Bin.clear(binAddr, mem); - else - Bin.first(binAddr, next, mem); - } - - relSize = Entry.size(cur, mem); - relAddr = cur; - - cnt--; - - totalCnt.decrement(); - } - else { - boolean clear = Entry.clearQueueAddress(cur, qAddr, mem); - - assert clear; - - relSize = Entry.size(cur, mem); - } - } - } - } - - // Remove from LRU. - lru.poll(qAddr); - } - finally { - writeUnlock(); - - // Remove current mapping outside of lock. - mem.release(relAddr, relSize); - } - - // Notify eviction. - if (keyBytes != null) { - assert evictLsnr != null; - - evictLsnr.onEvict(part, hash, keyBytes, valBytes); - } - - return relSize; - } - - /** - * @param hash Hash. - * @param keyBytes Key bytes. - * @param valBytes Value bytes. - */ - void insert(int hash, byte[] keyBytes, byte[] valBytes) { - if (cnt + 1 > threshold) - rehash(); - - int size = HEADER_SIZE + keyBytes.length + valBytes.length; - - boolean poll = !mem.reserve(size); - - // Allocate outside of lock. - long addr = mem.allocate(size, false, true); - - // Write as much as possible outside of lock. - Entry.write(addr, hash, keyBytes, valBytes, mem); - - long binAddr = writeLock(hash); - - try { - long first = Bin.first(binAddr, mem); - - Entry.nextAddress(addr, first, mem); - - Bin.first(binAddr, addr, mem); - - // lru.offer can throw GridOffHeapOutOfMemoryException. - long qAddr = lru == null ? 0 : lru.offer(part, addr, hash); - - Entry.queueAddress(addr, qAddr, mem); - - cnt++; - - totalCnt.increment(); - } - catch (GridOffHeapOutOfMemoryException e) { - mem.release(addr, size); - - throw e; - } - finally { - writeUnlock(); - - if (poll) - lruPoller.lruPoll(size); - - if (cnt > threshold) - rehash(); - } - } - - /** - * @param hash Hash. - * @param keyBytes Key bytes. - * @param valBytes Value bytes. - * @return {@code True} if new entry was created, {@code false} if existing value was updated. - */ - boolean put(int hash, byte[] keyBytes, byte[] valBytes) { - boolean isNew = true; - - boolean poll = false; - - int size = 0; - - int relSize = 0; - long relAddr = 0; - - long binAddr = writeLock(hash); - - try { - long first = Bin.first(binAddr, mem); - - long qAddr = 0; - - if (first != 0) { - long prev = 0; - long cur = first; - - while (true) { - long next = Entry.nextAddress(cur, mem); - - // If found match. - if (Entry.keyEquals(cur, keyBytes, mem)) { - // If value bytes have the same length, just update the value. - if (Entry.readValueLength(cur, mem) == valBytes.length) { - Entry.writeValueBytes(cur, valBytes, mem); - - isNew = false; - - if (lru != null) { - qAddr = Entry.queueAddress(cur, mem); - - if (qAddr == 0) { - qAddr = lru.offer(part, cur, hash); - - Entry.queueAddress(cur, qAddr, mem); - } - else - lru.touch(qAddr, cur); - } - - return false; - } - - if (prev != 0) - Entry.nextAddress(prev, next, mem); // Unlink. - else - first = next; - - qAddr = Entry.queueAddress(cur, mem); - - if (qAddr == 0 && lru != null) { - qAddr = lru.offer(part, cur, hash); - - Entry.queueAddress(cur, qAddr, mem); - } - - // Prepare release of memory. - relSize = Entry.size(cur, mem); - relAddr = cur; - - isNew = false; - - break; - } - - prev = cur; - cur = next; - - // If end of linked list. - if (next == 0) - break; - } - } - - size = HEADER_SIZE + keyBytes.length + valBytes.length; - - poll = !mem.reserve(size); - - long addr = mem.allocate(size, false, true); - - Bin.first(binAddr, addr, mem); - - if (isNew) { - cnt++; - - totalCnt.increment(); - - qAddr = lru == null ? 0 : lru.offer(part, addr, hash); - } - else if (lru != null) - lru.touch(qAddr, addr); - - Entry.write(addr, hash, keyBytes, valBytes, qAddr, first, mem); - - return isNew; - } - finally { - writeUnlock(); - - // Release memory outside of lock. - if (relAddr != 0) - mem.release(relAddr, relSize); - - if (poll) - lruPoller.lruPoll(size); - - if (isNew && cnt > threshold) - rehash(); - } - } - - /** - * @param hash Hash. - * @param keyBytes Key bytes. - * @return Removed value bytes. - */ - byte[] remove(int hash, byte[] keyBytes) { - return remove(hash, keyBytes, true, null); - } - - /** - * @param hash Hash. - * @param keyBytes Key bytes. - * @return {@code True} if value was removed. - */ - boolean removex(int hash, byte[] keyBytes) { - return remove(hash, keyBytes, false, null) == EMPTY_BYTES; - } - - /** - * @param hash Hash. - * @param keyBytes Key bytes. - * @param p Value predicate. - * @return {@code True} if value was removed. - */ - boolean removex(int hash, byte[] keyBytes, IgniteBiPredicate p) { - return remove(hash, keyBytes, false, p) == EMPTY_BYTES; - } - - /** - * @param hash Hash. - * @param keyBytes Key bytes. - * @param retval {@code True} if need removed value. - * @param p Value predicate. - * @return Removed value bytes. - */ - byte[] remove(int hash, byte[] keyBytes, boolean retval, @Nullable IgniteBiPredicate p) { - int relSize = 0; - long relAddr = 0; - long qAddr = 0; - - long binAddr = writeLock(hash); - - try { - byte[] valBytes = null; - - long first = Bin.first(binAddr, mem); - - if (first != 0) { - long prev = 0; - long cur = first; - - while (true) { - long next = Entry.nextAddress(cur, mem); - - // If found match. - if (Entry.keyEquals(cur, keyBytes, mem)) { - int keyLen = 0; - int valLen = 0; - - if (p != null) { - keyLen = Entry.readKeyLength(cur, mem); - valLen = Entry.readValueLength(cur, mem); - - long valPtr = cur + HEADER_SIZE + keyLen; - - if (!p.apply(valPtr, valLen)) - return null; - } - - if (prev != 0) - Entry.nextAddress(prev, next, mem); // Relink. - else { - if (next == 0) - Bin.clear(binAddr, mem); - else - Bin.first(binAddr, next, mem); - } - - if (retval) { - if (keyLen == 0) { - keyLen = Entry.readKeyLength(cur, mem); - valLen = Entry.readValueLength(cur, mem); - } - - valBytes = mem.readBytes(cur + HEADER_SIZE + keyLen, valLen); - } - else - valBytes = EMPTY_BYTES; - - // Prepare release of memory. - qAddr = Entry.queueAddress(cur, mem); - relSize = Entry.size(cur, mem); - relAddr = cur; - - cnt--; - - totalCnt.decrement(); - - break; - } - - // If end of linked list. - if (next == 0) - break; - - prev = cur; - cur = next; - } - } - - return valBytes; - } - finally { - // Remove current mapping. - if (relAddr != 0 && lru != null) { - if (qAddr != 0) - lru.remove(qAddr); - } - - writeUnlock(); - - // Release memory outside lock. - if (relAddr != 0) - mem.release(relAddr, relSize); - } - } - - /** - * @param hash Hash. - * @param keyBytes Key bytes. - * @return {@code True} if contains key. - */ - boolean contains(int hash, byte[] keyBytes) { - long binAddr = readLock(hash); - - try { - long addr = Bin.first(binAddr, mem); - - while (addr != 0) { - if (Entry.keyEquals(addr, keyBytes, mem)) - return true; - - addr = Entry.nextAddress(addr, mem); - } - - return false; - } - finally { - readUnlock(); - } - - } - - /** - * @param hash Hash. - * @param keyBytes Key bytes. - * @return Value pointer. - */ - @Nullable IgniteBiTuple valuePointer(int hash, byte[] keyBytes) { - long binAddr = readLock(hash); - - try { - long addr = Bin.first(binAddr, mem); - - while (addr != 0) { - if (Entry.keyEquals(addr, keyBytes, mem)) { - if (lru != null) { - long qAddr = Entry.queueAddress(addr, mem); - - if (qAddr != 0 && Entry.clearQueueAddress(addr, qAddr, mem)) - lru.remove(qAddr); - } - - int keyLen = Entry.readKeyLength(addr, mem); - int valLen = Entry.readValueLength(addr, mem); - - return new IgniteBiTuple<>(addr + HEADER_SIZE + keyLen, valLen); - } - - addr = Entry.nextAddress(addr, mem); - } - - return null; - } - finally { - readUnlock(); - } - } - - /** - * @param hash Hash. - * @param keyBytes Key bytes. - */ - void enableEviction(int hash, byte[] keyBytes) { - assert lru != null; - - long binAddr = writeLock(hash); - - try { - long addr = Bin.first(binAddr, mem); - - while (addr != 0) { - if (Entry.keyEquals(addr, keyBytes, mem)) { - long qAddr = Entry.queueAddress(addr, mem); - - if (qAddr == 0) { - qAddr = lru.offer(part, addr, hash); - - Entry.queueAddress(addr, qAddr, mem); - } - - return; - } - - addr = Entry.nextAddress(addr, mem); - } - } - finally { - writeUnlock(); - } - } - - /** - * @param hash Hash. - * @param keyBytes Key bytes. - * @return Value bytes. - */ - @Nullable byte[] get(int hash, byte[] keyBytes) { - long binAddr = readLock(hash); - - try { - long addr = Bin.first(binAddr, mem); - - while (addr != 0) { - if (Entry.keyEquals(addr, keyBytes, mem)) { - // TODO: GG-8123: Inlined as a workaround. Revert when 7u60 is released. -// return Entry.readValueBytes(addr, mem); - { - int keyLen = Entry.readKeyLength(addr, mem); - int valLen = Entry.readValueLength(addr, mem); - - return mem.readBytes(addr + HEADER_SIZE + keyLen, valLen); - } - } - - addr = Entry.nextAddress(addr, mem); - } - - return null; - } - finally { - readUnlock(); - } - } - } - - /** - * Bin structure. - */ - private static class Bin { - /** - * @param binAddr Bin address location. - * @param mem Memory. - */ - static void clear(long binAddr, GridUnsafeMemory mem) { - mem.writeLong(binAddr, 0L); // Clear pointer. - } - - /** - * Writes first entry address. - * - * @param binAddr Pointer. - * @param entryAddr Address. - * @param mem Memory. - */ - static void first(long binAddr, long entryAddr, GridUnsafeMemory mem) { - mem.writeLong(binAddr, entryAddr); - } - - /** - * Reads first entry address. - * - * @param binAddr Pointer. - * @param mem Memory. - * @return addr Address. - */ - static long first(long binAddr, GridUnsafeMemory mem) { - return mem.readLong(binAddr); - } - } - - /** - * Entry structure. - */ - private static class Entry { - /** - * @param keyBytes Key bytes. - * @param valBytes Value bytes. - * @return Entry memory size. - */ - static int size(byte[] keyBytes, byte[] valBytes) { - return HEADER_SIZE + keyBytes.length + valBytes.length; - } - - /** - * @param addr Address. - * @param mem Memory. - * @return Entry size. - */ - static int size(long addr, GridUnsafeMemory mem) { - return HEADER_SIZE + readKeyLength(addr, mem) + readValueLength(addr, mem); - } - - /** - * @param ptr Pointer. - * @param mem Memory. - * @return Hash. - */ - static int hash(long ptr, GridUnsafeMemory mem) { - return mem.readInt(ptr + HEADER_HASH_OFF); - } - - /** - * @param ptr Pointer. - * @param hash Hash. - * @param mem Memory. - */ - static void hash(long ptr, int hash, GridUnsafeMemory mem) { - mem.writeInt(ptr + HEADER_HASH_OFF, hash); - } - - /** - * @param ptr Pointer. - * @param mem Memory. - * @return Key length. - */ - static int readKeyLength(long ptr, GridUnsafeMemory mem) { - int len = mem.readInt(ptr + HEADER_KEY_SIZE_OFF); - - assert len >= 0 : "Invalid key length [addr=" + String.format("0x%08x", ptr) + - ", len=" + Long.toHexString(len) + ']'; - - return len; - } - - /** - * Writes key length. - * - * @param ptr Pointer. - * @param len Length. - * @param mem Memory. - */ - static void writeKeyLength(long ptr, int len, GridUnsafeMemory mem) { - mem.writeInt(ptr + HEADER_KEY_SIZE_OFF, len); - } - - /** - * @param ptr Pointer. - * @param mem Memory. - * @return Value length. - */ - static int readValueLength(long ptr, GridUnsafeMemory mem) { - int len = mem.readInt(ptr + HEADER_VALUE_SIZE); - - assert len >= 0 : "Invalid value length [addr=" + String.format("0x%08x", ptr) + - ", len=" + Integer.toHexString(len) + ']'; - - return len; - } - - /** - * Writes value length. - * - * @param ptr Pointer. - * @param len Length. - * @param mem Memory. - */ - static void writeValueLength(long ptr, int len, GridUnsafeMemory mem) { - mem.writeInt(ptr + HEADER_VALUE_SIZE, len); - } - - /** - * @param ptr Pointer. - * @param mem Memory. - * @return Queue address. - */ - static long queueAddress(long ptr, GridUnsafeMemory mem) { - return mem.readLong(ptr + HEADER_QUEUE_ADDR_OFF); - } - - /** - * @param ptr Pointer. - * @param qAddr Queue address. - * @param mem Memory. - */ - static void queueAddress(long ptr, long qAddr, GridUnsafeMemory mem) { - mem.writeLong(ptr + HEADER_QUEUE_ADDR_OFF, qAddr); - } - - /** - * @param ptr Pointer. - * @param qAddr Queue address. - * @param mem Memory. - * @return {@code True} if changed to zero. - */ - static boolean clearQueueAddress(long ptr, long qAddr, GridUnsafeMemory mem) { - return mem.casLong(ptr + HEADER_QUEUE_ADDR_OFF, qAddr, 0); - } - - /** - * @param ptr Pointer. - * @param mem Memory. - * @return Next address. - */ - static long nextAddress(long ptr, GridUnsafeMemory mem) { - return mem.readLong(ptr + HEADER_NEXT_ADDR_OFF); - } - - /** - * Writes next entry address. - * - * @param ptr Pointer. - * @param addr Address. - * @param mem Memory. - */ - static void nextAddress(long ptr, long addr, GridUnsafeMemory mem) { - mem.writeLong(ptr + HEADER_NEXT_ADDR_OFF, addr); - } - - /** - * @param ptr Pointer. - * @param mem Memory. - * @return Key bytes. - */ - static byte[] readKeyBytes(long ptr, GridUnsafeMemory mem) { - int keyLen = readKeyLength(ptr, mem); - - return mem.readBytes(ptr + HEADER_SIZE, keyLen); - } - - /** - * @param ptr Pointer. - * @param keyBytes Key bytes. - * @param mem Memory. - */ - static void writeKeyBytes(long ptr, byte[] keyBytes, GridUnsafeMemory mem) { - mem.writeBytes(ptr + HEADER_SIZE, keyBytes); - } - - /** - * @param ptr Pointer. - * @param mem Memory. - * @return Value bytes. - */ - static byte[] readValueBytes(long ptr, GridUnsafeMemory mem) { - int keyLen = readKeyLength(ptr, mem); - int valLen = readValueLength(ptr, mem); - - return mem.readBytes(ptr + HEADER_SIZE + keyLen, valLen); - } - - /** - * @param ptr Pointer. - * @param valBytes Value bytes. - * @param mem Memory. - */ - static void writeValueBytes(long ptr, byte[] valBytes, GridUnsafeMemory mem) { - writeValueBytes(ptr, readKeyLength(ptr, mem), valBytes, mem); - } - - /** - * @param ptr Pointer. - * @param keyLen Key length. - * @param valBytes Value bytes. - * @param mem Memory. - */ - static void writeValueBytes(long ptr, int keyLen, byte[] valBytes, GridUnsafeMemory mem) { - mem.writeBytes(ptr + HEADER_SIZE + keyLen, valBytes); - } - - /** - * Writes entry. - * - * @param ptr Pointer. - * @param hash Hash. - * @param keyBytes Key bytes. - * @param valBytes Value bytes. - * @param mem Memory. - */ - static void write(long ptr, int hash, byte[] keyBytes, byte[] valBytes, GridUnsafeMemory mem) { - hash(ptr, hash, mem); - writeKeyLength(ptr, keyBytes.length, mem); - writeValueLength(ptr, valBytes.length, mem); - writeKeyBytes(ptr, keyBytes, mem); - writeValueBytes(ptr, keyBytes.length, valBytes, mem); - } - - /** - * Writes entry. - * - * @param ptr Pointer. - * @param hash Hash. - * @param keyBytes Key bytes. - * @param valBytes Value bytes. - * @param queueAddr Queue address. - * @param next Next address. - * @param mem Memory. - */ - static void write(long ptr, int hash, byte[] keyBytes, byte[] valBytes, long queueAddr, long next, - GridUnsafeMemory mem) { - hash(ptr, hash, mem); - writeKeyLength(ptr, keyBytes.length, mem); - writeValueLength(ptr, valBytes.length, mem); - queueAddress(ptr, queueAddr, mem); - nextAddress(ptr, next, mem); - writeKeyBytes(ptr, keyBytes, mem); - writeValueBytes(ptr, keyBytes.length, valBytes, mem); - } - - /** - * Checks if keys are equal. - * - * @param ptr Pointer. - * @param keyBytes Key bytes to compare. - * @param mem Memory. - * @return {@code True} if equal. - */ - static boolean keyEquals(long ptr, byte[] keyBytes, GridUnsafeMemory mem) { - long len = readKeyLength(ptr, mem); - - return len == keyBytes.length && GridUnsafeMemory.compare(ptr + HEADER_SIZE, keyBytes); - } - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafePartitionedMap.java b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafePartitionedMap.java deleted file mode 100644 index 7022af225c74e..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafePartitionedMap.java +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap.unsafe; - -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.atomic.LongAdder; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.internal.util.GridCloseableIteratorAdapter; -import org.apache.ignite.internal.util.lang.GridCloseableIterator; -import org.apache.ignite.internal.util.offheap.GridOffHeapEventListener; -import org.apache.ignite.internal.util.offheap.GridOffHeapEvictListener; -import org.apache.ignite.internal.util.offheap.GridOffHeapMap; -import org.apache.ignite.internal.util.offheap.GridOffHeapPartitionedMap; -import org.apache.ignite.internal.util.typedef.CX2; -import org.apache.ignite.internal.util.typedef.T2; -import org.apache.ignite.lang.IgniteBiPredicate; -import org.apache.ignite.lang.IgniteBiTuple; -import org.jetbrains.annotations.Nullable; - -/** - * Off-heap map based on {@code Unsafe} implementation. - */ -public class GridUnsafePartitionedMap implements GridOffHeapPartitionedMap { - /** Minimum segment concurrency. */ - private static final int MIN_SEGMENT_CONCURRENCY = 4; - - /** Array holding maps for each partition. */ - private final GridUnsafeMap[] partMap; - - /** Total memory. */ - private final GridUnsafeMemory mem; - - /** Evict closure. */ - private GridOffHeapEvictListener evictLsnr; - - /** Event listener. */ - private GridOffHeapEventListener evtLsnr; - - /** Striped LRU policy. */ - private final GridUnsafeLru lru; - - /** Concurrency. */ - private final int concurrency; - - /** Load factor. */ - private final float load; - - /** Partitions. */ - private final int parts; - - /** */ - private final LongAdder totalCnt = new LongAdder(); - - /** - * @param parts Partitions. - * @param concurrency Concurrency. - * @param load Load factor. - * @param initCap Initial capacity. - * @param totalMem Total memory. - * @param lruStripes LRU stripes. - * @param evictLsnr Eviction callback. - */ - public GridUnsafePartitionedMap(int parts, int concurrency, float load, long initCap, long totalMem, - short lruStripes, @Nullable GridOffHeapEvictListener evictLsnr) { - this.parts = parts; - this.concurrency = concurrency; - this.load = load; - - // Unchecked assignment to avoid generic array creation. - partMap = new GridUnsafeMap[parts]; - - mem = new GridUnsafeMemory(totalMem); - - if (totalMem > 0) - this.evictLsnr = evictLsnr; - - lru = totalMem > 0 ? new GridUnsafeLru(lruStripes, mem) : null; - - long cnt = initCap / parts; - int mod = (int)(initCap % parts); - - // Since we have natural striping provided by partitioning, we can proportionally reduce - // number of segments in each map. - concurrency /= parts; - - if (concurrency < MIN_SEGMENT_CONCURRENCY) - concurrency = MIN_SEGMENT_CONCURRENCY; - - for (int p = 0; p < parts; p++) { - mod--; - - long init = mod >= 0 ? cnt + 1 : cnt; - - partMap[p] = new GridUnsafeMap(p, concurrency, load, init, totalCnt, mem, lru, evictLsnr, - new GridUnsafeLruPoller() { - @Override public void lruPoll(int size) { - if (lru == null) - return; - - int left = size; - - while (left > 0) { - // Pre-poll outside of lock. - long qAddr = lru.prePoll(); - - if (qAddr == 0) - return; // LRU is empty. - - short order = lru.order(qAddr); - - int part = lru.partition(order, qAddr); - - int released = partMap[part].freeSpace(order, qAddr); - - if (released == 0) - return; - - left -= released; - } - } - } - ); - } - } - - /** {@inheritDoc} */ - @Override public float loadFactor() { - return load; - } - - /** {@inheritDoc} */ - @Override public int concurrency() { - return concurrency; - } - - /** {@inheritDoc} */ - @Override public int partitions() { - return parts; - } - - /** - * @param p Partition. - * @return Map for partition. - */ - private GridOffHeapMap mapFor(int p) { - assert p < parts; - - return partMap[p]; - } - - /** {@inheritDoc} */ - @Override public boolean contains(int part, int hash, byte[] keyBytes) { - return mapFor(part).contains(hash, keyBytes); - } - - /** {@inheritDoc} */ - @Override public byte[] get(int p, int hash, byte[] keyBytes) { - return mapFor(p).get(hash, keyBytes); - } - - /** {@inheritDoc} */ - @Override public IgniteBiTuple valuePointer(int p, int hash, byte[] keyBytes) { - return mapFor(p).valuePointer(hash, keyBytes); - } - - /** {@inheritDoc} */ - @Override public void enableEviction(int p, int hash, byte[] keyBytes) { - if (lru == null) - return; - - mapFor(p).enableEviction(hash, keyBytes); - } - - /** {@inheritDoc} */ - @Override public byte[] remove(int p, int hash, byte[] keyBytes) { - return mapFor(p).remove(hash, keyBytes); - } - - /** {@inheritDoc} */ - @Override public boolean removex(int p, int hash, byte[] keyBytes) { - return mapFor(p).removex(hash, keyBytes); - } - - /** {@inheritDoc} */ - @Override public boolean removex(int part, - int hash, - byte[] keyBytes, - IgniteBiPredicate p) { - return mapFor(part).removex(hash, keyBytes, p); - } - - /** {@inheritDoc} */ - @Override public boolean put(int p, int hash, byte[] keyBytes, byte[] valBytes) { - return mapFor(p).put(hash, keyBytes, valBytes); - } - - /** {@inheritDoc} */ - @Override public void insert(int p, int hash, byte[] keyBytes, byte[] valBytes) { - mapFor(p).insert(hash, keyBytes, valBytes); - } - - /** {@inheritDoc} */ - @Override public long size() { - return totalCnt.sum(); - } - - /** {@inheritDoc} */ - @Override public long size(Set parts) { - int cnt = 0; - - for (Integer part : parts) - cnt += mapFor(part).size(); - - return cnt; - } - - /** {@inheritDoc} */ - @Override public long memorySize() { - return mem.totalSize(); - } - - /** {@inheritDoc} */ - @Override public long allocatedSize() { - return mem.allocatedSize(); - } - - /** {@inheritDoc} */ - @Override public long systemAllocatedSize() { - return mem.systemAllocatedSize(); - } - - /** {@inheritDoc} */ - @Override public long freeSize() { - return mem.freeSize(); - } - - /** {@inheritDoc} */ - @Override public boolean eventListener(GridOffHeapEventListener evtLsnr) { - if (this.evtLsnr != null) - return false; - - this.evtLsnr = evtLsnr; - - for (GridUnsafeMap m : partMap) - m.eventListener(evtLsnr); - - mem.listen(evtLsnr); - - return true; - } - - /** {@inheritDoc} */ - @Override public boolean evictListener(GridOffHeapEvictListener evictLsnr) { - if (this.evictLsnr != null) - return false; - - this.evictLsnr = evictLsnr; - - for (GridUnsafeMap m : partMap) - m.evictListener(evictLsnr); - - return true; - } - - /** {@inheritDoc} */ - @Override public void destruct() { - for (GridUnsafeMap m : partMap) - m.destruct(); - - if (lru != null) - lru.destruct(); - } - - /** {@inheritDoc} */ - @Override public GridCloseableIterator> iterator() { - return new PartitionedMapCloseableIterator>() { - @Override protected void advance() throws IgniteCheckedException { - curIt = null; - - while (p < parts) { - curIt = mapFor(p++).iterator(); - - if (curIt.hasNext()) - return; - else - curIt.close(); - } - - curIt = null; - } - }; - } - - /** {@inheritDoc} */ - @Override public GridCloseableIterator iterator(final CX2, T2, T> c) { - assert c != null; - - return new PartitionedMapCloseableIterator() { - @Override protected void advance() throws IgniteCheckedException { - curIt = null; - - while (p < parts) { - curIt = mapFor(p++).iterator(c); - - if (curIt.hasNext()) - return; - else - curIt.close(); - } - - curIt = null; - } - }; - } - - /** {@inheritDoc} */ - @Override public GridCloseableIterator iterator(final CX2, T2, T> c, - int part) { - return mapFor(part).iterator(c); - } - - /** {@inheritDoc} */ - @Override public GridCloseableIterator> iterator(int p) { - return mapFor(p).iterator(); - } - - /** - * Gets number of LRU stripes. - * - * @return Number of LRU stripes. - */ - public short lruStripes() { - return lru.concurrency(); - } - - /** - * Gets memory size occupied by LRU queue. - * - * @return Memory size occupied by LRU queue. - */ - public long lruMemorySize() { - return lru.memorySize(); - } - - /** - * Gets number of elements in LRU queue. - * - * @return Number of elements in LRU queue. - */ - public long lruSize() { - return lru.size(); - } - - /** - * Partitioned closable iterator. - */ - private abstract class PartitionedMapCloseableIterator extends GridCloseableIteratorAdapter { - /** Current partition. */ - protected int p; - - /** Current iterator. */ - protected GridCloseableIterator curIt; - - { - try { - advance(); - } - catch (IgniteCheckedException e) { - e.printStackTrace(); // Should never happen. - } - } - - /** - * Switch to next partition. - * - * @throws IgniteCheckedException If failed. - */ - abstract void advance() throws IgniteCheckedException; - - /** {@inheritDoc} */ - @Override protected T onNext() throws IgniteCheckedException { - if (curIt == null) - throw new NoSuchElementException(); - - T t = curIt.next(); - - if (!curIt.hasNext()) { - curIt.close(); - - advance(); - } - - return t; - } - - /** {@inheritDoc} */ - @Override protected boolean onHasNext() { - return curIt != null; - } - - /** {@inheritDoc} */ - @Override protected void onRemove() { - throw new UnsupportedOperationException(); - } - - /** {@inheritDoc} */ - @Override protected void onClose() throws IgniteCheckedException { - if (curIt != null) - curIt.close(); - } - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java index dcd3192d3d353..dee7c59a42e7f 100644 --- a/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java +++ b/modules/core/src/main/java/org/apache/ignite/mxbean/SnapshotMXBean.java @@ -129,11 +129,10 @@ public void restoreSnapshot( public void cancelSnapshotRestore(@MXBeanParameter(name = "snpName", description = "Snapshot name.") String name); /** - * Get the status of the current snapshot check or restore operation in the cluster. Doesn't provide status of - * snapshots check operations. + * Get the status of the current snapshot operation in the cluster. * - * @return The status of a current snapshot create or restore operation in the cluster. + * @return The status of a current snapshot operation in the cluster. */ - @MXBeanDescription("The status of a current snapshot create or restore operation in the cluster.") + @MXBeanDescription("The status of a current snapshot operation in the cluster.") public String status(); } diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index aefefd45c169f..daec0c4d8a722 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -2069,15 +2069,7 @@ org.apache.ignite.internal.util.nio.GridNioServer$ReadWriteSizeBasedBalancer org.apache.ignite.internal.util.nio.GridNioServer$SizeBasedBalancer org.apache.ignite.internal.util.nio.GridNioSessionMetaKey org.apache.ignite.internal.util.nio.ssl.GridNioSslHandler -org.apache.ignite.internal.util.offheap.GridOffHeapEvent org.apache.ignite.internal.util.offheap.GridOffHeapOutOfMemoryException -org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMap$2 -org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMap$3 -org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMap$Segment$1 -org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMap$Segment$2 -org.apache.ignite.internal.util.offheap.unsafe.GridUnsafePartitionedMap$2 -org.apache.ignite.internal.util.offheap.unsafe.GridUnsafePartitionedMap$3 -org.apache.ignite.internal.util.offheap.unsafe.GridUnsafePartitionedMap$PartitionedMapCloseableIterator org.apache.ignite.internal.util.tostring.SBLimitedLength org.apache.ignite.internal.util.typedef.C1 org.apache.ignite.internal.util.typedef.C2 diff --git a/modules/core/src/test/java/org/apache/ignite/internal/direct/DirectMarshallingMessagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/direct/DirectMarshallingMessagesTest.java index 5eab512eb6e2e..a2eb82da6419a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/direct/DirectMarshallingMessagesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/direct/DirectMarshallingMessagesTest.java @@ -22,11 +22,14 @@ import java.util.Collection; import java.util.EnumSet; import java.util.Map; +import java.util.Random; import org.apache.ignite.internal.CoreMessagesProvider; import org.apache.ignite.internal.managers.communication.IgniteMessageFactoryImpl; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageFactory; import org.apache.ignite.plugin.extensions.communication.MessageFactoryProvider; +import org.apache.ignite.plugin.extensions.communication.MessageReader; +import org.apache.ignite.plugin.extensions.communication.RawField; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.TransactionIsolation; import org.junit.Test; @@ -123,6 +126,35 @@ public void testNullContainers() { assertNull(resMsg.isolationsList); } + /** Raw field larger than the writer's temporary buffer is serialized in several pieces. */ + @Test + public void testLargeRawField() { + byte[] val = new byte[40_000]; + + new Random(42).nextBytes(val); + + DirectMessageWriter writer = new DirectMessageWriter(msgFactory); + + ByteBuffer buf = ByteBuffer.allocate(1 << 16); + + writer.setBuffer(buf); + + writer.addPendingRawField(1, w -> w.writeByteArray(val)); + + assertTrue(writer.writeRawFields(null)); + + buf.flip(); + + DirectMessageReader reader = new DirectMessageReader(msgFactory, null); + + reader.setBuffer(buf); + + RawField f = reader.readRawFields().get(0); + + assertEquals(1, f.tag()); + assertArrayEquals(val, reader.deserializeRawField(f, MessageReader::readByteArray)); + } + /** * @param srcMsg Message to marshal. * @param Message type. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/CompressedMessageTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/CompressedMessageTest.java index 2f9c77e429919..c1e259bd8e081 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/CompressedMessageTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/CompressedMessageTest.java @@ -61,7 +61,7 @@ public void testWriteReadHugeMessage() { writer.setBuffer(tmpBuf); - GridDhtPartitionsFullMessage fullMsg = fullMessage(); + GridDhtPartitionsFullMessage fullMsg = fullMessage(500); boolean finished = false; boolean checkChunkCnt = true; @@ -178,7 +178,7 @@ public void testUncompressFailsWithoutChunks() { public void testUncompressFailsOnUnderstatedDataSize() { byte[] data = new byte[1000]; - CompressedMessage sent = new CompressedMessage(ByteBuffer.wrap(data), Deflater.BEST_SPEED); + CompressedMessage sent = compress(ByteBuffer.wrap(data)); CompressedMessage rcvd = new CompressedMessage(); @@ -196,7 +196,7 @@ public void testUncompressFailsOnUnderstatedDataSizeMultiChunk() { new Random(42).nextBytes(data); - CompressedMessage sent = new CompressedMessage(ByteBuffer.wrap(data), Deflater.BEST_SPEED); + CompressedMessage sent = compress(ByteBuffer.wrap(data)); assertTrue(sent.chunks.size() > 1); @@ -218,13 +218,13 @@ public void testReadFailsOnTruncatedPayload() { writer.setBuffer(tmpBuf); - assertTrue(writer.writeMessage(fullMessage(), false, LATEST_SCHEMA)); + assertTrue(writer.writeMessage(fullMessage(500), false, LATEST_SCHEMA)); tmpBuf.flip(); tmpBuf.limit(tmpBuf.limit() - 5); // Truncate the serialized message. - CompressedMessage compressedMsg = new CompressedMessage(tmpBuf, Deflater.BEST_SPEED); + CompressedMessage compressedMsg = compress(tmpBuf); DirectMessageWriter wireWriter = new DirectMessageWriter(MSG_FACTORY); @@ -243,12 +243,99 @@ public void testReadFailsOnTruncatedPayload() { GridTestUtils.assertThrows(null, () -> reader.readMessage(true, LATEST_SCHEMA), IgniteException.class, "ended unexpectedly"); } - /** */ - private GridDhtPartitionsFullMessage fullMessage() { + /** Temporary buffer of the writer must not grow for a large compressed field. */ + @Test + public void testTemporaryBufferDoesNotGrow() { + DirectMessageWriter writer = new DirectMessageWriter(MSG_FACTORY); + + writeAndRead(writer, fullMessage(10)); + + int tmpBufCap = U.field(writer, "tmpBuf").capacity(); + + // Compressed size above the buffer capacity means the raw data was serialized in several pieces. + assertTrue(writeAndRead(writer, fullMessage(10_000)) > tmpBufCap); + + assertEquals(tmpBufCap, U.field(writer, "tmpBuf").capacity()); + } + + /** Writer must stay usable if serialization of a compressed field fails. */ + @Test + public void testWriterReusableAfterFailedCompressedSerialization() { + Map brokenSuppliers = new HashMap<>(); + + // Null value fails with NPE in the middle of the nested map serialization. + brokenSuppliers.put(new GroupPartitionIdPair(1, 2), null); + + GridDhtPartitionsFullMessage brokenMsg = new GridDhtPartitionsFullMessage(null, null, new AffinityTopologyVersion(0), + Map.of(UUID.randomUUID(), brokenSuppliers), Map.of()); + + DirectMessageWriter writer = new DirectMessageWriter(MSG_FACTORY); + + writer.setBuffer(ByteBuffer.allocate(4096)); + + GridTestUtils.assertThrows(null, () -> writer.writeMessage(brokenMsg, true, LATEST_SCHEMA), NullPointerException.class, null); + + writeAndRead(writer, fullMessage(500)); + } + + /** + * Writes the message with compression through the given writer and checks it is read back unchanged. + * + * @return Size of the written message. + */ + private int writeAndRead(DirectMessageWriter writer, GridDhtPartitionsFullMessage msg) { + writer.reset(); + + ByteBuffer buf = ByteBuffer.allocate(4096); + ByteBuffer msgBuf = ByteBuffer.allocate(1 << 20); + + writer.setBuffer(buf); + + boolean finished = false; + + while (!finished) { + finished = writer.writeMessage(msg, true, LATEST_SCHEMA); + + buf.flip(); + + msgBuf.put(buf); + + buf.clear(); + } + + msgBuf.flip(); + + DirectMessageReader reader = new DirectMessageReader(MSG_FACTORY, null); + + reader.setBuffer(msgBuf); + + Message readMsg = reader.readMessage(true, LATEST_SCHEMA); + + assertTrue(readMsg instanceof GridDhtPartitionsFullMessage); + + assertEqualsFullMsg(msg, (GridDhtPartitionsFullMessage)readMsg); + + return msgBuf.limit(); + } + + /** @return Message with the compressed buffer content. */ + private static CompressedMessage compress(ByteBuffer buf) { + try (CompressedMessage.Compressor compressor = new CompressedMessage.Compressor(Deflater.BEST_SPEED)) { + compressor.compress(buf); + + return compressor.message(); + } + } + + /** + * @param cnt Number of entries in the compressed maps. + * @return Full message. + */ + private GridDhtPartitionsFullMessage fullMessage(int cnt) { Map> partHistSuppliers = new HashMap<>(); Map>> partsToReload = new HashMap<>(); - for (int i = 0; i < 500; i++) { + for (int i = 0; i < cnt; i++) { UUID uuid = UUID.randomUUID(); partHistSuppliers.put(uuid, Map.of(new GroupPartitionIdPair(i, i + 1), i + 2L)); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/metric/MetricsConfigurationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/metric/MetricsConfigurationTest.java index 4bdd9989853c7..9a1c9ddd6924d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/metric/MetricsConfigurationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/metric/MetricsConfigurationTest.java @@ -199,6 +199,10 @@ public void testHistogramConfiguration() throws Exception { () -> bean.configureHistogramMetric(metricName(TX_METRICS, METRIC_SYSTEM_TIME_HISTOGRAM), new long[0]), IllegalArgumentException.class); + assertThrowsWithCause( + () -> bean.configureHistogramMetric(metricName(TX_METRICS, METRIC_SYSTEM_TIME_HISTOGRAM), new long[] {100, 50}), + IllegalArgumentException.class); + bean.configureHistogramMetric(metricName(TX_METRICS, METRIC_SYSTEM_TIME_HISTOGRAM), BOUNDS); HistogramMetric sysTime = g.context().metric().registry(TX_METRICS) @@ -226,6 +230,11 @@ public void testPageTimestampHistogramConfiguration() throws Exception { // Check buckets count including dummy bucket. assertEquals(PeriodicHistogramMetricImpl.DFLT_BUCKETS_CNT + 1, histogram.bucketsCount()); + // Buckets interval is set by the first two bounds. + assertThrowsWithCause( + () -> bean.configureHistogramMetric(metricName(registry, metricName), new long[] {60_000L}), + IllegalArgumentException.class); + // Reconfigure with 5 buckets and 1 minute interval. long interval = 60_000L; long[] bounds = new long[] {0L, interval, 0L, 0L, 0L}; @@ -235,6 +244,15 @@ public void testPageTimestampHistogramConfiguration() throws Exception { assertEquals(bounds.length + 1, histogram.bucketsCount()); assertEquals(interval, histogram.bucketsInterval()); + // Check configuration after reactivation. + g.cluster().state(ClusterState.INACTIVE); + g.cluster().state(ClusterState.ACTIVE); + + histogram = g.context().metric().registry(registry).findMetric(metricName); + + assertEquals(bounds.length + 1, histogram.bucketsCount()); + assertEquals(interval, histogram.bucketsInterval()); + // Check configuration after restart. stopGrid("persistent-0", false); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/MultiDcRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/MultiDcRebalancingTest.java index 9f77dc49626a2..9834e386c5d83 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/MultiDcRebalancingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/MultiDcRebalancingTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache.distributed.rebalancing; +import java.util.Set; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteException; @@ -28,20 +29,27 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.managers.communication.GridIoMessage; +import org.apache.ignite.internal.processors.cache.GridCacheProxyImpl; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap; +import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.WithSystemProperty; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; import static org.apache.ignite.IgniteSystemProperties.IGNITE_PREFER_WAL_REBALANCE; -/** */ -@WithSystemProperty(key = IGNITE_PREFER_WAL_REBALANCE, value = "true") +/** + * Tests the MDC rebalance decision logic: when the engine chooses full, historical, or + * skips rebalance after a node rejoins the cluster. Live network split-brain is covered + * by {@link MultiDcSplitBrainRebalanceTest}. + */ public class MultiDcRebalancingTest extends GridCommonAbstractTest { /** */ private static final String DC1 = "DC1"; @@ -91,16 +99,570 @@ public void testFullRebalanceCrossDC() throws Exception { /** */ @Test + @WithSystemProperty(key = IGNITE_PREFER_WAL_REBALANCE, value = "true") public void testHistoricalRebalanceSameDC() throws Exception { checkHistoricalRebalance(false); } /** */ @Test + @WithSystemProperty(key = IGNITE_PREFER_WAL_REBALANCE, value = "true") public void testHistoricalRebalanceCrossDC() throws Exception { checkHistoricalRebalance(true); } + /** + * Verifies that no rebalance demand is sent when a node restarts without any + * writes having occurred during its absence. + * + *

Scenario: 2 nodes, 1000 entries, checkpoint. Node 1 stops, nothing is + * written, node 1 restarts. Both nodes have identical partition counters, + * the CRD sees only one value per partition, no node is "behind", no demand. + */ + @Test + public void testNoRebalanceWhenNoDelta() throws Exception { + IgniteCache cache = startTwoNodeCluster(1); + + for (int i = 0; i < 1000; i++) + cache.put(i, i); + + forceCheckpoint(); + stopGrid(1); + IgniteEx ignite1 = startGrid(1, DC2); + awaitPartitionMapExchange(true, true, null); + RebalanceAwareCommSPI spi = commSPI(ignite1); + assertFalse(spi.historical); + assertFalse(spi.full); + } + + /** + * Verifies that partitions below {@code historical.rebalance.threshold} (default 500) + * are NOT eligible for historical rebalance. The node uses full rebalance instead. + * + *

Scenario: 2 nodes, only 100 entries (each partition has ~1 entry, well + * below threshold 500). Checkpoint. Node 1 stops, 5 more entries written + * (small delta), node 1 restarts. Despite the tiny delta, partition size < 500 + * means it is not a historical candidate, so full rebalance is used. + */ + @Test + public void testBelowThresholdForcesFull() throws Exception { + IgniteCache cache = startTwoNodeCluster(1); + + for (int i = 0; i < 100; i++) + cache.put(i, i); + + forceCheckpoint(); + stopGrid(1); + + for (int i = 100; i < 105; i++) + cache.put(i, i); + + IgniteEx ignite1 = startGrid(1, DC2); + waitRebalanceFinished(ignite1, DEFAULT_CACHE_NAME); + RebalanceAwareCommSPI spi = commSPI(ignite1); + assertTrue(spi.rebalanceMsgCnt > 0); + assertFalse(spi.historical); + assertTrue(spi.full); + } + + /** + * Verifies that with a very small dataset, only non-empty partitions appear in + * the rebalance demand (the skipZeros mechanism). + * + *

Scenario: 2 nodes, only 10 entries. Since there are 512 partitions, at most + * ~10 have counter > 0. Node 1 stops, 5 more entries written, node 1 restarts. + * The SingleMessage omits all empty partitions, the CRD only detects differences + * for the ~10 non-empty partitions, total demanded is well below 512. + */ + @Test + public void testSkipZerosReducesRebalancedPartitions() throws Exception { + IgniteCache cache = startTwoNodeCluster(1); + + for (int i = 0; i < 10; i++) + cache.put(i, i); + + forceCheckpoint(); + stopGrid(1); + + for (int i = 10; i < 15; i++) + cache.put(i, i); + + IgniteEx ignite1 = startGrid(1, DC2); + waitRebalanceFinished(ignite1, DEFAULT_CACHE_NAME); + RebalanceAwareCommSPI spi = commSPI(ignite1); + assertTrue(spi.rebalanceMsgCnt > 0); + assertTrue("Expected < 512 but got " + spi.totalDemandedParts, + spi.totalDemandedParts < 512); + } + + /** + * Verifies historical rebalance after a DC node comes back following an outage. + * + *

Scenario: 3 nodes — node0 (DC1), node1 (DC2), node2 (DC1). All in baseline, + * 10000 entries written. node1 (DC2) is stopped and writes nothing while down. + * The two DC1 nodes write 10000 more entries each. Then node1 restarts. + * Node1 must get cross-DC historical rebalance with the delta from both DC1 nodes. + */ + @Test + @WithSystemProperty(key = IGNITE_PREFER_WAL_REBALANCE, value = "true") + public void testDcNodeDownHistoricalRebalance() throws Exception { + IgniteCache cache = startThreeNodeCluster(2); + + for (int i = 0; i < 10_000; i++) + cache.put(i, i); + + forceCheckpoint(); + stopGrid(1); + + for (int i = 10_000; i < 20_000; i++) + cache.put(i, i); + + IgniteCache cache2 = grid(2).cache(DEFAULT_CACHE_NAME); + + for (int i = 20_000; i < 30_000; i++) + cache2.put(i, i); + + forceCheckpoint(); + IgniteEx ignite1 = startGrid(1, DC2); + waitRebalanceFinished(ignite1, DEFAULT_CACHE_NAME); + RebalanceAwareCommSPI spi = commSPI(ignite1); + assertTrue(spi.rebalanceMsgCnt > 0); + assertTrue(spi.historical); + assertTrue(spi.crossDcRebalance); + } + + /** + * Verifies that after a DC node comes back from an outage, no data is lost. Data + * written by both surviving DC1 nodes while the node was down is fully accessible + * from the returning DC2 node after recovery. + * + *

Scenario: 3 nodes, node1 (DC2) down. DC1 nodes write 20000 new entries while + * it is down. After recovery, all 30000 entries are readable from the DC2 node. + */ + @Test + @WithSystemProperty(key = IGNITE_PREFER_WAL_REBALANCE, value = "true") + public void testDcNodeDownNoDataLoss() throws Exception { + IgniteCache cache = startThreeNodeCluster(2); + + for (int i = 0; i < 10_000; i++) + cache.put(i, i); + + forceCheckpoint(); + stopGrid(1); + + for (int i = 10_000; i < 20_000; i++) + cache.put(i, i); + + IgniteCache cache2 = grid(2).cache(DEFAULT_CACHE_NAME); + + for (int i = 20_000; i < 30_000; i++) + cache2.put(i, i); + + forceCheckpoint(); + IgniteEx ignite1 = startGrid(1, DC2); + waitRebalanceFinished(ignite1, DEFAULT_CACHE_NAME); + IgniteCache cache1 = ignite1.cache(DEFAULT_CACHE_NAME); + + for (int i = 0; i < 30_000; i += 3000) { + Object val = cache1.get(i); + assertEquals(i, val); + } + } + + /** + * Verifies that after a DC node comes back from an outage, if partition size is + * below {@code historical.rebalance.threshold} (default 500), full rebalance is + * used (no PREFER_WAL_REBALANCE set). + * + *

Scenario: 3 nodes, 100 entries (below threshold). node1 (DC2) down, DC1 + * writes 200 more entries. After recovery, full rebalance is expected. + */ + @Test + public void testDcNodeDownBelowThresholdFullRebalance() throws Exception { + IgniteCache cache = startThreeNodeCluster(2); + + for (int i = 0; i < 100; i++) + cache.put(i, i); + + forceCheckpoint(); + stopGrid(1); + + for (int i = 100; i < 200; i++) + cache.put(i, i); + + IgniteCache cache2 = grid(2).cache(DEFAULT_CACHE_NAME); + + for (int i = 200; i < 300; i++) + cache2.put(i, i); + + forceCheckpoint(); + IgniteEx ignite1 = startGrid(1, DC2); + waitRebalanceFinished(ignite1, DEFAULT_CACHE_NAME); + RebalanceAwareCommSPI spi = commSPI(ignite1); + assertTrue(spi.rebalanceMsgCnt > 0); + assertFalse(spi.historical); + assertTrue(spi.full); + assertTrue(spi.crossDcRebalance); + } + + /** + * Verifies that after a DC node comes back from an outage, if the delta is larger + * than the partition size (and no PREFER_WAL_REBALANCE), full rebalance is used — + * transferring the full partition is more efficient than replaying the delta. + * + *

Scenario: 3 nodes, 1000 base entries. node1 (DC2) down. The DC1 nodes each + * write 20000 entries (delta >> partition size). After recovery, full rebalance + * is expected (not historical). + */ + @Test + public void testDcNodeDownLargeDeltaFullRebalance() throws Exception { + IgniteCache cache = startThreeNodeCluster(2); + + for (int i = 0; i < 1000; i++) + cache.put(i, i); + + forceCheckpoint(); + stopGrid(1); + + for (int i = 1000; i < 21_000; i++) + cache.put(i, i); + + IgniteCache cache2 = grid(2).cache(DEFAULT_CACHE_NAME); + + for (int i = 21_000; i < 41_000; i++) + cache2.put(i, i); + + forceCheckpoint(); + IgniteEx ignite1 = startGrid(1, DC2); + waitRebalanceFinished(ignite1, DEFAULT_CACHE_NAME); + RebalanceAwareCommSPI spi = commSPI(ignite1); + assertTrue(spi.rebalanceMsgCnt > 0); + assertFalse(spi.historical); + assertTrue(spi.full); + assertTrue(spi.crossDcRebalance); + } + + /** + * Verifies that if one of two available source nodes also goes down while the DC + * node is away, the surviving source node can still serve the historical rebalance. + * + *

Scenario: 3 nodes (node0 DC1, node1 DC2, node2 DC1). node1 (DC2) is stopped. + * Then node0 (one DC1 source) is ALSO stopped. Only node2 remains. After recovery, + * node1 must get historical rebalance from node2 alone. + */ + @Test + @WithSystemProperty(key = IGNITE_PREFER_WAL_REBALANCE, value = "true") + public void testDcNodeDownOneSourceNodeSurvives() throws Exception { + IgniteCache cache = startThreeNodeCluster(2); + + for (int i = 0; i < 10_000; i++) + cache.put(i, i); + + forceCheckpoint(); + stopGrid(1); + stopGrid(0); + IgniteCache cache2 = grid(2).cache(DEFAULT_CACHE_NAME); + + for (int i = 10_000; i < 20_000; i++) + cache2.put(i, i); + + forceCheckpoint(); + IgniteEx ignite1 = startGrid(1, DC2); + waitRebalanceFinished(ignite1, DEFAULT_CACHE_NAME); + RebalanceAwareCommSPI spi = commSPI(ignite1); + assertTrue(spi.rebalanceMsgCnt > 0); + assertTrue(spi.historical); + assertTrue(spi.crossDcRebalance); + IgniteCache cache1 = ignite1.cache(DEFAULT_CACHE_NAME); + + for (int i = 0; i < 20_000; i += 5000) { + Object val = cache1.get(i); + assertEquals(i, val); + } + } + + /** + * Verifies that concurrent writes on the source node during an in-progress + * historical rebalance do not corrupt data. After the rebalance completes, + * all entries (old + delta + concurrent) are accessible. + * + *

Scenario: 3 nodes. node1 (DC2) is stopped. DC1 writes 10000 entries (delta). + * node1 restarts (triggers rebalance). While the rebalance is in flight, + * DC1 writes another 5000 entries. After rebalance completes, verify all + * data is accessible. + */ + @Test + @WithSystemProperty(key = IGNITE_PREFER_WAL_REBALANCE, value = "true") + public void testConcurrentWritesDuringRecovery() throws Exception { + IgniteCache cache = startThreeNodeCluster(2); + + for (int i = 0; i < 10_000; i++) + cache.put(i, i); + + forceCheckpoint(); + stopGrid(1); + + for (int i = 10_000; i < 20_000; i++) + cache.put(i, i); + + forceCheckpoint(); + IgniteEx ignite1 = startGrid(1, DC2); + IgniteCache cache2 = grid(2).cache(DEFAULT_CACHE_NAME); + + for (int i = 20_000; i < 25_000; i++) + cache2.put(i, i); + + waitRebalanceFinished(ignite1, DEFAULT_CACHE_NAME); + IgniteCache cache1 = ignite1.cache(DEFAULT_CACHE_NAME); + + for (int i = 0; i < 25_000; i += 5000) { + Object val = cache1.get(i); + assertEquals(i, val); + } + } + + /** + * Verifies that after a DC node outage recovery, if a new node also joins + * (triggering affinity reassignment), the returning node gets a mixed demand: + * historical for its own partitions AND full for reassigned partitions. + * + *

Scenario: 2 nodes initially. node1 (DC2) is stopped, DC1 writes delta. + * Recovery: node1 restarts AND a new node (node2, DC1) joins simultaneously. + * The topology change triggers both historical rebalance (for DC2's old + * partitions) and full rebalance (for partitions reassigned to node2). + */ + @Test + @WithSystemProperty(key = IGNITE_PREFER_WAL_REBALANCE, value = "true") + public void testAffinityReassignmentMixedRebalance() throws Exception { + IgniteCache cache = startTwoNodeCluster(1); + + for (int i = 0; i < 10_000; i++) + cache.put(i, i); + + forceCheckpoint(); + stopGrid(1); + + for (int i = 10_000; i < 20_000; i++) + cache.put(i, i); + + forceCheckpoint(); + IgniteEx ignite1 = startGrid(1, DC2); + IgniteEx ignite2 = startGrid(2, DC1); + resetBaselineTopology(); + waitRebalanceFinished(ignite1, DEFAULT_CACHE_NAME); + waitRebalanceFinished(ignite2, DEFAULT_CACHE_NAME); + + RebalanceAwareCommSPI spi1 = commSPI(ignite1); + assertTrue(spi1.rebalanceMsgCnt > 0); + assertTrue(spi1.historical); + + RebalanceAwareCommSPI spi2 = commSPI(ignite2); + assertTrue("New node should receive full rebalance", spi2.full); + + IgniteCache cache1 = ignite1.cache(DEFAULT_CACHE_NAME); + + for (int i = 0; i < 20_000; i += 5000) { + Object val = cache1.get(i); + assertEquals(i, val); + } + } + + /** + * Verifies that with {@code rebalanceDelay > 0} the rebalance always uses the full + * strategy. + * + *

Documented limitation (see data-rebalancing.adoc, "Rebalance Strategy"): when a + * rebalance delay is configured, the rebalance is scheduled outside of the partition + * map exchange context, and the historical path (which is tied to that context) is + * never taken — every partition is restored with a full rebalance regardless of the + * delta size and regardless of {@code IGNITE_PREFER_WAL_REBALANCE}. + * + *

Related ticket: IGNITE-14524. + */ + @Test + @WithSystemProperty(key = IGNITE_PREFER_WAL_REBALANCE, value = "true") + public void testRebalanceDelayForcesFull() throws Exception { + pds = true; + IgniteEx ignite0 = startGrid(0, DC1); + startGrid(1, DC2); + startGrid(2, DC1); + ignite0.cluster().state(ClusterState.ACTIVE); + CacheConfiguration cacheCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); + cacheCfg.setBackups(2); + cacheCfg.setRebalanceDelay(3000); + IgniteCache cache = ignite0.createCache(cacheCfg); + + for (int i = 0; i < 10_000; i++) + cache.put(i, i); + + forceCheckpoint(); + stopGrid(1); + + for (int i = 10_000; i < 20_000; i++) + cache.put(i, i); + + forceCheckpoint(); + IgniteEx ignite1 = startGrid(1, DC2); + RebalanceAwareCommSPI spi = commSPI(ignite1); + + boolean demandSent = GridTestUtils.waitForCondition(() -> spi.rebalanceMsgCnt > 0, 30_000); + + assertTrue("No rebalance demand received within 30s", demandSent); + + awaitPartitionMapExchange(true, true, null); + assertTrue(spi.full); + assertFalse(spi.historical); + } + + /** + * Verifies that after MULTIPLE restarts (stop→write→start, repeated), the recovering + * node accumulates all data across all restart cycles. + * + *

Per IGNITE-17738: after a crash/restart, historical rebalance may not deliver + * updates between LWM and HWM. This test exercises two consecutive restart cycles + * and asserts full data integrity after each. + * + *

Scenario: 3 nodes. 10_000 base entries. + * Cycle 1: node1 down → node0 writes 10_000 → node1 restarts → verify 20_000. + * Cycle 2: node1 down → node0 writes 10_000 → node1 restarts → verify 30_000. + */ + @Test + @WithSystemProperty(key = IGNITE_PREFER_WAL_REBALANCE, value = "true") + public void testMultipleRestartsNoDataLoss() throws Exception { + IgniteCache cache = startThreeNodeCluster(2); + + for (int i = 0; i < 10_000; i++) + cache.put(i, i); + + forceCheckpoint(); + stopGrid(1); + + for (int i = 10_000; i < 20_000; i++) + cache.put(i, i); + + forceCheckpoint(); + IgniteEx ignite1 = startGrid(1, DC2); + waitRebalanceFinished(ignite1, DEFAULT_CACHE_NAME); + + IgniteCache cache1 = ignite1.cache(DEFAULT_CACHE_NAME); + + for (int i = 0; i < 20_000; i += 5000) { + Object val = cache1.get(i); + assertEquals(i, val); + } + + forceCheckpoint(); + stopGrid(1); + + for (int i = 20_000; i < 30_000; i++) + cache.put(i, i); + + forceCheckpoint(); + IgniteEx ignite1Restarted = startGrid(1, DC2); + waitRebalanceFinished(ignite1Restarted, DEFAULT_CACHE_NAME); + + IgniteCache cache1After = ignite1Restarted.cache(DEFAULT_CACHE_NAME); + + for (int i = 0; i < 30_000; i += 5000) { + Object val = cache1After.get(i); + assertEquals(i, val); + } + } + + /** + * Verifies that when historical (WAL) rebalance is impossible, the supplier falls + * back to a FULL rebalance and no data is lost. + * + *

Truncated WAL is simulated deterministically by marking the last checkpoint as + * inapplicable for WAL rebalance on the source nodes + * ({@code lastCheckpointInapplicableForWalRebalance}) — exactly the flag the supplier + * sets itself when a WAL segment turns out to be unreadable + * ({@code GridDhtPartitionSupplier} fallbackToFullRebalance, line 451). This is the + * production situation when WAL segments covering the delta were already deleted. + * + *

Assertion: despite {@code PREFER_WAL_REBALANCE=true} the recovering node gets a + * FULL rebalance (historical is impossible), and all entries are present afterwards. + */ + @Test + @WithSystemProperty(key = IGNITE_PREFER_WAL_REBALANCE, value = "true") + public void testWalTruncatedForcesFull() throws Exception { + IgniteCache cache = startThreeNodeCluster(2); + + for (int i = 0; i < 10_000; i++) + cache.put(i, i); + + forceCheckpoint(); + stopGrid(1); + + for (int i = 10_000; i < 20_000; i++) + cache.put(i, i); + + forceCheckpoint(); + + int grpId = ((GridCacheProxyImpl)grid(0).cachex(DEFAULT_CACHE_NAME)) + .context().group().groupId(); + + markWalRebalanceInapplicable(grid(0), grpId); + markWalRebalanceInapplicable(grid(2), grpId); + + IgniteEx ignite1 = startGrid(1, DC2); + waitRebalanceFinished(ignite1, DEFAULT_CACHE_NAME); + RebalanceAwareCommSPI spi = commSPI(ignite1); + + assertTrue(spi.full); + assertFalse(spi.historical); + + IgniteCache cache1 = ignite1.cache(DEFAULT_CACHE_NAME); + + for (int i = 0; i < 20_000; i += 5000) { + Object val = cache1.get(i); + assertEquals(i, val); + } + } + + /** + * Starts a 2-node MDC cluster (node0 in DC1, node1 in DC2) in ACTIVE state + * and creates a cache with the given number of backups. + * + * @param backups Number of backups for the cache. + * @return The created cache. + */ + private IgniteCache startTwoNodeCluster(int backups) throws Exception { + pds = true; + startGrid(0, DC1); + startGrid(1, DC2); + grid(0).cluster().state(ClusterState.ACTIVE); + + CacheConfiguration cacheCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); + cacheCfg.setBackups(backups); + return grid(0).createCache(cacheCfg); + } + + /** + * Starts a 3-node MDC cluster (node0 in DC1, node1 in DC2, node2 in DC1) in + * ACTIVE state and creates a cache with the given number of backups. + * + * @param backups Number of backups for the cache. + * @return The created cache. + */ + private IgniteCache startThreeNodeCluster(int backups) throws Exception { + pds = true; + startGrid(0, DC1); + startGrid(1, DC2); + startGrid(2, DC1); + grid(0).cluster().state(ClusterState.ACTIVE); + + CacheConfiguration cacheCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); + cacheCfg.setBackups(backups); + return grid(0).createCache(cacheCfg); + } + + /** */ + private void markWalRebalanceInapplicable(IgniteEx ignite, int grpId) { + IgniteCacheDatabaseSharedManager db = ignite.context().cache().context().database(); + db.lastCheckpointInapplicableForWalRebalance(grpId); + } + /** */ private void checkFullRebalance(boolean crossDC) throws Exception { startGrid(0, DC1); @@ -178,6 +740,12 @@ private static class RebalanceAwareCommSPI extends TcpCommunicationSpi { /** */ private boolean crossDcRebalance; + /** */ + private boolean full; + + /** */ + private int totalDemandedParts; + /** {@inheritDoc} */ @Override public void sendMessage( ClusterNode node, @@ -191,6 +759,20 @@ private static class RebalanceAwareCommSPI extends TcpCommunicationSpi { historical |= demandMsg.partitions().hasHistorical(); + IgniteDhtDemandedPartitionsMap parts = demandMsg.partitions(); + + full |= parts.hasFull(); + + if (parts.hasHistorical()) { + Set histSet = parts.historicalSet(); + totalDemandedParts += histSet.size(); + } + + if (parts.hasFull()) { + Set fullSet = parts.fullSet(); + totalDemandedParts += fullSet.size(); + } + if (!ignite.cluster().localNode().dataCenterId().equals(node.dataCenterId())) crossDcRebalance = true; } @@ -199,3 +781,4 @@ private static class RebalanceAwareCommSPI extends TcpCommunicationSpi { } } } + diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/MultiDcSplitBrainRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/MultiDcSplitBrainRebalanceTest.java new file mode 100644 index 0000000000000..f56fb9d548870 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/MultiDcSplitBrainRebalanceTest.java @@ -0,0 +1,446 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.rebalancing; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCluster; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterGroup; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.TopologyValidator; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.managers.communication.GridIoMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheTopologySplitAbstractTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; +import org.junit.Test; + +import static org.apache.ignite.IgniteSystemProperties.IGNITE_DATA_CENTER_ID; + +/** + * Tests MDC disaster-recovery behavior under a LIVE network partition: the two data + * centers stay up but lose sight of each other, and a topology validator makes the + * isolated minority data center read-only while the majority keeps serving writes. + * + *

Unlike {@link MultiDcRebalancingTest} (where a DC node is stopped and writes nothing + * while away), here both halves stay alive during the partition. Segmentation is achieved + * by faking discovery socket timeouts across the DC boundary (each half detects the other + * as failed) plus blocking cross-DC communication messages, reusing + * {@link IgniteCacheTopologySplitAbstractTest}. + */ +public class MultiDcSplitBrainRebalanceTest extends IgniteCacheTopologySplitAbstractTest { + /** */ + private static final String DC1 = "DC1"; + + /** */ + private static final String DC2 = "DC2"; + + /** */ + private static final int SEG_DC1 = 0; + + /** */ + private static final int SEG_DC2 = 1; + + /** Per-DC node layout: {0,1,2} -> DC1 (majority), {3} -> DC2 (minority). */ + private static final int[] NODE_DC = {0, 0, 0, 1}; + + /** Base discovery port; each node takes base + idx. */ + private static final int DISCO_PORT_BASE = TcpDiscoverySpi.DFLT_PORT; + + /** discovery port -> segment (0 = DC1, 1 = DC2). Filled on node start. */ + private final Map portToSeg = new ConcurrentHashMap<>(); + + /** */ + private boolean pds; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + int idx = Integer.parseInt(gridName.replaceAll("\\D+", "")); + cfg.setConsistentId(gridName); + + TcpDiscoverySpi discoSpi = (TcpDiscoverySpi)cfg.getDiscoverySpi(); + discoSpi.setLocalPort(DISCO_PORT_BASE + idx); + portToSeg.put(DISCO_PORT_BASE + idx, NODE_DC[idx] == 0 ? SEG_DC1 : SEG_DC2); + + List discoAddrs = new ArrayList<>(); + for (int i = 0; i < NODE_DC.length; i++) { + String addr = "127.0.0.1:" + (DISCO_PORT_BASE + i); + discoAddrs.add(addr); + } + TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder(); + ipFinder.setShared(true); + ipFinder.setAddresses(discoAddrs); + discoSpi.setIpFinder(ipFinder); + + cfg.setUserAttributes(F.asMap(IGNITE_DATA_CENTER_ID, NODE_DC[idx] == 0 ? DC1 : DC2)); + + DataRegionConfiguration dfltRgn = new DataRegionConfiguration(); + dfltRgn.setPersistenceEnabled(pds); + DataStorageConfiguration dsCfg = new DataStorageConfiguration(); + dsCfg.setDefaultDataRegionConfiguration(dfltRgn); + cfg.setDataStorageConfiguration(dsCfg); + cfg.setCommunicationSpi(new DemandRecordingCommSpi()); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected int segment(ClusterNode node) { + return DC2.equals(node.dataCenterId()) ? SEG_DC2 : SEG_DC1; + } + + /** {@inheritDoc} */ + @Override protected boolean isBlocked(int locPort, int rmtPort) { + Integer locSeg = portToSeg.get(locPort); + Integer rmtSeg = portToSeg.get(rmtPort); + if (locSeg == null || rmtSeg == null) + return false; + return !locSeg.equals(rmtSeg); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + portToSeg.clear(); + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + stopAllGrids(); + cleanPersistenceDir(); + } + + /** + * Disaster recovery: a network partition between the data centers makes the minority + * DC read-only while the majority DC keeps serving writes. The cache carries a + * topology validator that keeps it operative only while a node of the main DC (DC1) + * is visible in the topology; the isolated minority (DC2) no longer sees DC1, so the + * cache turns inoperative there and its writes are rejected, whereas the majority + * (DC1) stays writable and preserves its data. + * + *

Scenario: 4 nodes, DC1={node0,node1,node2} (majority, main DC), DC2={node3} + * (minority). After 1000 base entries the network is split; DC1 keeps writing, DC2 + * writes are rejected. + */ + @Test + public void testNetworkPartitionMinorityBecomesReadOnly() throws Exception { + IgniteCache cache0 = startSplitBrainCluster(new MainDcOnlyTopologyValidator()); + + splitAndWait(); + + cache0.put(1000, 1000); + assertEquals(1000, cache0.get(1000)); + + assertFalse("Minority DC2 must reject writes while partitioned", + tryPut(grid(3), 2000, 2000)); + + unsplit(); + + logRebalanceAfterHeal("MainDcOnly"); + + for (int i = 0; i <= 1000; i += 500) { + Object v = cache0.get(i); + assertEquals(i, v); + } + } + + /** + * Starts a 4-node MDC cluster (node0,1,2 in DC1, node3 in DC2) in ACTIVE state with + * baseline auto-adjustment disabled, creates a cache with backups=1 and the given + * topology validator (or none), writes 1000 base entries, and forces a checkpoint. + * + * @param validator Topology validator to apply, or {@code null} for no validator. + * @return The cache on node0. + */ + private IgniteCache startSplitBrainCluster(TopologyValidator validator) throws Exception { + pds = true; + + startGrid(0); + startGrid(3); + grid(0).cluster().state(ClusterState.ACTIVE); + grid(0).cluster().baselineAutoAdjustEnabled(false); + + CacheConfiguration cacheCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); + cacheCfg.setBackups(1); + if (validator != null) + cacheCfg.setTopologyValidator(validator); + IgniteCache cache = grid(0).createCache(cacheCfg); + + startGrid(1); + startGrid(2); + awaitServers(4, grid(0)); + + for (int i = 0; i < 1000; i++) + cache.put(i, i); + + forceCheckpoint(); + + return cache; + } + + /** */ + private boolean tryPut(IgniteEx ignite, int key, int val) { + try { + IgniteCache cache = ignite.cache(DEFAULT_CACHE_NAME); + cache.put(key, val); + + return true; + } + catch (Exception e) { + log.info(">>> Put rejected [node=" + ignite.name() + ", err=" + e.getMessage() + "]"); + + return false; + } + } + + /** + * After unsplit, logs the actual state and the rebalance demand counts per node. + * + *

Documented limitation: {@code splitAndWait}/{@code unsplit} cannot merge two + * half-rings back — once a half detects the loss of the other and processes + * {@code EVT_NODE_FAILED}, it stops reconnecting to those nodes. No new topology + * version is created across the halves, so no cross-DC rebalance demand is sent + * ({@code rebalanceMsgCnt == 0}). True split-brain merge with state-synchronization + * and rebalance is covered at the ducktape level (netem + control utility). + */ + private void logRebalanceAfterHeal(String testLabel) { + for (Ignite ig : G.allGrids()) { + IgniteConfiguration cfg = ig.configuration(); + DemandRecordingCommSpi spi = (DemandRecordingCommSpi)cfg.getCommunicationSpi(); + + IgniteCluster cluster = ig.cluster(); + ClusterGroup srvGrp = cluster.forServers(); + Collection srvNodes = srvGrp.nodes(); + int srvCnt = srvNodes.size(); + + log.info(">>> [" + testLabel + "] node=" + ig.name() + + ", servers_in_topo=" + srvCnt + + ", rebalanceMsgCnt=" + spi.rebalanceMsgCnt); + } + } + + /** + * Waits until the given node's cluster topology includes the expected number of + * server nodes. Returns as soon as the condition holds (30s safety cap). + */ + private void awaitServers(int expected, IgniteEx ignite) throws Exception { + IgniteCluster cluster = ignite.cluster(); + boolean done = GridTestUtils.waitForCondition(() -> { + ClusterGroup srvGrp = cluster.forServers(); + Collection srvNodes = srvGrp.nodes(); + return srvNodes.size() == expected; + }, 30_000); + assertTrue("Topology did not reach " + expected + " servers on " + ignite.name(), done); + } + + /** + * Verifies that with a validator requiring both DCs, the partition makes BOTH halves + * read-only: neither can see the other DC, so neither satisfies the validator. + * + *

Scenario: 4 nodes, DC1={0,1,2}, DC2={3}. Validator returns false unless at + * least one node from each DC is in the topology. After split, both sides reject + * writes. + */ + @Test + public void testNetworkPartitionBothHalvesReadOnly() throws Exception { + startSplitBrainCluster(new BothDcsTopologyValidator()); + + splitAndWait(); + + assertFalse("Majority DC1 must reject writes without DC2", + tryPut(grid(0), 1000, 1000)); + + assertFalse("Minority DC2 must reject writes without DC1", + tryPut(grid(3), 2000, 2000)); + + unsplit(); + + logRebalanceAfterHeal("BothDcsRequired"); + } + + /** + * Verifies that with a quorum validator (min server nodes), the majority keeps + * writing while the minority loses quorum and becomes read-only. + * + *

Scenario: 4 nodes, DC1={0,1,2} (3 nodes >= threshold=3), DC2={3} (1 node < 3). + * After split, DC1 passes the quorum check, DC2 does not. + */ + @Test + public void testNetworkPartitionMajorityKeepsQuorum() throws Exception { + IgniteCache cache0 = startSplitBrainCluster(new MinServerCountTopologyValidator(3)); + + splitAndWait(); + + cache0.put(1000, 1000); + assertEquals(1000, cache0.get(1000)); + + assertFalse("Minority DC2 must reject writes without quorum", + tryPut(grid(3), 2000, 2000)); + + unsplit(); + + logRebalanceAfterHeal("MinServerCount3"); + } + + /** + * Verifies that without a topology validator, BOTH halves stay writable during the + * partition (uncontrolled split-brain). Each half elects its own coordinator and + * accepts writes independently. + * + *

Scenario: 4 nodes, DC1={0,1,2}, DC2={3}. No validator set. After split, both + * DC1 and DC2 write successfully. + */ + @Test + public void testNetworkPartitionNoValidatorBothWritable() throws Exception { + IgniteCache cache0 = startSplitBrainCluster(null); + + splitAndWait(); + + cache0.put(1000, 1000); + assertEquals(1000, cache0.get(1000)); + + IgniteCache cache3 = grid(3).cache(DEFAULT_CACHE_NAME); + + cache3.put(2000, 2000); + assertEquals(2000, cache3.get(2000)); + + unsplit(); + + logRebalanceAfterHeal("NoValidator"); + } + + /** + * Keeps the cache operative only while at least one server node of the main data + * center (DC1) is present in the topology. When a partition isolates the minority + * DC2 from DC1, the cache becomes inoperative on the minority and rejects writes. + */ + private static class MainDcOnlyTopologyValidator implements TopologyValidator { + /** */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override public boolean validate(Collection nodes) { + for (ClusterNode node : nodes) { + if (!node.isClient() && DC1.equals(node.dataCenterId())) + return true; + } + + return false; + } + } + + /** + * Keeps the cache operative only while server nodes from BOTH DCs are present. + * When any DC is isolated, the validator fails on both sides. + */ + private static class BothDcsTopologyValidator implements TopologyValidator { + /** */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override public boolean validate(Collection nodes) { + boolean hasDc1 = false; + boolean hasDc2 = false; + + for (ClusterNode node : nodes) { + if (node.isClient()) + continue; + String dc = node.dataCenterId(); + if (DC1.equals(dc)) + hasDc1 = true; + if (DC2.equals(dc)) + hasDc2 = true; + } + + return hasDc1 && hasDc2; + } + } + + /** + * Keeps the cache operative only while at least {@code minCnt} server nodes are + * present in the topology. + */ + private static class MinServerCountTopologyValidator implements TopologyValidator { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private final int minCnt; + + /** */ + MinServerCountTopologyValidator(int minCnt) { + this.minCnt = minCnt; + } + + /** {@inheritDoc} */ + @Override public boolean validate(Collection nodes) { + int serverCnt = 0; + + for (ClusterNode node : nodes) { + if (!node.isClient()) + serverCnt++; + } + + return serverCnt >= minCnt; + } + } + + /** + * Communication SPI that records rebalance demand messages while inheriting the + * message-blocking behavior used for segmentation. + */ + private static class DemandRecordingCommSpi extends TestRecordingCommunicationSpi { + /** */ + private volatile int rebalanceMsgCnt; + + /** {@inheritDoc} */ + @Override public void sendMessage( + ClusterNode node, + Message msg, + IgniteInClosure ackClosure + ) throws IgniteSpiException { + if (msg instanceof GridIoMessage + && ((GridIoMessage)msg).message() instanceof GridDhtPartitionDemandMessage) + rebalanceMsgCnt++; + super.sendMessage(node, msg, ackClosure); + } + } +} + diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/NodeOrCacheStoppingDuringIdleVerifyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/NodeOrCacheStoppingDuringIdleVerifyTest.java new file mode 100644 index 0000000000000..6409082fffbcb --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/NodeOrCacheStoppingDuringIdleVerifyTest.java @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.OpenOption; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.failure.TestFailureHandler; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.management.cache.IdleVerifyResult; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.processors.cache.verify.GridNotIdleException; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.partitionFileName; +import static org.apache.ignite.internal.processors.pool.PoolProcessor.POOL_NAME_IDLE_VERIFY; +import static org.apache.ignite.testframework.GridTestUtils.runAsync; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** */ +@RunWith(Parameterized.class) +public class NodeOrCacheStoppingDuringIdleVerifyTest extends GridCommonAbstractTest { + /** */ + private static final int PARTS = 4; + + /** */ + private static final int KEYS = PARTS * 200; + + /** */ + private static final int NOT_IDLE_PART = 0; + + /** */ + private static volatile CountDownLatch partitionReadBlocked; + + /** */ + private static volatile CountDownLatch partitionReadUnblocked; + + /** */ + private static volatile CountDownLatch notIdlePartReadUnblocked; + + /** */ + @Parameterized.Parameter + public boolean checkCrc; + + /** */ + private TestFailureHandler failureHnd; + + /** */ + @Parameterized.Parameters(name = "checkCrc={0}") + public static Iterable parameters() { + return F.asList(false, true); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setDataStorageConfiguration(new DataStorageConfiguration() + .setFileIOFactory(new IdleVerifyThreadsBlockingFileIOFactory()) + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setPersistenceEnabled(true))) + .setFailureHandler(failureHnd); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cleanPersistenceDir(); + + failureHnd = new TestFailureHandler(false); + + partitionReadBlocked = new CountDownLatch(PARTS); + partitionReadUnblocked = new CountDownLatch(1); + notIdlePartReadUnblocked = new CountDownLatch(1); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + unblockPartitionRead(); + + stopAllGrids(); + + cleanPersistenceDir(); + + super.afterTest(); + } + + /** */ + @Test + public void testWithCacheDestruction() throws Exception { + prepareCluster(); + + CacheGroupContext grp = grid(0).context().cache().cacheGroup(CU.cacheId(DEFAULT_CACHE_NAME)); + + IgniteInternalFuture idleFut = startIdleVerify(); + + IgniteInternalFuture destroyFut = runAsync(() -> grid(0).destroyCache(DEFAULT_CACHE_NAME)); + + assertTrue(waitForCondition(grp::isPreparedToStop, getTestTimeout())); + + assertFalse(waitForCondition(destroyFut::isDone, 1_000)); + + unblockPartitionRead(); + + destroyFut.get(getTestTimeout()); + + assertFalse(F.isEmpty(idleFut.get(getTestTimeout()).exceptions())); + assertNull(failureHnd.failureContext()); + } + + /** */ + @Test + public void testWithDeactivation() throws Exception { + prepareCluster(); + + CacheGroupContext grp = grid(0).context().cache().cacheGroup(CU.cacheId(DEFAULT_CACHE_NAME)); + + IgniteInternalFuture idleFut = startIdleVerify(); + + IgniteInternalFuture deactivateFut = runAsync(() -> grid(0).cluster().state(ClusterState.INACTIVE)); + + assertTrue(waitForCondition(grp::isPreparedToStop, getTestTimeout())); + + assertFalse(waitForCondition(deactivateFut::isDone, 1_000)); + + unblockPartitionRead(); + + deactivateFut.get(getTestTimeout()); + + assertFalse(F.isEmpty(idleFut.get(getTestTimeout()).exceptions())); + + grid(0).cluster().state(ClusterState.ACTIVE); + + assertEquals(KEYS, grid(0).cache(DEFAULT_CACHE_NAME).size()); + assertNull(failureHnd.failureContext()); + } + + /** */ + @Test + public void testWithNodeStoppage() throws Exception { + prepareCluster(); + + IgniteCache cache = grid(0).cache(DEFAULT_CACHE_NAME); + + int key = partitionKeys(cache, NOT_IDLE_PART, 1, 0).get(0); + + cache.get(key); + + IgniteInternalFuture idleFut = startIdleVerify(); + + cache.put(key, -1); + + notIdlePartReadUnblocked.countDown(); + + assertTrue(waitForCondition( + () -> grid(0).context().pools().getIdleVerifyExecutorService().getCompletedTaskCount() > 0, + getTestTimeout() + )); + + GridKernalContext kctx = grid(0).context(); + + IgniteInternalFuture stopFut = runAsync(() -> stopGrid(0, true)); + + assertTrue(waitForCondition(kctx::isStopping, getTestTimeout())); + + unblockPartitionRead(); + + stopFut.get(getTestTimeout()); + + IdleVerifyResult res = idleFut.get(getTestTimeout()); + + assertTrue(X.hasCause(F.first(res.exceptions().values()), GridNotIdleException.class)); + assertNull(failureHnd.failureContext()); + } + + /** */ + private void prepareCluster() throws Exception { + startGrid(0); + + grid(0).cluster().state(ClusterState.ACTIVE); + + IgniteCache cache = grid(0).createCache(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setAffinity(new RendezvousAffinityFunction(false, PARTS))); + + for (int k = 0; k < KEYS; k++) + cache.put(k, k); + + stopGrid(0); + + startGrid(0); + + grid(0).cluster().state(ClusterState.ACTIVE); + } + + /** */ + private IgniteInternalFuture startIdleVerify() throws Exception { + IgniteInternalFuture fut = runAsync(() -> idleVerify(grid(0), checkCrc, DEFAULT_CACHE_NAME)); + + assertTrue(partitionReadBlocked.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + + return fut; + } + + /** */ + private static void unblockPartitionRead() { + notIdlePartReadUnblocked.countDown(); + partitionReadUnblocked.countDown(); + } + + /** */ + private static class IdleVerifyThreadsBlockingFileIOFactory implements FileIOFactory { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private final FileIOFactory delegate = new RandomAccessFileIOFactory(); + + /** {@inheritDoc} */ + @Override public FileIO create(File file, OpenOption... modes) throws IOException { + return new FileIODecorator(delegate.create(file, modes)) { + @Override public int readFully(ByteBuffer destBuf, long position) throws IOException { + int res = super.readFully(destBuf, position); + + if (Thread.currentThread().getName().startsWith(POOL_NAME_IDLE_VERIFY)) { + partitionReadBlocked.countDown(); + + U.awaitQuiet(file.getName().equals(partitionFileName(NOT_IDLE_PART)) + ? notIdlePartReadUnblocked + : partitionReadUnblocked); + } + + return res; + } + }; + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java index a5ee0b55f5cee..543cb07846ae0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; import java.io.File; +import java.io.Serializable; import java.nio.file.DirectoryStream; import java.nio.file.Path; import java.nio.file.Paths; @@ -42,6 +43,7 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.management.api.NoArg; import org.apache.ignite.internal.management.snapshot.SnapshotStatusTask; +import org.apache.ignite.internal.management.snapshot.SnapshotStatusTask.SnapshotOperation; import org.apache.ignite.internal.processors.rollingupgrade.AbstractRollingUpgradeTest; import org.apache.ignite.internal.util.distributed.SingleNodeMessage; import org.apache.ignite.internal.util.typedef.F; @@ -49,6 +51,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorTaskArgument; import org.apache.ignite.internal.visor.VisorTaskResult; +import org.apache.ignite.lang.IgniteFuture; import org.jetbrains.annotations.Nullable; import org.junit.Test; @@ -56,6 +59,7 @@ import static org.apache.ignite.internal.TestRecordingCommunicationSpi.spi; import static org.apache.ignite.internal.processors.rollingupgrade.feature.ManagementApiVersionValidationTest.withCoreVersion; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_METAS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.END_SNAPSHOT; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RU_PREPARE_VERSION_FINALIZATION; import static org.apache.ignite.testframework.GridTestUtils.runAsync; import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; @@ -117,8 +121,68 @@ public class IgniteClusterSnapshotCheckStatusRollingUpgradeTest extends Abstract /** */ @Test public void testClientOfThePreviousVersion() throws Exception { + doTestClientOfVersion(MASTER_VER_REF, "2.18.0"); + } + + /** */ + @Test + public void testClientOfTheNextVersion() throws Exception { + doTestClientOfVersion("2.18.0", MASTER_VER_REF); + } + + /** */ + @Test + public void testClientOfTheNextVersionWithCreateInProgress() throws Exception { + for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) + startGrid(i, "2.18.0", false); + + grid(0).cluster().state(ClusterState.ACTIVE); + + createCache(0); + + // Blocks the finalization of the create operation to keep it in progress while the status is requested. + for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) { + spi(grid(i)).blockMessages((node, msg) -> + msg instanceof SingleNodeMessage snm && snm.type() == END_SNAPSHOT.ordinal()); + } + + IgniteFuture createFut = snp(0).createSnapshot(SNP_NAME); + + try { + assertTrue("Snapshot create operation has not started.", waitForCondition(() -> { + for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) { + if (snp(i).currentCreateRequest() == null) + return false; + } + + return true; + }, 10_000L)); + + Collection nodes = Collections.singleton(nodeId(0)); + + Collection statuses = + snapshotStatusTaskWithClient(new VisorTaskArgument<>(nodes, new NoArg(), false)); + + assertEquals(1, statuses.size()); + + SnapshotStatusTask.SnapshotOperationStatus status = F.first(statuses); + + assertEquals(SnapshotOperation.CREATE, status.operation()); + assertEquals(SNP_NAME, status.name()); + assertEquals(ALL_GRIDS - CLIENTS, status.progress().size()); + } + finally { + for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) + spi(grid(i)).stopBlock(); + } + + createFut.get(10_000L); + } + + /** */ + private void doTestClientOfVersion(String clusterVer, String clientVer) throws Exception { for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) - startGrid(i, "2.19.4", false); + startGrid(i, clusterVer, false); grid(0).cluster().state(ClusterState.ACTIVE); @@ -126,10 +190,9 @@ public void testClientOfThePreviousVersion() throws Exception { Collection nodes = Collections.singleton(nodeId(0)); - // Client not supporting the snapshot check status. - VisorTaskArgument vtArg = withCoreVersion("2.18.0", () -> new VisorTaskArgument<>(nodes, new NoArg(), false)); + VisorTaskArgument vtArg = withCoreVersion(clientVer, () -> new VisorTaskArgument<>(nodes, new NoArg(), false)); - ensureSnapshotCheckStatus(false, vtArg); + validateSnapshotCheckStatusFeature(false, vtArg); } /** */ @@ -144,14 +207,14 @@ public void testParallelRollingUpgradeInProgress() throws Exception { createCacheAndSnapshot(testNodeIx); - ensureSnapshotCheckStatus(false); + validateSnapshotCheckStatusFeature(false); ru(grid(testNodeIx)).enableVersionUpgrade(); for (int i = 0; i < ALL_GRIDS; i++) { assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); - upgradeNodeVersion(i, "2.18.0", "2.19.4"); + upgradeNodeVersion(i, "2.18.0", MASTER_VER_REF); } spi(grid(testNodeIx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && @@ -161,7 +224,7 @@ public void testParallelRollingUpgradeInProgress() throws Exception { assertTrue(spi(grid(testNodeIx)).waitForBlocked(1, getTestTimeout())); - ensureSnapshotCheckStatus(false); + validateSnapshotCheckStatusFeature(false); spi(grid(testNodeIx)).stopBlock(); @@ -175,12 +238,12 @@ public void testParallelRollingUpgradeInProgress() throws Exception { assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout())); } - ensureSnapshotCheckStatus(true); + validateSnapshotCheckStatusFeature(true); } /** */ @Test - public void testSnapshotListFeature() throws Exception { + public void testSnapshotStatusFeature() throws Exception { for (int i = 0; i < ALL_GRIDS; i++) startGrid(i, "2.18.0", i >= ALL_GRIDS - CLIENTS); @@ -188,17 +251,17 @@ public void testSnapshotListFeature() throws Exception { createCacheAndSnapshot(1); - ensureSnapshotCheckStatus(false); + validateSnapshotCheckStatusFeature(false); ru(grid(0)).enableVersionUpgrade(); for (int i = 0; i < ALL_GRIDS; i++) { assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); - upgradeNodeVersion(i, "2.18.0", "2.19.4"); + upgradeNodeVersion(i, "2.18.0", MASTER_VER_REF); } - ensureSnapshotCheckStatus(false); + validateSnapshotCheckStatusFeature(false); ru(grid(1)).finalizeClusterVersion(); @@ -208,11 +271,18 @@ public void testSnapshotListFeature() throws Exception { assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout())); } - ensureSnapshotCheckStatus(true); + validateSnapshotCheckStatusFeature(true); } /** */ private void createCacheAndSnapshot(int gridIdx) { + createCache(gridIdx); + + createSnapshot(gridIdx); + } + + /** Creates a replicated cache and fills it with data. */ + private void createCache(int gridIdx) { int partsCnt = 4; int keysCnt = partsCnt * 5; @@ -225,8 +295,6 @@ private void createCacheAndSnapshot(int gridIdx) { for (int i = 0; i < keysCnt; i++) ds.addData(i, i); } - - createSnapshot(gridIdx); } /** */ @@ -235,7 +303,7 @@ private void createSnapshot(int gridIdx) { } /** */ - private void ensureSnapshotCheckStatus(boolean positive, @Nullable VisorTaskArgument clientArg) throws Exception { + private void validateSnapshotCheckStatusFeature(boolean supports, @Nullable VisorTaskArgument clientArg) throws Exception { int blockNodeIdx = 1; spi(grid(blockNodeIdx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && @@ -245,24 +313,26 @@ private void ensureSnapshotCheckStatus(boolean positive, @Nullable VisorTaskArgu assertTrue(waitForCondition(() -> spi(grid(blockNodeIdx)).hasBlockedMessages(), getTestTimeout())); - Collection snpStatuses; - if (clientArg != null) { - snpStatuses = snapshotStatusCli(clientArg); + Serializable rawRes = snapshotStatusTaskWithClient(clientArg); - if (positive) - assertTrue(snpStatuses != null && snpStatuses.size() == 1); + if (supports) { + assertTrue(rawRes instanceof Collection cll && !cll.isEmpty() && + F.first(cll) instanceof SnapshotStatusTask.SnapshotOperationStatus); + } else - assertTrue(snpStatuses == null); + assertTrue(rawRes == null); } else { for (Ignite ig : G.allGrids()) { - snpStatuses = snapshotStatusTask(ig); + Serializable rawRes = snapshotStatusTaskWithGrid(ig); - if (positive) - assertTrue(snpStatuses != null && snpStatuses.size() == 1); + if (supports) { + assertTrue(rawRes instanceof Collection cll && !cll.isEmpty() && + F.first(cll) instanceof SnapshotStatusTask.SnapshotOperationStatus); + } else - assertTrue(snpStatuses == null); + assertTrue(rawRes == null); } } @@ -280,8 +350,8 @@ private void ensureSnapshotCheckStatus(boolean positive, @Nullable VisorTaskArgu } /** */ - private void ensureSnapshotCheckStatus(boolean positive) throws Exception { - ensureSnapshotCheckStatus(positive, null); + private void validateSnapshotCheckStatusFeature(boolean supports) throws Exception { + validateSnapshotCheckStatusFeature(supports, null); } /** */ @@ -290,7 +360,7 @@ private IgniteSnapshotManager snp(int gridIdx) { } /** */ - private static T snapshotStatusTask(Ignite grid) throws Exception { + private static T snapshotStatusTaskWithGrid(Ignite grid) throws Exception { Collection nodes = grid.cluster().nodes().stream().map(ClusterNode::id).toList(); VisorTaskArgument vtArg = new VisorTaskArgument<>(nodes, new NoArg(), false); @@ -299,7 +369,7 @@ private static T snapshotStatusTask(Ignite grid) throws Exception { } /** */ - private T snapshotStatusCli(VisorTaskArgument vtArg) throws Exception { + private T snapshotStatusTaskWithClient(VisorTaskArgument vtArg) throws Exception { String taskName = SnapshotStatusTask.class.getName(); try (IgniteClient cli = Ignition.startClient(new ClientConfiguration().setAddresses(Config.SERVER))) { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java index cdb30ae79b828..a6538f646afcf 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java @@ -328,7 +328,7 @@ private void checkSnapshotStatus(boolean isCreating, boolean isRestoring, boolea String status = bean.status(); if (!isCreating && !isRestoring) { - assertContains(log, status, "There is no create or restore snapshot operation in progress"); + assertContains(log, status, SnapshotMXBeanImpl.NO_CREATE_OR_RESTORE_IN_PROGRESS_MSG); continue; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java index 56972144e6ca4..ab612e766c57d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java @@ -89,7 +89,6 @@ * 2.19.1 0-1 * 2.19.2 0-2 * 2.19.3 0-2,6 - * 2.19.4 0-1 * 2.20.0 2-5 * 2.20.1 2-6 * 2.21.0 6 diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_4.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_4.java deleted file mode 100644 index 1448c0adb263b..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_4.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.rollingupgrade.feature; - -/** */ -public class TestIgniteReleaseFeatures_2_19_4 { - /** */ - public static final IgniteFeature ROLLING_UPGRADE_FEATURE = TestIgniteReleaseFeatures_2_18_0.ROLLING_UPGRADE_FEATURE; - - /** */ - public static final IgniteFeature SNAPSHOT_CHECK_STATUS_FEATURE = CoreFeatureRegistry.SNAPSHOT_CHECK_STATUS_FEATURE; -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridByteArrayWrapper.java b/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridByteArrayWrapper.java deleted file mode 100644 index 2ae75c68acb42..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridByteArrayWrapper.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap; - -import java.util.Arrays; - -/** - * Array wrapper for correct equals and hash code. - */ -public class GridByteArrayWrapper { - /** Wrapped array. */ - private byte[] arr; - - /** - * Constructor. - * - * @param arr Wrapped array. - */ - public GridByteArrayWrapper(byte[] arr) { - this.arr = arr; - } - - /** - * @return Wrapped byte array. - */ - public byte[] array() { - return arr; - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (this == o) - return true; - - if (!(o instanceof GridByteArrayWrapper)) - return false; - - GridByteArrayWrapper that = (GridByteArrayWrapper)o; - - return Arrays.equals(arr, that.arr); - - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return Arrays.hashCode(arr); - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapMapAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapMapAbstractSelfTest.java deleted file mode 100644 index 40b429a28fb9f..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapMapAbstractSelfTest.java +++ /dev/null @@ -1,843 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap; - -import java.util.HashMap; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.util.lang.GridCloseableIterator; -import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMap; -import org.apache.ignite.internal.util.typedef.X; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.lang.IgniteBiTuple; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.junit.Test; - -/** - * Tests off-heap map. - */ -public abstract class GridOffHeapMapAbstractSelfTest extends GridCommonAbstractTest { - /** Random. */ - private static final Random RAND = new Random(); - - /** Unsafe map. */ - private GridOffHeapMap map; - - /** */ - protected float load = 0.75f; - - /** */ - protected int initCap = 100; - - /** */ - protected int concurrency = 16; - - /** */ - protected short lruStripes = 16; - - /** */ - protected GridOffHeapEvictListener evictLsnr; - - /** */ - protected long mem = 20L * 1024 * 1024; - - /** */ - protected int loadCnt = 100000; - - /** - * - */ - protected GridOffHeapMapAbstractSelfTest() { - super(false); - } - - /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - //resetLog4j(Level.INFO, true, false, ""); - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - if (map != null) - map.destruct(); - } - - /** - * @return New map. - */ - protected abstract GridOffHeapMap newMap(); - - /** - * - * @return New Object. - */ - private String string() { - String key = ""; - - for (int i = 0; i < 3; i++) - key += RAND.nextLong(); - - return key; - } - - /** - * @param len Length of byte array. - * @return Random byte array. - */ - private byte[] bytes(int len) { - byte[] b = new byte[len]; - - RAND.nextBytes(b); - - return b; - } - - /** - * @param key Key. - * @return Hash. - */ - private int hash(Object key) { - return hash(key.hashCode()); - } - - /** - * @param h Hashcode. - * @return Hash. - */ - private int hash(int h) { - // Apply base step of MurmurHash; see http://code.google.com/p/smhasher/ - // Despite two multiplies, this is often faster than others - // with comparable bit-spread properties. - h ^= h >>> 16; - h *= 0x85ebca6b; - h ^= h >>> 13; - h *= 0xc2b2ae35; - - return (h >>> 16) ^ h; - } - - /** - * @throws Exception If failed. - */ - @Test - public void testInsert() throws Exception { - map = newMap(); - - for (int i = 0; i < 10; i++) { - String key = string(); - String val = string(); - - map.insert(hash(key), key.getBytes(), val.getBytes()); - - assertTrue("Failed to insert for index: " + i, map.contains(hash(key), key.getBytes())); - assertEquals(val, new String(map.get(hash(key), key.getBytes()))); - assertEquals(i + 1, map.totalSize()); - } - - assert map.totalSize() == 10; - } - - /** - * @throws Exception If failed. - */ - @Test - public void testRehash() throws Exception { - initCap = 10; - - map = newMap(); - - Map kv = new HashMap<>(10); - - for (int i = 0; i < 10; i++) - kv.put(string(), string()); - - for (Map.Entry e : kv.entrySet()) { - String key = e.getKey(); - String val = e.getValue(); - - map.insert(hash(key), key.getBytes(), val.getBytes()); - - assertTrue(map.contains(hash(key), key.getBytes())); - assertEquals(val, new String(map.get(hash(key), key.getBytes()))); - } - - for (Map.Entry e : kv.entrySet()) { - String key = e.getKey(); - String val = e.getValue(); - - byte[] valBytes = map.get(hash(key), key.getBytes()); - - assertNotNull(valBytes); - assertEquals(val, new String(valBytes)); - } - - assert map.totalSize() == 10; - } - - /** - * @throws Exception If failed. - */ - @Test - public void testGet() throws Exception { - map = newMap(); - - for (int i = 0; i < 10; i++) { - String key = string(); - String val = string(); - - map.insert(hash(key), key.getBytes(), val.getBytes()); - - assertTrue(map.contains(hash(key), key.getBytes())); - assertEquals(val, new String(map.get(hash(key), key.getBytes()))); - assertEquals(i + 1, map.totalSize()); - } - - assert map.totalSize() == 10; - } - - /** - * @throws Exception If failed. - */ - @Test - public void testPut1() throws Exception { - map = newMap(); - - for (int i = 0; i < 10; i++) { - String key = string(); - String val = string(); - - assertTrue(map.put(hash(key), key.getBytes(), val.getBytes())); - - assertTrue(map.contains(hash(key), key.getBytes())); - assertEquals(val, new String(map.get(hash(key), key.getBytes()))); - assertEquals(i + 1, map.totalSize()); - } - - assertEquals(10, map.totalSize()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testPut2() throws Exception { - map = newMap(); - - for (int i = 0; i < 10; i++) { - String key = string(); - - String val1 = string(); - String val2 = string(); - - assertTrue(map.put(hash(key), key.getBytes(), val1.getBytes())); - - assertTrue(map.contains(hash(key), key.getBytes())); - assertEquals(val1, new String(map.get(hash(key), key.getBytes()))); - assertEquals(i + 1, map.totalSize()); - - assertFalse(map.put(hash(key), key.getBytes(), val2.getBytes())); - - assertTrue(map.contains(hash(key), key.getBytes())); - assertEquals(val2, new String(map.get(hash(key), key.getBytes()))); - assertEquals(i + 1, map.totalSize()); - } - - assertEquals(10, map.totalSize()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testRemove() throws Exception { - map = newMap(); - - for (int i = 0; i < 10; i++) { - String key = string(); - String val = string(); - - assertTrue(map.put(hash(key), key.getBytes(), val.getBytes())); - - assertTrue(map.contains(hash(key), key.getBytes())); - assertNotNull(map.get(hash(key), key.getBytes())); - assertEquals(new String(map.get(hash(key), key.getBytes())), val); - assertEquals(1, map.totalSize()); - - byte[] val2 = map.remove(hash(key), key.getBytes()); - - assertNotNull(val2); - assertEquals(val, new String(val2)); - assertFalse(map.contains(hash(key), key.getBytes())); - assertNull(map.get(hash(key), key.getBytes())); - assertEquals(0, map.totalSize()); - } - - assertEquals(0, map.totalSize()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testRemovex() throws Exception { - map = newMap(); - - for (int i = 0; i < 10; i++) { - String key = string(); - String val = string(); - - assertTrue(map.put(hash(key), key.getBytes(), val.getBytes())); - - assertTrue(map.contains(hash(key), key.getBytes())); - assertNotNull(map.get(hash(key), key.getBytes())); - assertEquals(new String(map.get(hash(key), key.getBytes())), val); - assertEquals(1, map.totalSize()); - - boolean rmvd = map.removex(hash(key), key.getBytes()); - - assertTrue(rmvd); - assertFalse(map.contains(hash(key), key.getBytes())); - assertNull(map.get(hash(key), key.getBytes())); - assertEquals(0, map.totalSize()); - } - - assertEquals(0, map.totalSize()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testIterator() throws Exception { - initCap = 10; - - map = newMap(); - - final AtomicInteger rehashes = new AtomicInteger(); - final AtomicInteger releases = new AtomicInteger(); - - map.eventListener(new GridOffHeapEventListener() { - @Override public void onEvent(GridOffHeapEvent evt) { - switch (evt) { - case REHASH: - rehashes.incrementAndGet(); - break; - case RELEASE: - releases.incrementAndGet(); - break; - - default: // No-op. - } - } - }); - - int max = 1024; - - Map m = new HashMap<>(max); - - for (int i = 0; i < max; i++) { - String key = string(); - String val = string(); - - // info("Storing [i=" + i + ", key=" + key + ", val=" + val + ']'); - - assertTrue(map.put(hash(key), key.getBytes(), val.getBytes())); - - assertTrue(map.contains(hash(key), key.getBytes())); - assertNotNull(map.get(hash(key), key.getBytes())); - assertEquals(new String(map.get(hash(key), key.getBytes())), val); - - m.put(key, val); - - int cnt = 0; - - try (GridCloseableIterator> it = map.iterator()) { - while (it.hasNext()) { - IgniteBiTuple t = it.next(); - - String k = new String(t.get1()); - String v = new String(t.get2()); - - // info("Entry [k=" + k + ", v=" + v + ']'); - - assertEquals(m.get(k), v); - - cnt++; - } - } - - assertEquals(map.totalSize(), cnt); - } - - assertEquals(max, map.totalSize()); - - info("Stats [size=" + map.totalSize() + ", rehashes=" + rehashes + ", releases=" + releases + ']'); - - assertTrue(rehashes.get() > 0); - assertEquals(rehashes.get(), releases.get()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testIteratorMultithreaded() throws Exception { - initCap = 10; - - map = newMap(); - - final AtomicInteger rehashes = new AtomicInteger(); - final AtomicInteger releases = new AtomicInteger(); - - map.eventListener(new GridOffHeapEventListener() { - @Override public void onEvent(GridOffHeapEvent evt) { - switch (evt) { - case REHASH: - rehashes.incrementAndGet(); - break; - case RELEASE: - releases.incrementAndGet(); - break; - - default: // No-op. - } - } - }); - - final int max = 1024; - - int threads = 5; - - final Map m = new ConcurrentHashMap<>(max); - - multithreaded(new Callable() { - @Override public Object call() throws Exception { - for (int i = 0; i < max; i++) { - String key = string(); - String val = string(); - - // info("Storing [i=" + i + ", key=" + key + ", val=" + val + ']'); - - m.put(key, val); - - assertTrue(map.put(hash(key), key.getBytes(), val.getBytes())); - - assertTrue(map.contains(hash(key), key.getBytes())); - assertNotNull(map.get(hash(key), key.getBytes())); - assertEquals(new String(map.get(hash(key), key.getBytes())), val); - - try (GridCloseableIterator> it = map.iterator()) { - while (it.hasNext()) { - IgniteBiTuple t = it.next(); - - String k = new String(t.get1()); - String v = new String(t.get2()); - - // info("Entry [k=" + k + ", v=" + v + ']'); - - assertEquals(m.get(k), v); - } - } - } - - return null; - } - }, threads); - - assertEquals(max * threads, map.totalSize()); - - info("Stats [size=" + map.totalSize() + ", rehashes=" + rehashes + ", releases=" + releases + ']'); - - assertTrue(rehashes.get() > 0); - assertEquals(rehashes.get(), releases.get()); - } - - /** - * - */ - @Test - public void testInsertLoad() { - map = newMap(); - - Map m = new HashMap<>(); - - for (int i = 0; i < loadCnt; i++) - m.put(string(), string()); - - int cnt = 0; - - for (Map.Entry e : m.entrySet()) { - String key = e.getKey(); - String val = e.getValue(); - - try { - map.insert(hash(key), key.getBytes(), val.getBytes()); - } - catch (GridOffHeapOutOfMemoryException ex) { - error("Map put failed for count: " + cnt, ex); - - throw ex; - } - - assertTrue(map.contains(hash(key), key.getBytes())); - assertNotNull(map.get(hash(key), key.getBytes())); - assertEquals(new String(map.get(hash(key), key.getBytes())), val); - assertEquals(++cnt, map.totalSize()); - } - } - - - /** - * - */ - @Test - public void testPutLoad() { - map = newMap(); - - Map m = new HashMap<>(); - - for (int i = 0; i < loadCnt; i++) - m.put(string(), string()); - - int cnt = 0; - - for (Map.Entry e : m.entrySet()) { - String key = e.getKey(); - String val = e.getValue(); - - try { - assertTrue(map.put(hash(key), key.getBytes(), val.getBytes())); - assertTrue(map.contains(hash(key), key.getBytes())); - assertFalse(map.put(hash(key), key.getBytes(), val.getBytes())); - } - catch (GridOffHeapOutOfMemoryException ex) { - error("Map put failed for count: " + cnt, ex); - - throw ex; - } - - assertTrue(map.contains(hash(key), key.getBytes())); - assertNotNull(map.get(hash(key), key.getBytes())); - assertEquals(new String(map.get(hash(key), key.getBytes())), val); - assertEquals(++cnt, map.totalSize()); - } - } - - /** - * - */ - @Test - public void testLru1() { - lruStripes = 1; - mem = 10; - - final AtomicInteger evictCnt = new AtomicInteger(); - - evictLsnr = new GridOffHeapEvictListener() { - @Override public void onEvict(int part, int hash, byte[] keyBytes, byte[] valBytes) { - String key = new String(keyBytes); - - info("Evicted key: " + key); - - evictCnt.incrementAndGet(); - } - - @Override public boolean removeEvicted() { - return true; - } - }; - - map = newMap(); - - for (int i = 0; i < 10; i++) { - String key = string(); - - byte[] keyBytes = key.getBytes(); - byte[] valBytes = bytes(100); - - map.insert(hash(key), keyBytes, valBytes); - - info("Evicted: " + evictCnt); - - assertEquals(1, evictCnt.get()); - assertEquals(0, map.totalSize()); - - assertTrue(evictCnt.compareAndSet(1, 0)); - } - } - - /** - * - */ - @Test - public void testLru2() { - mem = 1000 + 64 * 16; // Add segment size. - - lruStripes = 6; - concurrency = 8; - - final AtomicInteger evictCnt = new AtomicInteger(); - - evictLsnr = new GridOffHeapEvictListener() { - @Override public void onEvict(int part, int hash, byte[] k, byte[] v) { - evictCnt.incrementAndGet(); - } - - @Override public boolean removeEvicted() { - return true; - } - }; - - map = newMap(); - - for (int i = 0; i < 10000; i++) { - String key = string(); - - byte[] keyBytes = key.getBytes(); - byte[] valBytes = bytes(100); - - map.insert(hash(key), keyBytes, valBytes); - } - - assertTrue(evictCnt.get() > 10); - assertTrue("Invalid map free size [size=" + map.freeSize() + ", evictCnt=" + evictCnt + ']', - map.freeSize() >= 0); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testLruMultithreaded() throws Exception { - mem = 1000 + 64 * 16; // Add segment size. - - lruStripes = 3; - concurrency = 8; - - final AtomicInteger evictCnt = new AtomicInteger(); - - final AtomicInteger cnt = new AtomicInteger(); - - evictLsnr = new GridOffHeapEvictListener() { - @Override public void onEvict(int part, int hash, byte[] k, byte[] v) { - evictCnt.incrementAndGet(); - } - - @Override public boolean removeEvicted() { - return true; - } - }; - - map = newMap(); - - multithreaded(new Runnable() { - @Override public void run() { - for (int i = 0; i < 10000; i++) { - String key = string(); - - byte[] keyBytes = key.getBytes(); - byte[] valBytes = bytes(100); - - assert !map.contains(hash(key), keyBytes); - - map.insert(hash(key), keyBytes, valBytes); - - int n; - - if ((n = cnt.incrementAndGet()) % 10000 == 0) - info("Inserted entries: " + n); - } - } - }, 10); - - info("Map stats [evicted=" + evictCnt + ", size=" + map.totalSize() + ", allocated=" + map.allocatedSize() + - ", freeSize=" + map.freeSize() + ']'); - - assertTrue(map.freeSize() >= 0); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testIteratorAfterRehash() throws Exception { - mem = 0; - initCap = 10; - concurrency = 2; - - map = newMap(); - - final CountDownLatch startLatch = new CountDownLatch(1); - - final AtomicBoolean run = new AtomicBoolean(true); - - IgniteInternalFuture itFut = multithreadedAsync(new Runnable() { - @Override public void run() { - try { - startLatch.await(); - - while (run.get()) { - GridCloseableIterator> it = map.iterator(); - - while (it.hasNext()) - it.next(); - - it.close(); - } - } - catch (IgniteCheckedException e) { - e.printStackTrace(); - } - catch (InterruptedException e) { - e.printStackTrace(); - - Thread.currentThread().interrupt(); - } - } - }, 1); - - IgniteInternalFuture putFut = multithreadedAsync(new Runnable() { - @Override public void run() { - try { - startLatch.await(); - - Random rnd = new Random(); - - for (int size = 50; size < 5000; size++) { - int valSize = rnd.nextInt(50) + 1; - - for (int i = 0; i < size; i++) - map.put(i, U.intToBytes(i), new byte[valSize]); - } - } - catch (InterruptedException e) { - e.printStackTrace(); - - Thread.currentThread().interrupt(); - } - } - }, 1); - - startLatch.countDown(); - - putFut.get(); - - run.set(false); - - itFut.get(); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testMultithreadedOps() throws Exception { - mem = 1512; // Small enough for evictions. - - lruStripes = 3; - concurrency = 8; - initCap = 256; - - map = newMap(); - - long zeroAllocated = map.allocatedSize(); - - X.println("Empty map offheap size: " + zeroAllocated); - - final AtomicBoolean stop = new AtomicBoolean(); - - final byte[][] keys = new byte[127][16]; - - Random rnd = new Random(); - - for (int i = 0; i < keys.length; i++) { - rnd.nextBytes(keys[i]); - - keys[i][0] = (byte)i; // hash - } - - IgniteInternalFuture fut = multithreadedAsync(new Callable() { - @Override public Void call() throws IgniteCheckedException { - Random rnd = new Random(); - - while (!stop.get()) { - byte[] key = keys[rnd.nextInt(keys.length)]; - - int hash = key[0]; - - byte[] val = new byte[1 + rnd.nextInt(11)]; - rnd.nextBytes(val); - - switch (rnd.nextInt(5)) { - case 0: - map.put(hash, key, val); - - break; - case 1: - map.remove(hash, key); - - break; - case 2: - map.contains(hash, key); - - break; - case 3: - map.get(hash, key); - - break; - case 4: -// map.insert(hash, key, val); -// break; - case 5: - GridCloseableIterator> iter = map.iterator(); - - while (iter.hasNext()) - assertNotNull(iter.next()); - - iter.close(); - - break; - } - } - - return null; - } - }, 49); - - Thread.sleep(60000); - - stop.set(true); - - fut.get(); - - for (byte[] key : keys) - map.remove(key[0], key); - - assertEquals(0, map.totalSize()); - assertEquals(0, ((GridUnsafeMap)map).lruSize()); - - assertEquals(zeroAllocated, map.allocatedSize()); - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapMapPerformanceAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapMapPerformanceAbstractTest.java deleted file mode 100644 index 34b7f102f9636..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapMapPerformanceAbstractTest.java +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap; - -import java.util.HashMap; -import java.util.Map; -import java.util.Random; -import org.apache.ignite.internal.util.typedef.T3; -import org.apache.ignite.internal.util.typedef.X; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.junit.Test; - -/** - * Tests off-heap map. - */ -public abstract class GridOffHeapMapPerformanceAbstractTest extends GridCommonAbstractTest { - /** Random. */ - private static final Random RAND = new Random(); - - /** */ - protected static final int LOAD_CNT = 1024 * 1024; - - /** Sample map. */ - private static Map> kvMap = - new HashMap<>(LOAD_CNT); - - /** Unsafe map. */ - private GridOffHeapMap map; - - /** */ - protected float load = 0.75f; - - /** */ - protected int initCap = 1024 * 1024 * 1024; - - /** */ - protected int concurrency = 16; - - /** */ - protected short lruStripes = 16; - - /** */ - protected GridOffHeapEvictListener evictClo; - - /** */ - protected long mem = 12L * 1024L * 1024L * 1024L; - - /** */ - protected long dur = 60 * 1000; //2 * 60 * 60 * 1000; - - /** - * - */ - protected GridOffHeapMapPerformanceAbstractTest() { - super(false); - } - - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - map = newMap(); - - if (kvMap.isEmpty()) - for (int i = 0; i < LOAD_CNT; i++) { - String k = string(); - String v = string(); - - kvMap.put(k, new T3<>(v, k.getBytes(), v.getBytes())); - } - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - if (map != null) - map.destruct(); - } - - /** - * @return New map. - */ - protected abstract GridOffHeapMap newMap(); - - /** - * @param key Key. - * @return Hash. - */ - private int hash(Object key) { - return hash(key.hashCode()); - } - - /** - * @param h Hashcode. - * @return Hash. - */ - private int hash(int h) { - // Apply base step of MurmurHash; see http://code.google.com/p/smhasher/ - // Despite two multiplies, this is often faster than others - // with comparable bit-spread properties. - h ^= h >>> 16; - h *= 0x85ebca6b; - h ^= h >>> 13; - h *= 0xc2b2ae35; - - return (h >>> 16) ^ h; - } - - /** - * - * @return New Object. - */ - private String string() { - String key = ""; - - for (int i = 0; i < 3; i++) - key += RAND.nextLong(); - - return key; - } - - /** - * Test plain hash map. - */ - @Test - public void testHashMapPutRemove() { - Map map = new HashMap<>(LOAD_CNT); - - info("Starting standard HashMap performance test..."); - - long cnt = 0; - - long start = System.currentTimeMillis(); - - boolean rmv = false; - - boolean done = false; - - while (!done) { - for (Map.Entry> e : kvMap.entrySet()) { - String key = e.getKey(); - T3 t = e.getValue(); - - try { - if (rmv) - map.remove(key); - else - map.put(key, t.get1()); - } - catch (GridOffHeapOutOfMemoryException ex) { - error("Map put failed for count: " + cnt, ex); - - throw ex; - } - - if (cnt > 0 && cnt % 10000000 == 0) { - long cur = System.currentTimeMillis(); - - long throughput = cnt * 1000 / (cur - start); - - X.println("Insert [cnt=" + cnt + ", ops/sec=" + throughput + ']'); - - if ((cur - start) > dur) { - done = true; - - break; - } - } - - cnt++; - } - - rmv = !rmv; - } - } - - /** - * - */ - @Test - public void testInsertRemoveLoad() { - info("Starting insert performance test..."); - - long cnt = 0; - - long start = System.currentTimeMillis(); - - boolean rmv = false; - - boolean done = false; - - while (!done) { - for (Map.Entry> e : kvMap.entrySet()) { - String key = e.getKey(); - T3 t = e.getValue(); - - try { - if (rmv) - map.remove(hash(key), t.get2()); - else - map.insert(hash(key), t.get2(), t.get3()); - } - catch (GridOffHeapOutOfMemoryException ex) { - error("Map put failed for count: " + cnt, ex); - - throw ex; - } - - if (cnt > 0 && cnt % 10000000 == 0) { - long cur = System.currentTimeMillis(); - - long throughput = cnt * 1000 / (cur - start); - - X.println("Insert [cnt=" + cnt + ", ops/sec=" + throughput + ']'); - - if ((cur - start) > dur) { - done = true; - - break; - } - } - - cnt++; - } - - rmv = !rmv; - } - } - - - /** - * - */ - @Test - public void testPutRemoveLoad() { - info("Starting put performance test..."); - - long cnt = 0; - - long start = System.currentTimeMillis(); - - boolean rmv = false; - - boolean done = false; - - while (!done) { - for (Map.Entry> e : kvMap.entrySet()) { - String key = e.getKey(); - T3 t = e.getValue(); - - try { - if (rmv) - map.remove(hash(key), t.get2()); - else - map.put(hash(key), t.get2(), t.get3()); - } - catch (GridOffHeapOutOfMemoryException ex) { - error("Map put failed for count: " + cnt, ex); - - throw ex; - } - - if (cnt > 0 && cnt % 10000000 == 0) { - long cur = System.currentTimeMillis(); - - long throughput = cnt * 1000 / (cur - start); - - X.println("Put [cnt=" + cnt + ", ops/sec=" + throughput + ']'); - - if ((cur - start) > dur) { - done = true; - - break; - } - } - - cnt++; - } - - rmv = cnt % 3 == 0; - } - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapPartitionedMapAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapPartitionedMapAbstractSelfTest.java deleted file mode 100644 index ae3a0c95e82a3..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapPartitionedMapAbstractSelfTest.java +++ /dev/null @@ -1,1178 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap; - -import java.util.HashMap; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.cache.affinity.AffinityFunction; -import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; -import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.util.lang.GridCloseableIterator; -import org.apache.ignite.internal.util.lang.GridTuple; -import org.apache.ignite.lang.IgniteBiTuple; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.junit.Test; - -/** - * Tests off-heap map. - */ -public abstract class GridOffHeapPartitionedMapAbstractSelfTest extends GridCommonAbstractTest { - /** Random. */ - private static final Random RAND = new Random(); - - /** Unsafe map. */ - private GridOffHeapPartitionedMap map; - - /** */ - protected float load = 0.75f; - - /** */ - protected int initCap = 100; - - /** */ - protected int concurrency = 16; - - /** */ - protected short lruStripes = 16; - - /** */ - protected GridOffHeapEvictListener evictLsnr; - - /** */ - protected long mem = 20L * 1024 * 1024; - - /** */ - protected int loadCnt = 100000; - - /** */ - protected int parts = 17; - - /** - * - */ - protected GridOffHeapPartitionedMapAbstractSelfTest() { - super(false); - } - - /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - //resetLog4j(Level.INFO, true, false, ""); - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - if (map != null) - map.destruct(); - } - - /** - * @return New map. - */ - protected abstract GridOffHeapPartitionedMap newMap(); - - /** - * @param key Key. - * @return Hash. - */ - private int hash(Object key) { - return hash(key.hashCode()); - } - - /** - * @param h Hash code. - * @return Hash. - */ - private int hash(int h) { - // Apply base step of MurmurHash; see http://code.google.com/p/smhasher/ - // Despite two multiplies, this is often faster than others - // with comparable bit-spread properties. - h ^= h >>> 16; - h *= 0x85ebca6b; - h ^= h >>> 13; - h *= 0xc2b2ae35; - - return (h >>> 16) ^ h; - } - - /** - * - * @return New Object. - */ - private String string() { - String key = ""; - - for (int i = 0; i < 3; i++) - key += RAND.nextLong(); - - return key; - } - - /** - * @param len Length of byte array. - * @return Random byte array. - */ - private byte[] bytes(int len) { - byte[] b = new byte[len]; - - RAND.nextBytes(b); - - return b; - } - - /** - * @throws Exception If failed. - */ - @Test - public void testInsert() throws Exception { - map = newMap(); - - for (int p = 0; p < parts; p++) { - for (int i = 0; i < 10; i++) { - String key = string(); - String val = string(); - - map.insert(p, hash(key), key.getBytes(), val.getBytes()); - - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertEquals(val, new String(map.get(p, hash(key), key.getBytes()))); - assertEquals(10 * p + (i + 1), map.size()); - } - } - - assertEquals(parts * 10, map.size()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testRehash() throws Exception { - initCap = 10; - - map = newMap(); - - for (int p = 0; p < parts; p++) { - Map kv = new HashMap<>(10); - - for (int i = 0; i < 10; i++) - kv.put(string(), string()); - - for (Map.Entry e : kv.entrySet()) { - String key = e.getKey(); - String val = e.getValue(); - - map.insert(p, hash(key), key.getBytes(), val.getBytes()); - - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertEquals(val, new String(map.get(p, hash(key), key.getBytes()))); - } - - for (Map.Entry e : kv.entrySet()) { - String key = e.getKey(); - String val = e.getValue(); - - byte[] valBytes = map.get(p, hash(key), key.getBytes()); - - assertNotNull(valBytes); - assertEquals(val, new String(valBytes)); - } - } - - assertEquals(parts * 10, map.size()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testPointerAfterRehash() throws Exception { - initCap = 10; - - map = newMap(); - - Map kv = new HashMap<>(1000); - Map ptrs = new HashMap<>(1000); - - for (int i = 0; i < 1000; i++) - kv.put(string(), string()); - - for (Map.Entry e : kv.entrySet()) { - String key = e.getKey(); - String val = e.getValue(); - - assertTrue(map.put(1, hash(key), key.getBytes(), val.getBytes())); - - IgniteBiTuple ptr = map.valuePointer(1, hash(key), key.getBytes()); - - assertNotNull(ptr); - - assertEquals((Integer)val.getBytes().length, ptr.get2()); - - assertFalse(map.put(1, hash(key), key.getBytes(), val.getBytes())); - - ptrs.put(key, ptr.get1()); - } - - for (Map.Entry e : kv.entrySet()) { - String key = e.getKey(); - - IgniteBiTuple ptr = map.valuePointer(1, hash(key), key.getBytes()); - - assertNotNull(ptr); - - assertEquals(ptrs.get(key), ptr.get1()); - } - } - - /** - * @throws Exception If failed. - */ - @Test - public void testPutRandomKeys() throws Exception { - map = newMap(); - - AffinityFunction aff = new RendezvousAffinityFunction(parts, null); - - GridByteArrayWrapper[] keys = new GridByteArrayWrapper[512]; - Random rnd = new Random(); - - for (int i = 0; i < keys.length; i++) { - byte[] key = new byte[rnd.nextInt(64) + 1]; - rnd.nextBytes(key); - - keys[i] = new GridByteArrayWrapper(key); - } - - for (int i = 0; i < 10000; i++) { - int idx = rnd.nextInt(keys.length); - - GridByteArrayWrapper key = keys[idx]; - - map.put(aff.partition(key), key.hashCode(), key.array(), new byte[64]); - } - } - - /** - * @throws Exception If failed. - */ - @Test - public void testGet() throws Exception { - map = newMap(); - - for (int p = 0; p < parts; p++) { - for (int i = 0; i < 10; i++) { - String key = string(); - String val = string(); - - map.insert(p, hash(key), key.getBytes(), val.getBytes()); - - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertEquals(val, new String(map.get(p, hash(key), key.getBytes()))); - assertEquals(10 * p + (i + 1), map.size()); - } - } - - assertEquals(parts * 10, map.size()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testPut1() throws Exception { - map = newMap(); - - for (int p = 0; p < parts; p++) { - for (int i = 0; i < 10; i++) { - String key = string(); - String val = string(); - - assertTrue(map.put(p, hash(key), key.getBytes(), val.getBytes())); - - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertEquals(val, new String(map.get(p, hash(key), key.getBytes()))); - assertEquals(10 * p + i + 1, map.size()); - } - } - - assertEquals(parts * 10, map.size()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testPut2() throws Exception { - map = newMap(); - - for (int p = 0; p < parts; p++) { - for (int i = 0; i < 10; i++) { - String key = string(); - - String val1 = string(); - String val2 = string(); - - assertTrue(map.put(p, hash(key), key.getBytes(), val1.getBytes())); - - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertEquals(val1, new String(map.get(p, hash(key), key.getBytes()))); - assertEquals(10 * p + i + 1, map.size()); - - assertFalse(map.put(p, hash(key), key.getBytes(), val2.getBytes())); - - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertEquals(val2, new String(map.get(p, hash(key), key.getBytes()))); - assertEquals(10 * p + i + 1, map.size()); - } - } - - assertEquals(parts * 10, map.size()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testRemove() throws Exception { - map = newMap(); - - for (int p = 0; p < parts; p++) { - for (int i = 0; i < 10; i++) { - String key = string(); - String val = string(); - - assertTrue(map.put(p, hash(key), key.getBytes(), val.getBytes())); - - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertNotNull(map.get(p, hash(key), key.getBytes())); - assertEquals(new String(map.get(p, hash(key), key.getBytes())), val); - assertEquals(1, map.size()); - - byte[] val2 = map.remove(p, hash(key), key.getBytes()); - - assertNotNull(val2); - assertEquals(val, new String(val2)); - assertFalse(map.contains(p, hash(key), key.getBytes())); - assertNull(map.get(p, hash(key), key.getBytes())); - assertEquals(0, map.size()); - } - } - - assertEquals(0, map.size()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testRemovex() throws Exception { - map = newMap(); - - for (int p = 0; p < parts; p++) { - for (int i = 0; i < 10; i++) { - String key = string(); - String val = string(); - - assertTrue(map.put(p, hash(key), key.getBytes(), val.getBytes())); - - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertNotNull(map.get(p, hash(key), key.getBytes())); - assertEquals(new String(map.get(p, hash(key), key.getBytes())), val); - assertEquals(1, map.size()); - - boolean rmvd = map.removex(p, hash(key), key.getBytes()); - - assertTrue(rmvd); - assertFalse(map.contains(p, hash(key), key.getBytes())); - assertNull(map.get(p, hash(key), key.getBytes())); - assertEquals(0, map.size()); - } - } - - assertEquals(0, map.size()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testIterator() throws Exception { - initCap = 10; - - map = newMap(); - - final AtomicInteger rehashes = new AtomicInteger(); - final AtomicInteger releases = new AtomicInteger(); - - map.eventListener(new GridOffHeapEventListener() { - @Override public void onEvent(GridOffHeapEvent evt) { - switch (evt) { - case REHASH: - rehashes.incrementAndGet(); - break; - case RELEASE: - releases.incrementAndGet(); - break; - - default: // No-op. - } - } - }); - - int max = 1024; - - Map m = new HashMap<>(max * parts); - - for (int p = 0; p < parts; p++) { - for (int i = 0; i < max; i++) { - String key = string(); - String val = string(); - - // info("Storing [i=" + i + ", key=" + key + ", val=" + val + ']'); - - assertTrue(map.put(p, hash(key), key.getBytes(), val.getBytes())); - - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertNotNull(map.get(p, hash(key), key.getBytes())); - assertEquals(new String(map.get(p, hash(key), key.getBytes())), val); - - m.put(key, val); - } - } - - int cnt = 0; - - try (GridCloseableIterator> it = map.iterator()) { - while (it.hasNext()) { - IgniteBiTuple t = it.next(); - - String k = new String(t.get1()); - String v = new String(t.get2()); - - // info("Entry [k=" + k + ", v=" + v + ']'); - - assertEquals(m.get(k), v); - - cnt++; - } - } - - assertEquals(map.size(), cnt); - assertEquals(max * parts, map.size()); - - info("Stats [size=" + map.size() + ", rehashes=" + rehashes + ", releases=" + releases + ']'); - - assertTrue(rehashes.get() > 0); - assertEquals(rehashes.get(), releases.get()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testIteratorMultithreaded() throws Exception { - initCap = 10; - - map = newMap(); - - final AtomicInteger rehashes = new AtomicInteger(); - final AtomicInteger releases = new AtomicInteger(); - - map.eventListener(new GridOffHeapEventListener() { - @Override public void onEvent(GridOffHeapEvent evt) { - switch (evt) { - case REHASH: - rehashes.incrementAndGet(); - break; - case RELEASE: - releases.incrementAndGet(); - break; - - default: // No-op. - } - } - }); - - final int max = 1024; - - final ConcurrentMap m = new ConcurrentHashMap<>(max * parts); - - final AtomicInteger part = new AtomicInteger(); - - multithreaded(new Callable() { - @Override public Object call() throws Exception { - int p = part.getAndIncrement(); - - for (int i = 0; i < max; i++) { - String key = string(); - String val = string(); - - // info("Storing [i=" + i + ", key=" + key + ", val=" + val + ']'); - - String old = m.putIfAbsent(key, val); - - if (old != null) - val = old; - - assertTrue(map.put(p, hash(key), key.getBytes(), val.getBytes())); - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertNotNull(map.get(p, hash(key), key.getBytes())); - - assertEquals(new String(map.get(p, hash(key), key.getBytes())), val); - } - - try (GridCloseableIterator> it = map.iterator()) { - while (it.hasNext()) { - IgniteBiTuple t = it.next(); - - String k = new String(t.get1()); - String v = new String(t.get2()); - - // info("Entry [k=" + k + ", v=" + v + ']'); - - assertEquals(m.get(k), v); - } - } - - return null; - } - }, parts); - - int cnt = 0; - - try (GridCloseableIterator> it = map.iterator()) { - while (it.hasNext()) { - IgniteBiTuple t = it.next(); - - String k = new String(t.get1()); - String v = new String(t.get2()); - - // info("Entry [k=" + k + ", v=" + v + ']'); - - assertEquals(m.get(k), v); - - cnt++; - } - } - - assertEquals(map.size(), cnt); - assertEquals(max * parts, map.size()); - - info("Stats [size=" + map.size() + ", rehashes=" + rehashes + ", releases=" + releases + ']'); - - assertTrue(rehashes.get() > 0); - assertEquals(rehashes.get(), releases.get()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testIteratorRemoveMultithreaded() throws Exception { - initCap = 10; - - map = newMap(); - - final int max = 1024; - - final Map m = new ConcurrentHashMap<>(max * parts); - - for (int i = 0; i < max; i++) { - String key = string(); - String val = string(); - - m.put(key, val); - - map.put(0, hash(key), key.getBytes(), val.getBytes()); - } - - final AtomicBoolean running = new AtomicBoolean(true); - - IgniteInternalFuture iterFut = multithreadedAsync(new Runnable() { - @Override public void run() { - try { - while (running.get()) { - try (GridCloseableIterator> it = map.iterator()) { - while (it.hasNext()) { - IgniteBiTuple tup = it.next(); - - String key = new String(tup.get1()); - String val = new String(tup.get2()); - - String exp = m.get(key); - - assertEquals(exp, val); - } - } - } - } - catch (IgniteCheckedException e) { - fail("Unexpected exception caught: " + e); - } - } - }, 1); - - for (String key : m.keySet()) - map.remove(0, hash(key), key.getBytes()); - - running.set(false); - - iterFut.get(); - - map.destruct(); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testPartitionIterator() throws Exception { - initCap = 10; - - map = newMap(); - - final AtomicInteger rehashes = new AtomicInteger(); - final AtomicInteger releases = new AtomicInteger(); - - map.eventListener(new GridOffHeapEventListener() { - @Override public void onEvent(GridOffHeapEvent evt) { - switch (evt) { - case REHASH: - rehashes.incrementAndGet(); - break; - case RELEASE: - releases.incrementAndGet(); - break; - - default: // No-op. - } - } - }); - - int max = 1024; - - for (int p = 0; p < parts; p++) { - Map m = new HashMap<>(max); - - for (int i = 0; i < max; i++) { - String key = string(); - String val = string(); - - // info("Storing [i=" + i + ", key=" + key + ", val=" + val + ']'); - - assertTrue(map.put(p, hash(key), key.getBytes(), val.getBytes())); - - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertNotNull(map.get(p, hash(key), key.getBytes())); - assertEquals(new String(map.get(p, hash(key), key.getBytes())), val); - - m.put(key, val); - - int cnt = 0; - - try (GridCloseableIterator> it = map.iterator(p)) { - while (it.hasNext()) { - IgniteBiTuple t = it.next(); - - String k = new String(t.get1()); - String v = new String(t.get2()); - - // info("Entry [k=" + k + ", v=" + v + ']'); - - assertEquals(m.get(k), v); - - cnt++; - } - } - - assertEquals(map.size(), p * max + cnt); - } - } - - assertEquals(max * parts, map.size()); - - info("Stats [size=" + map.size() + ", rehashes=" + rehashes + ", releases=" + releases + ']'); - - assertTrue(rehashes.get() > 0); - assertEquals(rehashes.get(), releases.get()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testPartitionIteratorMultithreaded() throws Exception { - initCap = 10; - - map = newMap(); - - final AtomicInteger rehashes = new AtomicInteger(); - final AtomicInteger releases = new AtomicInteger(); - - map.eventListener(new GridOffHeapEventListener() { - @Override public void onEvent(GridOffHeapEvent evt) { - switch (evt) { - case REHASH: - rehashes.incrementAndGet(); - break; - case RELEASE: - releases.incrementAndGet(); - break; - - default: // No-op. - } - } - }); - - final int max = 64; - - int threads = 5; - - final Map m = new ConcurrentHashMap<>(max); - - multithreaded(new Callable() { - @Override public Object call() throws Exception { - for (int p = 0; p < parts; p++) { - for (int i = 0; i < max; i++) { - String key = string(); - String val = string(); - - // info("Storing [i=" + i + ", key=" + key + ", val=" + val + ']'); - - m.put(key, val); - - assertTrue(map.put(p, hash(key), key.getBytes(), val.getBytes())); - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertNotNull(map.get(p, hash(key), key.getBytes())); - assertEquals(new String(map.get(p, hash(key), key.getBytes())), val); - - try (GridCloseableIterator> it = map.iterator(p)) { - while (it.hasNext()) { - IgniteBiTuple t = it.next(); - - String k = new String(t.get1()); - String v = new String(t.get2()); - - // info("Entry [k=" + k + ", v=" + v + ']'); - - assertEquals(m.get(k), v); - } - } - } - } - - return null; - } - }, threads); - - assertEquals(max * threads * parts, map.size()); - - info("Stats [size=" + map.size() + ", rehashes=" + rehashes + ", releases=" + releases + ']'); - - assertTrue(rehashes.get() > 0); - assertEquals(rehashes.get(), releases.get()); - } - - /** - * - */ - @Test - public void testInsertLoad() { - mem = 0; // Disable LRU. - - loadCnt = 50000; - - map = newMap(); - - int cnt = 0; - - for (int p = 0; p < parts; p++) { - Map m = new HashMap<>(); - - for (int i = 0; i < loadCnt; i++) - m.put(string(), string()); - - for (Map.Entry e : m.entrySet()) { - String key = e.getKey(); - String val = e.getValue(); - - try { - map.insert(p, hash(key), key.getBytes(), val.getBytes()); - } - catch (GridOffHeapOutOfMemoryException ex) { - error("Map put failed for count: " + cnt, ex); - - throw ex; - } - - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertNotNull(map.get(p, hash(key), key.getBytes())); - assertEquals(new String(map.get(p, hash(key), key.getBytes())), val); - assertEquals(++cnt, map.size()); - } - } - } - - - /** - * - */ - @Test - public void testPutLoad() { - mem = 0; // Disable LRU. - - loadCnt = 50000; - - map = newMap(); - - int cnt = 0; - - for (int p = 0; p < parts; p++) { - Map m = new HashMap<>(); - - for (int i = 0; i < loadCnt; i++) - m.put(string(), string()); - - for (Map.Entry e : m.entrySet()) { - String key = e.getKey(); - String val = e.getValue(); - - try { - assertTrue(map.put(p, hash(key), key.getBytes(), val.getBytes())); - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertFalse(map.put(p, hash(key), key.getBytes(), val.getBytes())); - } - catch (GridOffHeapOutOfMemoryException ex) { - error("Map put failed for count: " + cnt, ex); - - throw ex; - } - - assertTrue(map.contains(p, hash(key), key.getBytes())); - assertNotNull(map.get(p, hash(key), key.getBytes())); - assertEquals(new String(map.get(p, hash(key), key.getBytes())), val); - assertEquals(++cnt, map.size()); - } - } - } - - /** - * - */ - @Test - public void testLru1() { - lruStripes = 1; - mem = 10; - - final AtomicInteger evictCnt = new AtomicInteger(); - - evictLsnr = new GridOffHeapEvictListener() { - @Override public void onEvict(int part, int hash, byte[] k, byte[] v) { - String key = new String(k); - - info("Evicted key: " + key); - - evictCnt.incrementAndGet(); - } - - @Override public boolean removeEvicted() { - return true; - } - }; - - map = newMap(); - - for (int p = 0; p < parts; p++) { - for (int i = 0; i < 10; i++) { - String key = string(); - - byte[] keyBytes = key.getBytes(); - byte[] valBytes = bytes(100); - - map.insert(p, hash(key), keyBytes, valBytes); - - info("Evicted: " + evictCnt); - - assertEquals(1, evictCnt.get()); - assertEquals(0, map.size()); - - assertTrue(evictCnt.compareAndSet(1, 0)); - } - } - } - - /** - * - */ - @Test - public void testLru2() { - mem = 1000 + 64 * 16 * parts; // Add segment size. - - lruStripes = 6; - concurrency = 8; - - final AtomicInteger evictCnt = new AtomicInteger(); - - for (int p = 0; p < parts; p++) { - evictLsnr = new GridOffHeapEvictListener() { - @Override public void onEvict(int part, int hash, byte[] k, byte[] v) { - evictCnt.incrementAndGet(); - } - - @Override public boolean removeEvicted() { - return true; - } - }; - - map = newMap(); - - for (int i = 0; i < 10000; i++) { - String key = string(); - - byte[] keyBytes = key.getBytes(); - byte[] valBytes = bytes(100); - - map.insert(p, hash(key), keyBytes, valBytes); - } - - assertTrue(evictCnt.get() > 10 * parts); - assertTrue("Invalid map free size [size=" + map.freeSize() + ", evictCnt=" + evictCnt + ']', - map.freeSize() >= 0); - } - } - - /** - * @throws Exception If failed. - */ - @Test - public void testLruMultithreaded() throws Exception { - mem = 1000 + 64 * 16 * parts; // Add segment size. - - lruStripes = 3; - concurrency = 8; - - final AtomicInteger evictCnt = new AtomicInteger(); - - final AtomicInteger cnt = new AtomicInteger(); - - evictLsnr = new GridOffHeapEvictListener() { - @Override public void onEvict(int part, int hash, byte[] k, byte[] v) { - evictCnt.incrementAndGet(); - } - - @Override public boolean removeEvicted() { - return true; - } - }; - - map = newMap(); - - assertEquals(0, map.allocatedSize()); - - multithreaded(new Runnable() { - @Override public void run() { - for (int p = 0; p < parts; p++) { - for (int i = 0; i < 10000; i++) { - String key = string(); - - byte[] keyBytes = key.getBytes(); - byte[] valBytes = bytes(100); - - assert !map.contains(p, hash(key), keyBytes); - - map.insert(p, hash(key), keyBytes, valBytes); - - int n; - - if ((n = cnt.incrementAndGet()) % 100000 == 0) - info("Inserted entries: " + n); - } - } - } - }, 10); - - assert map.allocatedSize() <= mem; - - info("Map stats [evicted=" + evictCnt + ", size=" + map.size() + ", allocated=" + map.allocatedSize() + - ", freeSize=" + map.freeSize() + ']'); - - assertTrue("Invalid map free size [size=" + map.freeSize() + ", evictCnt=" + evictCnt + ']', - map.freeSize() >= 0); - } - - /** - * - */ - @SuppressWarnings("TooBroadScope") - @Test - public void testValuePointerEvict() { - mem = 90; - - final GridTuple evicted = new GridTuple<>(); - - evictLsnr = new GridOffHeapEvictListener() { - @Override public void onEvict(int part, int hash, byte[] k, byte[] v) { - String key = new String(k); - - evicted.set(key); - } - - @Override public boolean removeEvicted() { - return true; - } - }; - - map = newMap(); - - final String k1 = "k1"; - final String k2 = "k2"; - - map.put(1, k1.hashCode(), k1.getBytes(), bytes(20)); - - assertNull(evicted.get()); - - assertNotNull(map.valuePointer(1, k1.hashCode(), k1.getBytes())); - - assertNull(evicted.get()); - - map.put(1, k2.hashCode(), k2.getBytes(), bytes(20)); - - assertEquals(k2, evicted.get()); - - evicted.set(null); - - assertTrue(map.contains(1, k1.hashCode(), k1.getBytes())); - - map.put(1, k1.hashCode(), k1.getBytes(), bytes(20)); - - assertNull(evicted.get()); - - map.put(1, k2.hashCode(), k2.getBytes(), bytes(20)); - - assertEquals(k1, evicted.get()); - - assertNull(map.valuePointer(1, k1.hashCode(), k1.getBytes())); - - evicted.set(null); - - assertNotNull(map.valuePointer(1, k2.hashCode(), k2.getBytes())); - - assertNull(evicted.get()); - - map.put(1, k1.hashCode(), k1.getBytes(), bytes(21)); - - assertEquals(k1, evicted.get()); - - evicted.set(null); - - map.put(1, k2.hashCode(), k2.getBytes(), bytes(21)); - } - - /** - * - */ - @SuppressWarnings("TooBroadScope") - @Test - public void testValuePointerEnableEviction() { - mem = 90; - - final GridTuple evicted = new GridTuple<>(); - - evictLsnr = new GridOffHeapEvictListener() { - @Override public void onEvict(int part, int hash, byte[] k, byte[] v) { - String key = new String(k); - - evicted.set(key); - } - - @Override public boolean removeEvicted() { - return true; - } - }; - - map = newMap(); - - final String k1 = "k1"; - final String k2 = "k2"; - - map.put(1, k1.hashCode(), k1.getBytes(), bytes(20)); - - assertNull(evicted.get()); - - assertNotNull(map.valuePointer(1, k1.hashCode(), k1.getBytes())); - - assertNull(evicted.get()); - - map.put(1, k2.hashCode(), k2.getBytes(), bytes(20)); - - assertEquals(k2, evicted.get()); - - evicted.set(null); - - assertTrue(map.contains(1, k1.hashCode(), k1.getBytes())); - - map.enableEviction(1, k1.hashCode(), k1.getBytes()); - - assertNull(evicted.get()); - - map.put(1, k2.hashCode(), k2.getBytes(), bytes(20)); - - assertEquals(k1, evicted.get()); - - assertNull(map.valuePointer(1, k1.hashCode(), k1.getBytes())); - - evicted.set(null); - - assertNotNull(map.valuePointer(1, k2.hashCode(), k2.getBytes())); - - assertNull(evicted.get()); - - map.put(1, k1.hashCode(), k1.getBytes(), bytes(21)); - - assertEquals(k1, evicted.get()); - - evicted.set(null); - - map.put(1, k2.hashCode(), k2.getBytes(), bytes(21)); - } - - /** - * - */ - @Test - public void testValuePointerRemove() { - map = newMap(); - - final String k = "k1"; - - map.put(1, k.hashCode(), k.getBytes(), bytes(10)); - - assertNotNull(map.valuePointer(1, k.hashCode(), k.getBytes())); - - map.remove(1, k.hashCode(), k.getBytes()); - - assertNull(map.valuePointer(1, k.hashCode(), k.getBytes())); - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapPartitionedMapPerformanceAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapPartitionedMapPerformanceAbstractTest.java deleted file mode 100644 index 2f80209f98e61..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/GridOffHeapPartitionedMapPerformanceAbstractTest.java +++ /dev/null @@ -1,436 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap; - -import java.util.Map; -import java.util.Random; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import org.apache.ignite.cache.affinity.AffinityFunction; -import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; -import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.util.typedef.T3; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.junit.Test; - -/** - * Performance test for partitioned offheap hash map. - */ -@SuppressWarnings({"unchecked", "NonThreadSafeLazyInitialization"}) -public abstract class GridOffHeapPartitionedMapPerformanceAbstractTest extends GridCommonAbstractTest { - /** */ - protected static final int LOAD_CNT = 256; - - /** Sample keys. */ - private static T3 keys[]; - - /** Wrapped keys. */ - private static GridByteArrayWrapper[] wrappers; - - /** Unsafe map. */ - private GridOffHeapPartitionedMap map; - - /** */ - protected float load = 0.75f; - - /** */ - protected int concurrency = 16; - - /** */ - protected short lruStripes = 16; - - /** */ - protected long mem = 2L * 1024L * 1024L * 1024L; - - /** */ - protected long dur = 120 * 1000; - - /** - * - */ - protected GridOffHeapPartitionedMapPerformanceAbstractTest() { - super(false); - } - - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - map = newMap(); - - if (keys == null) { - keys = new T3[LOAD_CNT]; - wrappers = new GridByteArrayWrapper[LOAD_CNT]; - - AffinityFunction aff = new RendezvousAffinityFunction(); - - Random rnd = new Random(); - - for (int i = 0; i < LOAD_CNT; i++) { - byte[] key = new byte[rnd.nextInt(511) + 1]; - - rnd.nextBytes(key); - - GridByteArrayWrapper wrap = new GridByteArrayWrapper(key); - - keys[i] = new T3<>(aff.partition(wrap), wrap.hashCode(), key); - wrappers[i] = wrap; - } - } - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - if (map != null) - map.destruct(); - } - - /** - * @return New map. - */ - protected abstract GridOffHeapPartitionedMap newMap(); - - /** - * @throws Exception If failed. - */ - @Test - public void testPuts() throws Exception { - info("Warming up..."); - - checkPuts(1, 20000); - - info("Warm up finished."); - - checkPuts(Runtime.getRuntime().availableProcessors(), dur); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testPutsConcurrentMap() throws Exception { - info("Warming up..."); - - checkPutsConcurrentMap(1, 20000); - - info("Warm up finished."); - - checkPutsConcurrentMap(Runtime.getRuntime().availableProcessors(), dur); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testPutRemoves() throws Exception { - info("Warming up..."); - - checkPutRemoves(2, 20000); - - info("Warm up finished."); - - checkPutRemoves(Runtime.getRuntime().availableProcessors(), dur); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testPutRemovesConcurrentMap() throws Exception { - info("Warming up..."); - - checkPutRemovesConcurrentMap(2, 20000); - - info("Warm up finished."); - - checkPutRemovesConcurrentMap(Runtime.getRuntime().availableProcessors(), dur); - } - - /** - * @throws Exception If failed. - */ - private void checkPuts(int threadCnt, long duration) throws Exception { - final AtomicLong opCnt = new AtomicLong(); - final AtomicLong totalOpCnt = new AtomicLong(); - - final AtomicBoolean done = new AtomicBoolean(); - - long start = System.currentTimeMillis(); - - IgniteInternalFuture fut = multithreadedAsync(new Callable() { - @Override public Object call() throws Exception { - Random rnd = new Random(); - - byte[] val = new byte[1024]; - - long locTotalOpCnt = 0; - - while (!done.get()) { - for (int i = 0; i < 500; i++) { - T3 key = randomKey(rnd); - - map.put(key.get1(), key.get2(), key.get3(), val); - } - - locTotalOpCnt += 500; - opCnt.addAndGet(500); - } - - totalOpCnt.addAndGet(locTotalOpCnt); - - return null; - } - }, threadCnt); - - final int step = 2000; - - while (System.currentTimeMillis() - start < duration) { - U.sleep(step); - - long ops = opCnt.getAndSet(0); - - info("Putting " + (ops * 1000) / step + " ops/sec"); - } - - done.set(true); - - fut.get(); - - long end = System.currentTimeMillis(); - - info("Average put performance: " + (totalOpCnt.get() * 1000) / (end - start) + " ops/sec"); - } - - /** - * @throws Exception If failed. - */ - private void checkPutRemoves(int threadCnt, long duration) throws Exception { - final AtomicLong opCnt = new AtomicLong(); - final AtomicLong totalOpCnt = new AtomicLong(); - - final AtomicBoolean done = new AtomicBoolean(); - - long start = System.currentTimeMillis(); - - IgniteInternalFuture fut = multithreadedAsync(new Callable() { - @Override public Object call() throws Exception { - Random rnd = new Random(); - - byte[] val = new byte[1024]; - - long locTotalOpCnt = 0; - - while (!done.get()) { - for (int i = 0; i < 500; i++) { - T3 key = randomKey(rnd); - - int op = rnd.nextInt(2); - - switch (op) { - case 0: - map.put(key.get1(), key.get2(), key.get3(), val); - - break; - - case 1: - map.remove(key.get1(), key.get2(), key.get3()); - - break; - - default: - assert false; - } - } - - locTotalOpCnt += 500; - opCnt.addAndGet(500); - } - - totalOpCnt.addAndGet(locTotalOpCnt); - - return null; - } - }, threadCnt); - - final int step = 2000; - - while (System.currentTimeMillis() - start < duration) { - U.sleep(step); - - long ops = opCnt.getAndSet(0); - - info("Putting " + (ops * 1000) / step + " ops/sec"); - } - - done.set(true); - - fut.get(); - - long end = System.currentTimeMillis(); - - info("Average random operation performance: " + (totalOpCnt.get() * 1000) / (end - start) + " ops/sec"); - } - - /** - * @throws Exception If failed. - */ - private void checkPutsConcurrentMap(int threadCnt, long duration) throws Exception { - final Map map = new ConcurrentHashMap<>(); - - final AtomicLong opCnt = new AtomicLong(); - final AtomicLong totalOpCnt = new AtomicLong(); - - final AtomicBoolean done = new AtomicBoolean(); - - long start = System.currentTimeMillis(); - - IgniteInternalFuture fut = multithreadedAsync(new Callable() { - @Override public Object call() throws Exception { - Random rnd = new Random(); - - long locTotalOpCnt = 0; - - while (!done.get()) { - for (int i = 0; i < 500; i++) { - GridByteArrayWrapper key = randomKeyWrapper(rnd); - - map.put(key, new byte[1024]); - } - - locTotalOpCnt += 500; - opCnt.addAndGet(500); - } - - totalOpCnt.addAndGet(locTotalOpCnt); - - return null; - } - }, threadCnt); - - final int step = 2000; - - while (System.currentTimeMillis() - start < duration) { - U.sleep(step); - - long ops = opCnt.getAndSet(0); - - info("Putting " + (ops * 1000) / step + " ops/sec"); - } - - done.set(true); - - fut.get(); - - long end = System.currentTimeMillis(); - - info("Average put performance: " + (totalOpCnt.get() * 1000) / (end - start) + " ops/sec"); - } - - /** - * @throws Exception If failed. - */ - private void checkPutRemovesConcurrentMap(int threadCnt, long duration) throws Exception { - final Map map = new ConcurrentHashMap<>(); - - final AtomicLong opCnt = new AtomicLong(); - final AtomicLong totalOpCnt = new AtomicLong(); - - final AtomicBoolean done = new AtomicBoolean(); - - long start = System.currentTimeMillis(); - - IgniteInternalFuture fut = multithreadedAsync(new Callable() { - @Override public Object call() throws Exception { - Random rnd = new Random(); - - byte[] val = new byte[1024]; - - long locTotalOpCnt = 0; - - while (!done.get()) { - for (int i = 0; i < 500; i++) { - GridByteArrayWrapper key = randomKeyWrapper(rnd); - - int op = rnd.nextInt(2); - - switch (op) { - case 0: - map.put(key, val); - - break; - - case 1: - map.remove(key); - - break; - - default: - assert false; - } - } - - locTotalOpCnt += 500; - opCnt.addAndGet(500); - } - - totalOpCnt.addAndGet(locTotalOpCnt); - - return null; - } - }, threadCnt); - - final int step = 2000; - - while (System.currentTimeMillis() - start < duration) { - U.sleep(step); - - long ops = opCnt.getAndSet(0); - - info("Putting " + (ops * 1000) / step + " ops/sec"); - } - - done.set(true); - - fut.get(); - - long end = System.currentTimeMillis(); - - info("Average random operation performance: " + (totalOpCnt.get() * 1000) / (end - start) + " ops/sec"); - } - - /** - * Gets random key from pregenerated array. - * - * @param rnd Random to use. - * @return Tuple with key. - */ - private T3 randomKey(Random rnd) { - return keys[rnd.nextInt(keys.length)]; - } - - /** - * Gets random key from pregenerated array. - * - * @param rnd Random to use. - * @return Tuple with key. - */ - private GridByteArrayWrapper randomKeyWrapper(Random rnd) { - return wrappers[rnd.nextInt(keys.length)]; - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMapPerformanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMapPerformanceTest.java deleted file mode 100644 index f7a03452f2c85..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMapPerformanceTest.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap.unsafe; - -import org.apache.ignite.internal.util.offheap.GridOffHeapMap; -import org.apache.ignite.internal.util.offheap.GridOffHeapMapFactory; -import org.apache.ignite.internal.util.offheap.GridOffHeapMapPerformanceAbstractTest; - -/** - * Unsafe map test. - */ -public class GridUnsafeMapPerformanceTest extends GridOffHeapMapPerformanceAbstractTest { - /** {@inheritDoc} */ - @Override protected GridOffHeapMap newMap() { - return GridOffHeapMapFactory.unsafeMap(concurrency, load, initCap, mem, lruStripes, evictClo); - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMapSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMapSelfTest.java deleted file mode 100644 index ded6ee180f91f..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMapSelfTest.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap.unsafe; - -import org.apache.ignite.internal.util.offheap.GridOffHeapMap; -import org.apache.ignite.internal.util.offheap.GridOffHeapMapAbstractSelfTest; -import org.apache.ignite.internal.util.offheap.GridOffHeapMapFactory; - -/** - * Unsafe map test. - */ -public class GridUnsafeMapSelfTest extends GridOffHeapMapAbstractSelfTest { - /** {@inheritDoc} */ - @Override protected GridOffHeapMap newMap() { - return GridOffHeapMapFactory.unsafeMap(concurrency, load, initCap, mem, lruStripes, evictLsnr); - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafePartitionedMapPerformanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafePartitionedMapPerformanceTest.java deleted file mode 100644 index e3ad34608c479..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafePartitionedMapPerformanceTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap.unsafe; - -import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; -import org.apache.ignite.internal.util.offheap.GridOffHeapMapFactory; -import org.apache.ignite.internal.util.offheap.GridOffHeapPartitionedMap; -import org.apache.ignite.internal.util.offheap.GridOffHeapPartitionedMapPerformanceAbstractTest; - -/** - * Unsafe partitioned map performance test. - */ -public class GridUnsafePartitionedMapPerformanceTest extends GridOffHeapPartitionedMapPerformanceAbstractTest { - /** {@inheritDoc} */ - @Override protected GridOffHeapPartitionedMap newMap() { - return GridOffHeapMapFactory.unsafePartitionedMap(RendezvousAffinityFunction.DFLT_PARTITION_COUNT, - Runtime.getRuntime().availableProcessors(), load, LOAD_CNT, mem, lruStripes, null); - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafePartitionedMapSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafePartitionedMapSelfTest.java deleted file mode 100644 index e71c11a5594ac..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafePartitionedMapSelfTest.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap.unsafe; - -import org.apache.ignite.internal.util.offheap.GridOffHeapMapFactory; -import org.apache.ignite.internal.util.offheap.GridOffHeapPartitionedMap; -import org.apache.ignite.internal.util.offheap.GridOffHeapPartitionedMapAbstractSelfTest; - -/** - * Unsafe map test. - */ -public class GridUnsafePartitionedMapSelfTest extends GridOffHeapPartitionedMapAbstractSelfTest { - /** {@inheritDoc} */ - @Override protected GridOffHeapPartitionedMap newMap() { - return GridOffHeapMapFactory.unsafePartitionedMap(parts, concurrency, load, initCap, mem, lruStripes, evictLsnr); - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeStripedLruSefTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeStripedLruSefTest.java deleted file mode 100644 index 8a7caa8109812..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeStripedLruSefTest.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap.unsafe; - -import java.util.Collection; -import java.util.HashSet; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.junit.Test; - -/** - * Striped LRU test. - */ -@SuppressWarnings("FieldCanBeLocal") -public class GridUnsafeStripedLruSefTest extends GridCommonAbstractTest { - /** Number of stripes. */ - private short stripes = 1; - - /** Memory size. */ - private long memSize = 1024 * 1024; - - /** */ - private GridUnsafeMemory mem; - - /** */ - private GridUnsafeLru lru; - - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - stripes = 1; - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - if (lru != null) - lru.destruct(); - - lru = null; - mem = null; - } - - /** - * - */ - private void init() { - mem = new GridUnsafeMemory(memSize); - - lru = new GridUnsafeLru(stripes, mem); - } - - /** - * - */ - @Test - public void testOffer1() { - checkOffer(1000); - } - - /** - * - */ - @Test - public void testOffer2() { - stripes = 11; - - checkOffer(1000); - } - - /** - * @param cnt Count. - */ - private void checkOffer(int cnt) { - init(); - - for (int i = 0; i < cnt; i++) - lru.offer(0, i, i); - - assertEquals(cnt, lru.size()); - - info("Finished check offer for stripes count: " + stripes); - } - - /** - * - */ - @Test - public void testRemove1() { - checkRemove(1000); - } - - /** - * - */ - @Test - public void testRemove2() { - stripes = 35; - - checkRemove(1000); - } - - /** - * @param cnt Count. - */ - private void checkRemove(int cnt) { - init(); - - Collection set = new HashSet<>(cnt); - - for (int i = 0; i < cnt; i++) - assertTrue(set.add(lru.offer(0, i, i))); - - assertEquals(cnt, lru.size()); - - for (long addr : set) - lru.remove(addr); - - assertEquals(0, lru.size()); - } - - /** - * - */ - @Test - public void testPoll1() { - checkPoll(1000); - } - - /** - * - */ - @Test - public void testPoll2() { - stripes = 20; - - checkPoll(1000); - } - - /** - * @param cnt Count. - */ - private void checkPoll(int cnt) { - init(); - - int step = 10; - - assert cnt % step == 0; - - Collection set = new HashSet<>(step); - - for (int i = 0; i < cnt; i++) - lru.offer(0, i, i); - - assertEquals(cnt, lru.size()); - - for (int i = 0; i < cnt; i += step) { - for (int j = 0; j < step; j++) { - long qAddr = lru.prePoll(); - - assertTrue(qAddr != 0); - assertTrue(set.add(qAddr)); - } - - for (long qAddr : set) - lru.poll(qAddr); - - set.clear(); - } - - assertEquals(0, lru.size()); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testLruMultithreaded() throws Exception { - checkLruMultithreaded(1000000); - } - - /** - * @throws Exception If failed. - */ - private void checkLruMultithreaded(final int cnt) throws Exception { - init(); - - final AtomicInteger idGen = new AtomicInteger(); - - multithreaded(new Runnable() { - @Override public void run() { - int id = idGen.getAndIncrement(); - - int step = 10; - - assert cnt % step == 0; - - int start = id * cnt; - int end = start + cnt; - - Collection set = new HashSet<>(step); - - for (int i = start; i < end; i++) - lru.offer(0, i, i); - - for (int i = start; i < end; i += step) { - for (int j = 0; j < step; j++) { - long qAddr = lru.prePoll(); - - assertTrue(qAddr != 0); - assertTrue(set.add(qAddr)); - } - - for (long qAddr : set) - lru.poll(qAddr); - - set.clear(); - } - } - }, 10); - - assertEquals(0, lru.size()); - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/ssl/SSLContextWrapperTest.java b/modules/core/src/test/java/org/apache/ignite/ssl/SSLContextWrapperTest.java new file mode 100644 index 0000000000000..63bb1c5f2148d --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/ssl/SSLContextWrapperTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ssl; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLParameters; +import org.junit.Test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** Tests that a context wrapper preserves engine addresses and applies its SSL parameters. */ +public class SSLContextWrapperTest { + /** Both engine overloads apply the parameters, and the addressed overload retains the peer host and port. */ + @Test + public void testEnginePeerAddressAndParameters() throws Exception { + SSLContext delegate = SSLContext.getInstance("TLS"); + + delegate.init(null, null, null); + + String[] protocols = {"TLSv1.2"}; + String[] ciphers = {"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"}; + String host = "peer.example"; + int port = 10800; + SSLContext wrapper = new SSLContextWrapper(delegate, new SSLParameters(ciphers, protocols)); + SSLEngine addressed = wrapper.createSSLEngine(host, port); + + assertEquals(host, addressed.getPeerHost()); + assertEquals(port, addressed.getPeerPort()); + assertArrayEquals(protocols, addressed.getEnabledProtocols()); + assertArrayEquals(ciphers, addressed.getEnabledCipherSuites()); + + SSLEngine unaddressed = wrapper.createSSLEngine(); + + assertNull(unaddressed.getPeerHost()); + assertEquals(-1, unaddressed.getPeerPort()); + assertArrayEquals(protocols, unaddressed.getEnabledProtocols()); + assertArrayEquals(ciphers, unaddressed.getEnabledCipherSuites()); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java index 2ad409701d0f1..2eb1c465dfa64 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java @@ -668,18 +668,21 @@ protected void awaitPartitionMapExchange( else startTime = g0.context().discovery().gridStartTime(); - IgniteInternalFuture exchFut = - g0.context().cache().context().exchange().affinityReadyFuture(waitTopVer); + GridCachePartitionExchangeManager nodeExchMgr = g0.context().cache().context().exchange(); - if (exchFut != null && !exchFut.isDone()) { - try { - exchFut.get(timeout); - } - catch (IgniteCheckedException e) { - log.error("Failed to wait for exchange [topVer=" + waitTopVer + - ", node=" + g0.name() + ']', e); + try { + AffinityTopologyVersion completedVer = nodeExchMgr.affinityReadyFuture(waitTopVer).get(timeout); + + // The exchange future completes before its listener publishes the ready affinity version. + if (!waitForCondition(() -> nodeExchMgr.readyAffinityVersion().compareTo(completedVer) >= 0, timeout)) { + throw new IgniteException("Timeout waiting for ready affinity publication [node=" + g0.name() + + ", completedVer=" + completedVer + ", readyVer=" + nodeExchMgr.readyAffinityVersion() + ']'); } } + catch (IgniteCheckedException e) { + throw new IgniteException("Failed to wait for exchange [topVer=" + waitTopVer + + ", node=" + g0.name() + ']', e); + } for (IgniteCacheProxy c : g0.context().cache().jcaches()) { CacheConfiguration cfg = c.context().config(); @@ -2209,11 +2212,12 @@ protected void forceCheckpoint(Collection nodes, String reason) throws I * concurrently updated. * * @param ig Ignite instance. + * @param checkCrc Whether to check the CRC sums of partition pages. * @param caches Cache names (if null, all user caches will be verified). * @return Conflicts result. * @throws IgniteException If none caches or node found. */ - protected IdleVerifyResult idleVerify(Ignite ig, @Nullable String... caches) throws Exception { + protected IdleVerifyResult idleVerify(Ignite ig, boolean checkCrc, @Nullable String... caches) throws Exception { log.info("Starting idleVerify ..."); IgniteEx ig0 = (IgniteEx)ig; @@ -2236,6 +2240,7 @@ protected IdleVerifyResult idleVerify(Ignite ig, @Nullable String... caches) thr CacheIdleVerifyCommandArg taskArg = new CacheIdleVerifyCommandArg(); taskArg.caches(cacheNames.toArray(U.EMPTY_STRS)); + taskArg.checkCrc(checkCrc); return ((VisorTaskResult)ig.compute().execute( IdleVerifyTask.class.getName(), @@ -2243,6 +2248,11 @@ protected IdleVerifyResult idleVerify(Ignite ig, @Nullable String... caches) thr )).result(); } + /** @see #idleVerify(Ignite, boolean, String...) */ + protected IdleVerifyResult idleVerify(Ignite ig, @Nullable String... caches) throws Exception { + return idleVerify(ig, false, caches); + } + /** * Checks if all txs and mvcc futures are finished. */ diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite19.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite19.java index 6d41d046f4311..32d82bf131e83 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite19.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite19.java @@ -34,6 +34,7 @@ import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest; import org.apache.ignite.internal.processors.cache.distributed.rebalancing.MultiDcRebalancingTest; +import org.apache.ignite.internal.processors.cache.distributed.rebalancing.MultiDcSplitBrainRebalanceTest; import org.apache.ignite.internal.processors.cache.persistence.CleanupRestoredCachesSlowTest; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.DynamicSuite; @@ -72,6 +73,7 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, GridCacheRebalancingCancelTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, MultiDcRebalancingTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, MultiDcSplitBrainRebalanceTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheStoreTxPutAllMultiNodeTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CleanupRestoredCachesSlowTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, ClientCreateCacheGroupOnJoinNodeMapsTest.class, ignoredTests); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java index 3b93a139c372c..2e2c6234f799a 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java @@ -23,6 +23,7 @@ import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithExpiryPolicy; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsNodeJoinWithCachesStopping; +import org.apache.ignite.internal.processors.cache.persistence.NodeOrCacheStoppingDuringIdleVerifyTest; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.DynamicSuite; import org.junit.runner.RunWith; @@ -47,6 +48,7 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, IgnitePdsContinuousRestartTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgnitePdsContinuousRestartTestWithExpiryPolicy.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgnitePdsNodeJoinWithCachesStopping.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, NodeOrCacheStoppingDuringIdleVerifyTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePerformanceTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePerformanceTestSuite.java index 8b44a525a0cb1..3f1db6aa80ecd 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePerformanceTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePerformanceTestSuite.java @@ -24,8 +24,6 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedAffinityExcludeNeighborsPerformanceTest; import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicyPerformanceTest; import org.apache.ignite.internal.processors.datastreamer.IgniteDataStreamerPerformanceTest; -import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMapPerformanceTest; -import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafePartitionedMapPerformanceTest; import org.apache.ignite.lang.GridBasicPerformanceTest; import org.apache.ignite.lang.GridFuncPerformanceTest; import org.apache.ignite.lang.GridFutureListenPerformanceTest; @@ -87,8 +85,6 @@ GridSingleSplitsRedeployLoadTest.class, GridStealingLoadTest.class, GridTcpCommunicationSpiLanLoadTest.class, - GridUnsafeMapPerformanceTest.class, - GridUnsafePartitionedMapPerformanceTest.class, IgniteDataStreamerPerformanceTest.class, SortedEvictionPolicyPerformanceTest.class, IgniteCacheStartStopLoadTest.class, diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java index 86802121999c7..6b2ccfa4ed443 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite9.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotCheckStatusRollingUpgradeTest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteRollingUpgradeTest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteTest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotListRollingUpgradeTest; @@ -47,6 +48,7 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotDeleteTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotListTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotListRollingUpgradeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotCheckStatusRollingUpgradeTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java index a630abf950873..232193aa00bed 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java @@ -45,10 +45,7 @@ import org.apache.ignite.internal.util.nio.GridNioSessionMetaKeySelfTest; import org.apache.ignite.internal.util.nio.GridNioSslSelfTest; import org.apache.ignite.internal.util.nio.impl.GridNioFilterChainSelfTest; -import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMapSelfTest; import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemorySelfTest; -import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafePartitionedMapSelfTest; -import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeStripedLruSefTest; import org.apache.ignite.internal.util.tostring.CircularStringBuilderSelfTest; import org.apache.ignite.internal.util.tostring.GridToStringBuilderSelfTest; import org.apache.ignite.internal.util.tostring.IncludeSensitiveAtomicTest; @@ -129,9 +126,6 @@ // Unsafe. GridUnsafeMemorySelfTest.class, - GridUnsafeStripedLruSefTest.class, - GridUnsafeMapSelfTest.class, - GridUnsafePartitionedMapSelfTest.class, // NIO. GridNioSessionMetaKeySelfTest.class, diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/SecurityTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/SecurityTestSuite.java index da768a22290fb..37fa57703b572 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/SecurityTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/SecurityTestSuite.java @@ -79,6 +79,7 @@ import org.apache.ignite.internal.thread.context.OperationContextAttributesTest; import org.apache.ignite.plugin.security.SecurityBasicPermissionSetSerializationTest; import org.apache.ignite.ssl.MultipleSSLContextsTest; +import org.apache.ignite.ssl.SSLContextWrapperTest; import org.apache.ignite.tools.junit.JUnitTeamcityReporter; import org.junit.BeforeClass; import org.junit.runner.RunWith; @@ -141,6 +142,7 @@ IgniteSecurityProcessorTest.class, MultipleSSLContextsTest.class, + SSLContextWrapperTest.class, MaintenanceModeNodeSecurityTest.class, ServiceAuthorizationTest.class, ServiceStaticConfigTest.class, diff --git a/modules/ducktests/README.md b/modules/ducktests/README.md index c789bc25d98b4..66dc5ccb06315 100644 --- a/modules/ducktests/README.md +++ b/modules/ducktests/README.md @@ -173,6 +173,9 @@ You can modify test environments at execution time using global flags injected t | **cluster_size** | Controls the cluster size for tests. Overrides the default num_nodes value passed in the @cluster annotation. Default value is determined by the test framework. | ```{"cluster_size": 13}``` | | **ignite_versions** | List of Ignite versions for testing. Tests run for each version in the list. Values are folder names in /opt/ (e.g., "ignite-dev" for master branch, "ignite-2.18.0" for release). Default is determined by test annotations. | ```{"ignite_versions": ["ignite-dev", "ignite-2.18.0"]}``` | | **failure_detection_timeout** | Timeout in milliseconds for failure detection in discovery. Used to detect node failures in the cluster. Default value is 10000 ms (10 seconds). | ```{"failure_detection_timeout": 20000}``` | +| **disk_full_tests** | Boolean flag to run the reproducers in `disk_full_test.py`, which fill a disk volume of a server node and check what the node does. Default is False: the tests are ignored. | ```{"disk_full_tests": true}``` | + +*Note on reproducers:* A test marked with `@reproducer("")` runs only when that global is set to true; otherwise it is ignored. The mark is for tests that reproduce a behavior on demand and guard nothing on a regular run. #### Security & Authentication diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java new file mode 100644 index 0000000000000..875b17ca12231 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.disk_full; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Puts values under growing keys, with a new binary type every 1000 puts, so the partitions, the WAL and the binary + * metadata keep growing. Retries failed puts and records {@code progress-> <-} once a second. + */ +public class DiskFullLoadApplication extends IgniteAwareApplication { + /** Value size, in bytes. */ + private static final int VALUE_SIZE = 1024; + + /** Puts per new binary type. */ + private static final int BINARY_TYPE_EVERY = 1000; + + /** Successful puts after which the application reports it is initialized. */ + private static final int WARM_UP_PUTS = 1000; + + /** Pause after a failed put, in milliseconds. */ + private static final long ERROR_PAUSE = 200; + + /** {@inheritDoc} */ + @Override protected void run(JsonNode jsonNode) throws Exception { + ClientCache cache = client.getOrCreateCache(jsonNode.get("cacheName").asText()); + + byte[] val = new byte[VALUE_SIZE]; + + long key = 0; + long puts = 0; + long errs = 0; + long nextReport = System.currentTimeMillis() + 1000; + + while (active()) { + try { + if (key % BINARY_TYPE_EVERY == 0) + cache.put(key, client.binary().builder("DiskFullType" + key).setField("val", val).build()); + else + cache.put(key, val); + + key++; + puts++; + + // The first record comes before the marker, so the test can read the counters once start() returns. + if (puts == WARM_UP_PUTS) { + recordResult("progress", puts + " " + errs); + + markInitialized(); + } + } + catch (Exception e) { + errs++; + + log.warn("Put failed [key=" + key + ", err=" + e + ']'); + + Thread.sleep(ERROR_PAUSE); + } + + long now = System.currentTimeMillis(); + + if (now >= nextReport) { + recordResult("progress", puts + " " + errs); + + nextReport = now + 1000; + } + } + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java new file mode 100644 index 0000000000000..1272054346e39 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.disk_full; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Deploys {@link StdoutFloodServiceImpl} on every server node, records {@code lines-><-}, the lines the service + * wrote, once a second, and cancels the service on stop. + */ +public class StdoutFloodApplication extends IgniteAwareApplication { + /** Service name. */ + private static final String SVC_NAME = "stdout-flood"; + + /** {@inheritDoc} */ + @Override protected void run(JsonNode jsonNode) throws Exception { + ignite.services(ignite.cluster().forServers()).deployNodeSingleton(SVC_NAME, new StdoutFloodServiceImpl()); + + StdoutFloodService svc = ignite.services().serviceProxy(SVC_NAME, StdoutFloodService.class, false); + + recordResult("lines", svc.lines()); + + markInitialized(); + + while (active()) { + Thread.sleep(1000); + + recordResult("lines", svc.lines()); + } + + ignite.services().cancel(SVC_NAME); + + markFinished(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLruPoller.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodService.java similarity index 75% rename from modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLruPoller.java rename to modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodService.java index a9d35e77c18fb..e3b277cbd9a63 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeLruPoller.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodService.java @@ -15,16 +15,12 @@ * limitations under the License. */ -package org.apache.ignite.internal.util.offheap.unsafe; +package org.apache.ignite.internal.ducktest.tests.disk_full; /** - * Polls LRU. + * Writes lines to {@code System.out} without end, as user code left with debug output does. */ -interface GridUnsafeLruPoller { - /** - * Frees space from LRU queue. - * - * @param size Size of the space to free. - */ - public void lruPoll(int size); +public interface StdoutFloodService { + /** @return Lines written so far. */ + public long lines(); } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java new file mode 100644 index 0000000000000..3904e8a840984 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.disk_full; + +import java.util.concurrent.atomic.AtomicLong; +import org.apache.ignite.resources.ServiceContextResource; +import org.apache.ignite.services.Service; +import org.apache.ignite.services.ServiceContext; + +/** + * Writes 1 KB lines to {@code System.out} at about 10 MB/s until cancelled. + */ +public class StdoutFloodServiceImpl implements Service, StdoutFloodService { + /** Serial version UID. */ + private static final long serialVersionUID = 0L; + + /** Line, 1 KB with its line separator. */ + private static final String LINE = "x".repeat(1023); + + /** Lines written per millisecond. */ + private static final int LINES_PER_MS = 10; + + /** Service context. */ + @ServiceContextResource + private ServiceContext ctx; + + /** Lines written. */ + private final AtomicLong lines = new AtomicLong(); + + /** {@inheritDoc} */ + @Override public void execute() throws Exception { + while (!ctx.isCancelled()) { + for (int i = 0; i < LINES_PER_MS; i++) { + System.out.println(LINE); + + lines.incrementAndGet(); + } + + Thread.sleep(1); + } + } + + /** {@inheritDoc} */ + @Override public long lines() { + return lines.get(); + } +} diff --git a/modules/ducktests/tests/checks/utils/check_ignite_spec.py b/modules/ducktests/tests/checks/utils/check_ignite_spec.py index 1e2cc4d3920a1..41ff5909202a7 100644 --- a/modules/ducktests/tests/checks/utils/check_ignite_spec.py +++ b/modules/ducktests/tests/checks/utils/check_ignite_spec.py @@ -17,13 +17,14 @@ Checks Spec class that describes config and command line to start Ignite-aware service. """ import os -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest from ignitetest.services.utils import IgniteServiceType from ignitetest.services.utils.gc_params import CLIENT_ROLE, SERVER_ROLE -from ignitetest.services.utils.ignite_spec import IgniteApplicationSpec, IgniteNodeSpec, service_role +from ignitetest.services.utils.ignite_spec import IgniteApplicationSpec, IgniteNodeSpec, IgniteNodeSpecFileOutput, \ + service_role from ignitetest.services.utils.jvm_utils import GC_PROFILES, GC_G1, MultipleGcSelectedError from ignitetest.utils.ignite_test import JFR_ENABLED @@ -287,3 +288,18 @@ def check_heap_override__from_jvm_opts__still_wins(service): assert "-Xms4G" in spec.jvm_opts assert "-Xmx1G" not in spec.jvm_opts assert "-Xms1G" not in spec.jvm_opts + + +""" +Checks the shell redirection of a node's output. +""" + + +def check_node_spec_file_output__appends_stdout_and_stderr_to_console_log(service): + service.log_dir = "/logs" + + with patch.object(IgniteNodeSpecFileOutput, "envs", return_value={}): + cmd = IgniteNodeSpecFileOutput(service).command(None) + + assert cmd.endswith(f" >> {os.path.join(service.log_dir, 'console.log')} 2>&1 &") + assert "tee" not in cmd diff --git a/modules/ducktests/tests/checks/utils/check_parametrized.py b/modules/ducktests/tests/checks/utils/check_parametrized.py index 7c814fec45780..3bc84d161fd53 100644 --- a/modules/ducktests/tests/checks/utils/check_parametrized.py +++ b/modules/ducktests/tests/checks/utils/check_parametrized.py @@ -24,7 +24,7 @@ from ducktape.mark import parametrized, parametrize, matrix, ignore from ducktape.mark.mark_expander import MarkedFunctionExpander -from ignitetest.utils import ignite_versions, ignore_if +from ignitetest.utils import ignite_versions, ignore_if, reproducer from ignitetest.utils._mark import IgniteVersionParametrize from ignitetest.utils.version import IgniteVersion, V_2_8_0, V_2_8_1, V_2_7_6, DEV_BRANCH @@ -267,3 +267,31 @@ def function_3(ignite_version): assert next(filter(lambda x: x.injected_args['ver'] == str(V_2_8_0), context_list)).ignore assert not next(filter(lambda x: x.injected_args['ver'] == str(DEV_BRANCH), context_list)).ignore + + +class CheckReproducer: + """ + Checks @reproducer. + """ + @pytest.mark.parametrize(['global_args', 'ignored'], [({}, True), ({'my_tests': False}, True), + ({'my_tests': True}, False)]) + def check_ignored_unless_global_is_set(self, global_args, ignored): + """ + Check that a test is ignored unless its global is set, with and without @ignite_versions. + """ + @reproducer('my_tests') + @ignite_versions(str(DEV_BRANCH), str(V_2_8_0)) + def versioned(ignite_version): + return IgniteVersion(ignite_version) + + @reproducer('my_tests') + def plain(): + pass + + sess_ctx = mock_session_ctx(global_args=global_args) + + context_list = expand_function(func=versioned, sess_ctx=sess_ctx) + context_list += MarkedFunctionExpander(session_context=sess_ctx, function=plain).expand() + + assert len(context_list) == 3 + assert all(ctx.ignore == ignored for ctx in context_list) diff --git a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py index a9ccbe4496d3c..b5b57e780a426 100644 --- a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py +++ b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py @@ -34,6 +34,7 @@ from ignitetest.services.network_group.manager import NetworkGroupManager from ignitetest.services.utils.control_utility import ControlUtility from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, TcpCommunicationSpi +from ignitetest.services.utils.ignite_configuration.data_storage import DataStorageConfiguration from ignitetest.services.utils.ignite_configuration.discovery import TcpDiscoverySpi, from_ignite_cluster, \ from_ignite_services from ignitetest.services.utils.ssl.client_connector_configuration import ClientConnectorConfiguration @@ -101,9 +102,12 @@ def __init__(self, test, ignite_version: str, srv_per_dc: Union[int, Dict[str, i loaders_per_dc: Union[int, Dict[str, int]] = 0, client_connector: bool = False, network_timeout: int = 5_000, - tcp_connect_timeout: int = 5_000): + tcp_connect_timeout: int = 5_000, + data_storage: DataStorageConfiguration = None, + jvm_opts: List[str] = None): self.test_context = test.test_context self.logger = test.logger + self._jvm_opts = jvm_opts or [] # A single discovery SPI (hence a single ip finder) shared by both DCs' server # services is what makes the two DCs form ONE cluster: prepare_on_start() @@ -115,7 +119,8 @@ def __init__(self, test, ignite_version: str, srv_per_dc: Union[int, Dict[str, i "version": IgniteVersion(ignite_version), "discovery_spi": TcpDiscoverySpi(), "network_timeout": network_timeout, - "communication_spi": TcpCommunicationSpi(connect_timeout=tcp_connect_timeout) + "communication_spi": TcpCommunicationSpi(connect_timeout=tcp_connect_timeout), + "data_storage": data_storage } if client_connector: @@ -126,7 +131,8 @@ def __init__(self, test, ignite_version: str, srv_per_dc: Union[int, Dict[str, i self.srv_per_dc = _per_dc(srv_per_dc) self.servers: Dict[str, IgniteService] = { - dc: IgniteService(self.test_context, self.ignite_config, num_nodes=num, jvm_opts=dc_jvm_opts(dc), + dc: IgniteService(self.test_context, self.ignite_config, num_nodes=num, + jvm_opts=dc_jvm_opts(dc) + self._jvm_opts, startup_timeout_sec=IGNITE_STARTUP_TIMEOUT_SEC) for dc, num in self.srv_per_dc.items() if num > 0} @@ -162,9 +168,11 @@ def sync_service_discovery(self): service.config = service.config._replace(discovery_spi=discovery_spi) def _app_service(self, dc: str) -> IgniteApplicationService: - client_cfg = self.ignite_config._replace(client_mode=True, discovery_spi=from_ignite_cluster(self.servers[dc])) + client_cfg = self.ignite_config._replace( + client_mode=True, discovery_spi=from_ignite_cluster(self.servers[dc]), + data_storage=None) - return IgniteApplicationService(self.test_context, client_cfg, jvm_opts=dc_jvm_opts(dc)) + return IgniteApplicationService(self.test_context, client_cfg, jvm_opts=dc_jvm_opts(dc) + self._jvm_opts) def register(self, dc: str, service): """ diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index e0f81ab35b785..822d422fad9a5 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -48,6 +48,9 @@ class IgniteAwareService(BackgroundThreadService, IgnitePathAware, JvmProcessMix """ The base class to build services aware of Ignite. """ + # Mount source of the volumes mount_tmpfs makes: clean_node finds them by it. + TMPFS_SOURCE = "ignitetest-tmpfs" + @constructible class NetPart(IntEnum): """ @@ -79,6 +82,8 @@ def __init__(self, context, config, num_nodes, startup_timeout_sec, shutdown_tim self.disconnected_nodes = [] + self.tmpfs_volumes = {} + @property def product(self): return str(self.config.version) @@ -109,7 +114,15 @@ def await_started(self, nodes=None): self.await_event("Topology snapshot", self.startup_timeout_sec, nodes=nodes, from_the_beginning=True) + def mount_tmpfs(self, path, size_mb): + """ + Mounts a tmpfs of size_mb over the path on every node when it starts, so writes there fail once it is full. + """ + self.tmpfs_volumes[path] = size_mb + def start_node(self, node, **kwargs): + self.__mount_tmpfs_volumes(node) + self.init_shared(node) self.init_persistent(node) @@ -186,8 +199,26 @@ def clean(self, **kwargs): def clean_node(self, node, **kwargs): super().clean_node(node, **kwargs) + self.__unmount_tmpfs_volumes(node) + node.account.ssh("rm -rf -- %s" % self.persistent_root, allow_fail=False) + def __mount_tmpfs_volumes(self, node): + """ + Mounts the volumes registered by mount_tmpfs, unless they are still mounted from the node's previous start. + """ + for path, size_mb in self.tmpfs_volumes.items(): + node.account.ssh(f"findmnt -rn -S {self.TMPFS_SOURCE} -M {path} > /dev/null || (mkdir -p {path} && " + f"sudo mount -t tmpfs -o size={size_mb}m,uid=$(id -u),gid=$(id -g) {self.TMPFS_SOURCE} " + f"{path})", allow_fail=False) + + def __unmount_tmpfs_volumes(self, node): + """ + Unmounts every volume mount_tmpfs made on the node, also one left by an aborted run. + """ + node.account.ssh(f"findmnt -rn -S {self.TMPFS_SOURCE} -o TARGET | xargs -r -n1 sudo umount -l", + allow_fail=False) + def init_persistent(self, node): """ Init persistent directory. diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py index 54e5be421668e..0484c95aafdaa 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py @@ -50,6 +50,7 @@ class DataStorageConfiguration(NamedTuple): metrics_rate_time_interval: int = None page_size: int = None regions: list = [] + storage_path: str = None wal_archive_path: str = None wal_buffer_size: int = None wal_compaction_enabled: bool = None diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py index ae7036df48c2e..891c56619fce1 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -373,6 +373,12 @@ def _jvm_opts(self): opts = ["-J%s" % o for o in self.jvm_opts] return " ".join(opts) + def _output_redirect(self): + """ + :return: shell redirection of the service's stdout and stderr: through tee, appended to console.log. + """ + return "2>&1 | tee -a %s" % os.path.join(self.service.log_dir, "console.log") + def _runcmd(self, cmd): self.service.logger.debug(cmd) proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -388,16 +394,26 @@ class IgniteNodeSpec(IgniteSpec): """ def command(self, node): - cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \ + cmd = "%s %s %s %s %s &" % \ (envs_to_exports(self.envs()), self.service.script("ignite.sh"), self._jvm_opts(), self.config_file_path(), - os.path.join(self.service.log_dir, "console.log")) + self._output_redirect()) return cmd +class IgniteNodeSpecFileOutput(IgniteNodeSpec): + """ + Spec to run ignite node with stdout and stderr appended to console.log by the shell, as nohup does, so the JVM + writes them itself instead of a pipe to tee. + """ + + def _output_redirect(self): + return ">> %s 2>&1" % os.path.join(self.service.log_dir, "console.log") + + class IgniteApplicationSpec(IgniteSpec): """ Spec to run ignite application @@ -422,12 +438,12 @@ def command(self, node): str(base64.b64encode(json.dumps(self.service.params).encode('utf-8')), 'utf-8') ] - cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \ + cmd = "%s %s %s %s %s &" % \ (envs_to_exports(self.envs()), self.service.script("ignite.sh"), self._jvm_opts(), ",".join(args), - os.path.join(self.service.log_dir, "console.log")) + self._output_redirect()) return cmd diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py new file mode 100644 index 0000000000000..adedac946a8a1 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py @@ -0,0 +1,458 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Fills one volume of a persistent server node under load and checks whether the node survives. + +The tests reproduce what Ignite does today and guard no behavior, so they run only with +--globals '{"disk_full_tests": true}'. +""" +import os +import time + +from ducktape.mark import matrix +from ducktape.utils.util import wait_until + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.control_utility import ControlUtility +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, IgniteThinClientConfiguration, \ + DataStorageConfiguration +from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster +from ignitetest.services.utils.ignite_spec import IgniteNodeSpecFileOutput +from ignitetest.services.utils.jvm_utils import merge_jvm_settings +from ignitetest.services.utils.ssl.client_connector_configuration import ClientConnectorConfiguration +from ignitetest.utils import cluster, ignite_versions, reproducer +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion + +MB = 1024 * 1024 + +# A full tmpfs still takes writes into the rest of a file's last page. +PAGE = 4096 + +FILLER = "filler" + +# Failure type and failed writer the node logs when the volume fills. +FAILURES = { + "wal_archive": ("SYSTEM_WORKER_TERMINATION", "Failed to archive WAL segment"), + "partitions": ("CRITICAL_ERROR", "StorageException"), + "work": ("CRITICAL_ERROR", "BinaryMetadataFileStore") +} + + +class DiskFullTest(IgniteTest): + """ + Each test puts a size-limited tmpfs over a directory of the node, fills it and records what the node does today. + """ + LOAD_APP = "org.apache.ignite.internal.ducktest.tests.disk_full.DiskFullLoadApplication" + + FLOOD_APP = "org.apache.ignite.internal.ducktest.tests.disk_full.StdoutFloodApplication" + + @cluster(num_nodes=2) + @reproducer("disk_full_tests") + @ignite_versions(str(DEV_BRANCH)) + def test_node_survives_and_restarts_on_full_log_volume(self, ignite_version): + """ + The node serves load and restarts while the volume of its log4j2 log, console.log and gc.log is full. + """ + ignite = self.__ignite(ignite_version) + ignite.mount_tmpfs(ignite.log_dir, 32) + + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) + + fill_volume(ignite, ignite.log_dir) + + check_serves_load_for(ignite, load, seconds=15) + assert ControlUtility(ignite).cluster_state().state == "ACTIVE" + + for log in log_files(ignite): + check_write_failed(ignite, log) + + ignite.stop() + + # Waits for a failed put, so the count of puts read next is final until the node is back. + _, errors = progress(load) + wait_until(lambda: progress(load)[1] > errors, timeout_sec=30, err_msg="The load still reaches the node") + + puts, _ = progress(load) + + # start() awaits a line in the node's new log, which the full volume refuses. + ignite.start_async(clean=False) + + wait_until(lambda: progress(load)[0] > puts, timeout_sec=60, err_msg="No puts after the restart") + + check_serves_load_for(ignite, load, seconds=15) + assert ControlUtility(ignite).cluster_state().state == "ACTIVE" + assert file_size(ignite, ignite.nodes[0].log_file) == 0, "The restarted node wrote its log" + + free_volume(ignite, ignite.log_dir) + + for log in log_files(ignite): + check_grows(ignite, log) + + load.stop() + + @cluster(num_nodes=3) + @reproducer("disk_full_tests") + @ignite_versions(str(DEV_BRANCH)) + def test_node_survives_log_volume_filled_by_stdout(self, ignite_version): + """ + A service on the node writes to System.out until its output fills the log volume. + """ + ignite = self.__ignite(ignite_version) + ignite.mount_tmpfs(ignite.log_dir, 32) + + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) + + flood = self.__start_flood_service(ignite) + + wait_until(lambda: free_space(ignite, ignite.log_dir) == 0, timeout_sec=60, + err_msg="System.out did not fill the log volume") + + lines = printed_lines(flood) + + check_serves_load_for(ignite, load, seconds=15) + assert ControlUtility(ignite).cluster_state().state == "ACTIVE" + assert printed_lines(flood) > lines, "The service stopped writing to System.out" + + for log in log_files(ignite): + check_write_failed(ignite, log) + + flood.stop() + + enlarge_volume(ignite, ignite.log_dir, 128) + + for log in log_files(ignite): + check_grows(ignite, log) + + load.stop() + + @cluster(num_nodes=2) + @reproducer("disk_full_tests") + @ignite_versions(str(DEV_BRANCH)) + def test_jvm_exits_on_full_jfr_volume(self, ignite_version): + """ + JFR ends the JVM once it can't write its recording. + """ + ignite = self.__ignite(ignite_version) + repository = os.path.join(ignite.persistent_root, "jfr_repository") + jfr_opts = ["-XX:StartFlightRecording", f"-XX:FlightRecorderOptions=repository={repository}"] + ignite.spec.jvm_opts = merge_jvm_settings(ignite.spec.jvm_opts, jfr_opts) + ignite.mount_tmpfs(repository, 32) + + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) + + fill_volume(ignite, repository) + + ignite.await_event("An irrecoverable error in Jfr", 60, from_the_beginning=True, log_file="console.log") + check_stops(ignite) + + load.stop() + + @cluster(num_nodes=2) + @reproducer("disk_full_tests") + @ignite_versions(str(DEV_BRANCH)) + def test_node_stops_on_full_volume_shared_by_logs_and_data(self, ignite_version): + """ + One volume holds the logs and all the data, as in Ignite's default layout. + """ + ignite = self.__ignite(ignite_version) + ignite.mount_tmpfs(ignite.persistent_root, 256) + + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) + + fill_volume(ignite, ignite.persistent_root) + + check_stops(ignite) + + load.stop() + + @cluster(num_nodes=2) + @reproducer("disk_full_tests") + @ignite_versions(str(DEV_BRANCH)) + def test_node_survives_full_wal_volume(self, ignite_version): + """ + WAL segments are preallocated, so puts go on into a full WAL volume. + """ + ignite = self.__ignite(ignite_version) + wal = os.path.join(ignite.persistent_root, "wal") + ignite.mount_tmpfs(wal, 64) + + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) + + segments = ignite.exec_command(ignite.nodes[0], f"ls {wal}/*/*.wal 2>/dev/null | wc -l") + assert int(segments) > 0, "No WAL segments on the volume" + + fill_volume(ignite, wal) + + check_serves_load_for(ignite, load, seconds=15) + + load.stop() + + @cluster(num_nodes=1) + @reproducer("disk_full_tests") + @ignite_versions(str(DEV_BRANCH)) + def test_node_stops_on_wal_volume_smaller_than_its_segments(self, ignite_version): + """ + The node formats its WAL segments at start, and they don't fit the volume. + """ + ignite = self.__ignite(ignite_version) + + # Ten 4 MB segments need 40 MB. + ignite.mount_tmpfs(os.path.join(ignite.persistent_root, "wal"), 16) + + ignite.start_async() + + check_stops(ignite) + check_failure(ignite, "CRITICAL_ERROR", "Failed to format WAL segment file") + + @cluster(num_nodes=2) + @reproducer("disk_full_tests") + @ignite_versions(str(DEV_BRANCH)) + @matrix(volume=["wal_archive", "partitions", "work"]) + def test_node_stops_on_full_volume(self, ignite_version, volume): + """ + The node stops once it can't write the WAL archive, the partitions or the binary metadata (in work). + """ + ignite = self.__ignite(ignite_version) + path = os.path.join(ignite.persistent_root, volume) + ignite.mount_tmpfs(path, 64) + + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) + + fill_volume(ignite, path) + + check_stops(ignite) + check_failure(ignite, *FAILURES[volume]) + + load.stop() + + @cluster(num_nodes=2) + @reproducer("disk_full_tests") + @ignite_versions(str(DEV_BRANCH)) + def test_node_stops_when_cdc_fills_wal_archive_volume(self, ignite_version): + """ + CDC with no consumer and no cdcWalDirectoryMaxSize keeps every archived segment. + """ + ignite = self.__ignite(ignite_version) + archive = os.path.join(ignite.persistent_root, "wal_archive") + data_storage = ignite.config.data_storage + + # CDC hard-links archived segments, so its directory must be on the archive's volume. + ignite.config = ignite.config._replace(data_storage=data_storage._replace( + cdc_wal_path=os.path.join(archive, "cdc"), default=data_storage.default._replace(cdc_enabled=True))) + ignite.mount_tmpfs(archive, 256) + + ignite.start() + ControlUtility(ignite).activate() + + load = self.__start_load_client(ignite) + + check_stops(ignite, timeout_sec=120) + check_failure(ignite, *FAILURES["wal_archive"]) + + load.stop() + + def __ignite(self, ignite_version): + """ + :return: Persistent server node that writes its stdout and stderr into console.log itself, with the + partitions, the WAL and its archive in directories of their own. + """ + ignite = IgniteService(self.test_context, IgniteConfiguration( + version=IgniteVersion(ignite_version), + client_connector_configuration=ClientConnectorConfiguration(), + metrics_log_frequency=1000, + data_storage=DataStorageConfiguration( + checkpoint_frequency=1000, + wal_segment_size=4 * MB, + max_wal_archive_size=32 * MB, + default=DataRegionConfiguration(persistence_enabled=True, max_size=256 * MB))), num_nodes=1) + + ignite.spec = ignite.spec.rebuild_as(IgniteNodeSpecFileOutput) + + root = ignite.persistent_root + + ignite.config = ignite.config._replace(data_storage=ignite.config.data_storage._replace( + storage_path=os.path.join(root, "partitions"), + wal_path=os.path.join(root, "wal"), + wal_archive_path=os.path.join(root, "wal_archive"))) + + return ignite + + def __start_load_client(self, ignite): + """ + Starts a load that records its puts and errors once a second. + :return: Load application service. + """ + address = f"{ignite.nodes[0].account.hostname}:{ignite.config.client_connector_configuration.port}" + + load = IgniteApplicationService( + self.test_context, + IgniteThinClientConfiguration(addresses=[address], version=ignite.config.version), + java_class_name=self.LOAD_APP, + params={"cacheName": "disk-full"}) + + load.start() + + return load + + def __start_flood_service(self, ignite): + """ + Starts the application that deploys the System.out flood service on the server. + :return: Flood application service. + """ + flood = IgniteApplicationService( + self.test_context, + IgniteConfiguration(version=ignite.config.version, client_mode=True, + discovery_spi=from_ignite_cluster(ignite)), + java_class_name=self.FLOOD_APP) + + flood.start() + + return flood + + +def log_files(ignite): + """ + :return: The node's log4j2 log, console.log and gc.log, all on the log volume. + """ + return [ignite.nodes[0].log_file] + [os.path.join(ignite.log_dir, log) for log in ("console.log", "gc.log")] + + +def fill_volume(ignite, path): + """ + Writes a file into the path until its volume refuses the write. + """ + out = ignite.exec_command(ignite.nodes[0], f"dd if=/dev/zero of={os.path.join(path, FILLER)} bs=1M 2>&1") + + assert "No space left on device" in out, f"{path} did not fill up: {out}" + + +def free_volume(ignite, path): + """ + Removes the file fill_volume wrote. + """ + ignite.nodes[0].account.ssh(f"rm -f {os.path.join(path, FILLER)}", allow_fail=False) + + +def enlarge_volume(ignite, path, size_mb): + """ + Grows the tmpfs mounted over the path, keeping its files and what they hold, so writes there go on. + """ + ignite.nodes[0].account.ssh(f"sudo mount -o remount,size={size_mb}m {path}", allow_fail=False) + + +def free_space(ignite, path): + """ + :return: Bytes free on the volume of the path. + """ + return int(ignite.exec_command(ignite.nodes[0], f"df --output=avail -B1 {path} | tail -1")) + + +def printed_lines(flood): + """ + :return: Lines the service wrote to System.out, as the flood application recorded last. + """ + return int(flood.extract_results("lines")[-1]) + + +def file_size(ignite, path): + """ + :return: Bytes written into the file on the node. + """ + return int(ignite.exec_command(ignite.nodes[0], f"stat -c %s {path}")) + + +def check_write_failed(ignite, path): + """ + Checks that the file ends on a page boundary: a write the full tmpfs refused filled its last page. + """ + assert file_size(ignite, path) % PAGE == 0, f"No write into {path} hit the full volume" + + +def check_grows(ignite, path): + """ + Checks that the file grows within 15 s. + """ + size = file_size(ignite, path) + + wait_until(lambda: file_size(ignite, path) > size, timeout_sec=15, err_msg=f"{path} did not grow") + + +def check_serves_load_for(ignite, load, seconds): + """ + Waits the given seconds and checks that the node is alive and the puts went on without errors. + """ + puts, errors = progress(load) + + time.sleep(seconds) + + assert ignite.alive(ignite.nodes[0]), "The node stopped" + + puts_after, errors_after = progress(load) + + assert puts_after > puts, f"No puts in {seconds} s" + assert errors_after == errors, f"{errors_after - errors} puts failed in {seconds} s" + + +def check_stops(ignite, timeout_sec=60): + """ + Checks that the node's process ends. + """ + wait_until(lambda: not ignite.alive(ignite.nodes[0]), timeout_sec=timeout_sec, + err_msg=f"The node is still up {timeout_sec} s after its volume filled up") + + +def check_failure(ignite, failure_type, cause): + """ + Checks that the node logged a FailureContext of the type, caused by the writer and "No space left on device". + """ + node = ignite.nodes[0] + + failure = ignite.exec_command(node, f"grep -h -m1 -F -A60 'FailureContext [type={failure_type}' " + f"{node.log_file} || true") + + assert failure, f"No {failure_type} failure in the node's log" + assert cause in failure, f"The {failure_type} failure is not caused by '{cause}': {failure[:2000]}" + assert "No space left on device" in failure, f"The {failure_type} failure is not ENOSPC: {failure[:2000]}" + + +def progress(load): + """ + :return: Puts and failed puts the load application recorded last. + """ + puts, errors = load.extract_results("progress")[-1].split() + + return int(puts), int(errors) diff --git a/modules/ducktests/tests/ignitetest/tests/mdc/mdc_historical_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/mdc/mdc_historical_rebalance_test.py new file mode 100644 index 0000000000000..da3ac5c0f1ac3 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/mdc/mdc_historical_rebalance_test.py @@ -0,0 +1,142 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +MDC historical rebalance test: a returning data center must restore its partitions +from local disk (only the delta written during its absence), not receive a full +rebalance from the surviving DC. +""" +import re +from time import sleep + +from ignitetest.services.mdc.mdc_cluster import MdcCluster, cross_dc_network, DC_1, DC_2 +from ignitetest.services.utils.ignite_aware import IgniteAwareService +from ignitetest.services.utils.ignite_configuration.data_storage import DataStorageConfiguration, \ + DataRegionConfiguration +from ignitetest.utils import cluster, ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH + +CACHE_NAME = "mdc-historical-rebalance" + +BACKUPS = 1 + +BASE_ENTRIES = 1024 +DELTA_ENTRIES = 256 + +JVM_OPTS = ["-Xmx384M", "-Xms384M", "-DIGNITE_PREFER_WAL_REBALANCE=true"] + + +def _persistence_config() -> DataStorageConfiguration: + """ + Data storage configuration with persistence enabled on a small default data region. + Short checkpoint frequency ensures data reaches disk before DC failure. + """ + return DataStorageConfiguration( + checkpoint_frequency=10, + default=DataRegionConfiguration(persistence_enabled=True, + initial_size=16 * 1024 * 1024, + max_size=64 * 1024 * 1024) + ) + + +def _rebalance_metric(svc, metric: str) -> int: + """ + Extracts a numeric field from the last 'Completed (final) rebalancing' log line + of any node in the service. + """ + for node in svc.nodes: + cmd = f"grep 'Completed (final) rebalancing' {node.log_file} | tail -1" + output = IgniteAwareService.exec_command(node, cmd) + + if output: + m = re.search(rf"{metric}=(\d+)", output) + + if m: + return int(m.group(1)) + + raise AssertionError(f"Rebalance metric '{metric}' not found in {svc} logs") + + +class MdcHistoricalRebalanceTest(IgniteTest): + """ + Verifies that a data center returning after a full stop (DC failure) restores + its partitions from local disk and only receives the delta written during its + absence, rather than a full rebalance from the surviving DC. + """ + @cluster(num_nodes=4) + @ignite_versions(str(DEV_BRANCH)) + def test_returning_dc_recovers_from_disk(self, ignite_version): + """ + DC1 + DC2 with persistence enabled. Data is written, then DC2 goes down + (full stop, not a network partition). While DC2 is away, DC1 accepts new + writes (the delta). DC2 returns with clean=False (disk data preserved) and + must recover: all base data from disk, only the delta via rebalance. + """ + mdc = MdcCluster(self, ignite_version, srv_per_dc=1, runners_per_dc=1, + data_storage=_persistence_config(), + jvm_opts=JVM_OPTS) + + with cross_dc_network(self.logger, mdc): + mdc.start_servers() + + # With persistence the cluster starts INACTIVE; activate it explicitly. + mdc.control(DC_1).activate() + + # Phase 1: populate base data across both DCs. + mdc.generate_data(DC_1, CACHE_NAME, 0, BASE_ENTRIES // 2, backups=BACKUPS) + mdc.generate_data(DC_2, CACHE_NAME, BASE_ENTRIES // 2, BASE_ENTRIES, backups=BACKUPS) + + mdc.check_data(DC_1, CACHE_NAME, 0, BASE_ENTRIES) + mdc.check_data(DC_2, CACHE_NAME, 0, BASE_ENTRIES) + + mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=1) + + # Allow checkpoint to flush data to disk before the DC failure. + sleep(15) + + # Phase 2: DC2 goes down (full stop, simulating DC failure). + mdc.servers[DC_2].stop() + + # DC1 must remain healthy and accept writes. + mdc.check_put_admissibility(DC_1, CACHE_NAME, True) + + # Phase 3: write the delta on the surviving DC1. + mdc.generate_data(DC_1, CACHE_NAME, BASE_ENTRIES, BASE_ENTRIES + DELTA_ENTRIES, + backups=BACKUPS) + + mdc.check_data(DC_1, CACHE_NAME, 0, BASE_ENTRIES + DELTA_ENTRIES) + + # Phase 4: DC2 returns (disk data preserved, clean=False). + mdc.servers[DC_2].start(clean=False) + mdc.servers[DC_2].await_rebalance() + + # Phase 5: verify historical rebalance actually restored data from disk. + hist_entries = _rebalance_metric(mdc.servers[DC_2], "histEntries") + + self.logger.info(f"Historical rebalance: histEntries={hist_entries}") + + assert hist_entries > 0, \ + f"Expected data restored from local disk [histEntries={hist_entries}]" + + # Phase 6: verify full data integrity on both DCs. + mdc.check_data(DC_1, CACHE_NAME, 0, BASE_ENTRIES + DELTA_ENTRIES) + mdc.check_data(DC_2, CACHE_NAME, 0, BASE_ENTRIES + DELTA_ENTRIES) + + mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=1) + + mdc.verify_servers_log_clean() + + mdc.stop_servers() diff --git a/modules/ducktests/tests/ignitetest/utils/__init__.py b/modules/ducktests/tests/ignitetest/utils/__init__.py index 6d6da0d14ed12..2cc208c44baf0 100644 --- a/modules/ducktests/tests/ignitetest/utils/__init__.py +++ b/modules/ducktests/tests/ignitetest/utils/__init__.py @@ -17,6 +17,6 @@ This module contains convenient utils for test. """ -from ._mark import ignore_if, ignite_versions, cluster +from ._mark import ignore_if, ignite_versions, cluster, reproducer -__all__ = ['ignore_if', 'ignite_versions', 'cluster'] +__all__ = ['ignore_if', 'ignite_versions', 'cluster', 'reproducer'] diff --git a/modules/ducktests/tests/ignitetest/utils/_mark.py b/modules/ducktests/tests/ignitetest/utils/_mark.py index 9d0a06df4edc2..5964312fcc3c2 100644 --- a/modules/ducktests/tests/ignitetest/utils/_mark.py +++ b/modules/ducktests/tests/ignitetest/utils/_mark.py @@ -51,6 +51,26 @@ def __eq__(self, other): return super().__eq__(other) and self.condition == other.condition +class Reproducer(Ignore): + """ + Ignore test unless the global that enables it is set. + """ + def __init__(self, global_name): + super().__init__() + self.global_name = global_name + + def apply(self, seed_context, context_list): + assert len(context_list) > 0, "reproducer decorator is not being applied to any test cases" + + for ctx in context_list: + ctx.ignore = ctx.ignore or not ctx.globals.get(self.global_name) + + return context_list + + def __eq__(self, other): + return super().__eq__(other) and self.global_name == other.global_name + + class IgniteVersionParametrize(Mark): """ Parametrize function with ignite_version @@ -219,6 +239,20 @@ def ignorer(func): return ignorer +def reproducer(global_name): + """ + Mark decorated test method as IGNORE unless the global is set, e.g. --globals '{"": true}'. + For tests that reproduce a behavior on demand rather than guard it on every run. + + :param global_name: name of the global that enables the test. + """ + def ignorer(func): + Mark.mark(func, Reproducer(global_name)) + return func + + return ignorer + + def cluster(**kwargs): """Test method decorator used to provide hints about how the test will use the given cluster. diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/ResumeCreateIndexTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/ResumeCreateIndexTest.java index b9bc0844a380e..777c4c953d903 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/ResumeCreateIndexTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/ResumeCreateIndexTest.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; +import java.util.function.Consumer; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; @@ -41,6 +42,7 @@ import org.junit.Test; import static org.apache.ignite.IgniteSystemProperties.IGNITE_INDEX_REBUILD_BATCH_SIZE; +import static org.apache.ignite.cluster.ClusterState.ACTIVE; import static org.apache.ignite.internal.processors.query.aware.IndexBuildStatusHolder.Status.COMPLETE; import static org.apache.ignite.internal.processors.query.aware.IndexBuildStatusHolder.Status.INIT; import static org.apache.ignite.internal.processors.query.aware.IndexBuildStatusStorage.KEY_PREFIX; @@ -181,7 +183,7 @@ public void testErrorFlow() throws Exception { public void testConcurrentBuildNewIndexAndRebuildIndexes0() throws Exception { String cacheName = DEFAULT_CACHE_NAME; - IgniteEx n = prepareNodeToCreateNewIndex(cacheName, 100_000, true); + IgniteEx n = prepareNodeToCreateNewIndex(cacheName, 100_000, true, cfg -> cfg.setBuildIndexThreadPoolSize(2)); String idxName = "IDX0"; SlowdownBuildIndexConsumer slowdownIdxCreateConsumer = addSlowdownIdxCreateConsumer(n, idxName, 0); @@ -231,7 +233,7 @@ public void testConcurrentBuildNewIndexAndRebuildIndexes0() throws Exception { public void testConcurrentBuildNewIndexAndRebuildIndexes1() throws Exception { String cacheName = DEFAULT_CACHE_NAME; - IgniteEx n = prepareNodeToCreateNewIndex(cacheName, 100_000, true); + IgniteEx n = prepareNodeToCreateNewIndex(cacheName, 100_000, true, cfg -> cfg.setBuildIndexThreadPoolSize(2)); SlowdownBuildIndexConsumer slowdownRebuildIdxConsumer = addSlowdownRebuildIndexConsumer(n, cacheName, 10); assertTrue(forceRebuildIndexes(n, n.cachex(cacheName).context()).isEmpty()); @@ -426,19 +428,38 @@ private void checkStatus( assertEquals(expNewIdx, status.buildNewIndexes()); } + /** */ + private IgniteEx prepareNodeToCreateNewIndex(String cacheName, int cnt, boolean disableCp) throws Exception { + return prepareNodeToCreateNewIndex(cacheName, cnt, disableCp, null); + } + /** * Creating a node and filling the cache. * * @param cacheName Cache name. * @param cnt Entry count. * @param disableCp Disable checkpoint. + * @param cfgC Configuration mutator. * @return New node. * @throws Exception If failed. */ - private IgniteEx prepareNodeToCreateNewIndex(String cacheName, int cnt, boolean disableCp) throws Exception { + private IgniteEx prepareNodeToCreateNewIndex( + String cacheName, + int cnt, + boolean disableCp, + Consumer cfgC + ) throws Exception { IndexesRebuildTaskEx.prepareBeforeNodeStart(); - IgniteEx n = startGrid(0); + IgniteEx n; + + if (cfgC == null) + n = startGrid(0); + else { + n = startGrid(0, cfgC); + + n.cluster().state(ACTIVE); + } populate(n.cache(cacheName), cnt); diff --git a/modules/unsafe/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapEvent.java b/modules/unsafe/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapEvent.java deleted file mode 100644 index f2406789d6bd4..0000000000000 --- a/modules/unsafe/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapEvent.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap; - -import org.jetbrains.annotations.Nullable; - -/** - * Off-heap event types. - */ -public enum GridOffHeapEvent { - /** Rehash event. */ - REHASH, - - /** Memory allocate event. */ - ALLOCATE, - - /** Memory release event. */ - RELEASE; - - /** Enum values. */ - private static final GridOffHeapEvent[] VALS = values(); - - /** - * Efficiently gets enumerated value from its ordinal. - * - * @param ord Ordinal value. - * @return Enumerated value. - */ - @Nullable public static GridOffHeapEvent fromOrdinal(int ord) { - return ord >= 0 && ord < VALS.length ? VALS[ord] : null; - } -} diff --git a/modules/unsafe/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapEventListener.java b/modules/unsafe/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapEventListener.java deleted file mode 100644 index 856a0dee2712a..0000000000000 --- a/modules/unsafe/src/main/java/org/apache/ignite/internal/util/offheap/GridOffHeapEventListener.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.util.offheap; - -/** - * Off-heap event listener. - */ -public interface GridOffHeapEventListener { - /** Event notification. */ - public void onEvent(GridOffHeapEvent evt); -} diff --git a/modules/unsafe/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMemory.java b/modules/unsafe/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMemory.java index cbdc60cb88d9a..153c41705acf2 100644 --- a/modules/unsafe/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMemory.java +++ b/modules/unsafe/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMemory.java @@ -20,23 +20,16 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.ignite.IgniteCommonsSystemProperties; import org.apache.ignite.internal.util.GridUnsafe; -import org.apache.ignite.internal.util.offheap.GridOffHeapEventListener; import org.apache.ignite.internal.util.offheap.GridOffHeapOutOfMemoryException; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.lang.IgniteBiTuple; import static org.apache.ignite.IgniteCommonsSystemProperties.IGNITE_OFFHEAP_SAFE_RELEASE; -import static org.apache.ignite.internal.util.offheap.GridOffHeapEvent.ALLOCATE; -import static org.apache.ignite.internal.util.offheap.GridOffHeapEvent.RELEASE; /** * Unsafe memory. */ public class GridUnsafeMemory { - /** Free byte. */ - private static final byte FREE = (byte)0; - /** Safe offheap release flag. */ private static final boolean SAFE_RELEASE = IgniteCommonsSystemProperties.getBoolean(IGNITE_OFFHEAP_SAFE_RELEASE); @@ -48,13 +41,6 @@ public class GridUnsafeMemory { @GridToStringInclude private final AtomicLong allocated; - /** Total amount of memory allocated for system structures. */ - @GridToStringInclude - private final AtomicLong sysAllocated; - - /** Event listener. */ - private GridOffHeapEventListener lsnr; - /** * @param total Total size, {@code 0} for unlimited. */ @@ -64,17 +50,6 @@ public GridUnsafeMemory(long total) { this.total = total; allocated = new AtomicLong(); - - sysAllocated = new AtomicLong(); - } - - /** - * Sets event listener. - * - * @param lsnr Event listener. - */ - public void listen(GridOffHeapEventListener lsnr) { - this.lsnr = lsnr; } /** @@ -108,18 +83,6 @@ public long allocate(long size) throws GridOffHeapOutOfMemoryException { return allocate(size, false, false); } - /** - * Allocates memory of given size in bytes. - * - * @param size Size of allocated block. - * @param init Flag to zero-out the initialized memory or not. - * @return Allocated block address. - * @throws GridOffHeapOutOfMemoryException If Memory could not be allocated. - */ - public long allocate(long size, boolean init) throws GridOffHeapOutOfMemoryException { - return allocate(size, init, false); - } - /** * Allocates memory of given size in bytes. * @@ -133,18 +96,6 @@ public long allocate(long size, boolean init, boolean reserved) throws GridOffHe return allocate0(size, init, reserved, allocated); } - /** - * Allocates memory of given size in bytes, adds to system memory counter. - * - * @param size Size of allocated block. - * @param init Whether or not allocated block is zeroed upon return. - * @return Allocated block address. - * @throws GridOffHeapOutOfMemoryException If memory could not be allocated. - */ - public long allocateSystem(long size, boolean init) throws GridOffHeapOutOfMemoryException { - return allocate0(size, init, false, sysAllocated); - } - /** * Performs actual memory allocation. * @@ -170,9 +121,6 @@ private long allocate0(long size, boolean init, boolean reserved, if (init) GridUnsafe.zeroMemory(ptr, size); - if (lsnr != null) - lsnr.onEvent(ALLOCATE); - return ptr; } catch (OutOfMemoryError ignore) { @@ -202,16 +150,6 @@ public void release(long ptr, long size) { release0(ptr, size, allocated); } - /** - * Releases memory allocated by {@link #allocateSystem(long, boolean)}. - * - * @param ptr Address of memory block to deallocate. - * @param size Size of allocated block. - */ - public void releaseSystem(long ptr, long size) { - release0(ptr, size, sysAllocated); - } - /** * Internal release procedure. Decreases size of corresponding counter. * @@ -227,9 +165,6 @@ private void release0(long ptr, long size, AtomicLong cnt) { GridUnsafe.freeMemory(ptr); cnt.addAndGet(-size); - - if (lsnr != null) - lsnr.onEvent(RELEASE); } } @@ -397,66 +332,6 @@ public void writeByteVolatile(long ptr, byte v) { GridUnsafe.putByteVolatile(null, ptr, v); } - /** - * Stores value to the specified memory location. If specified pointer is {@code 0}, then will - * allocate required space. If size of allocated space is not enough to hold given values, will - * reallocate memory. - * - * @param ptr Optional pointer to allocated memory. First 4 bytes in allocated region must contain - * size of allocated chunk. - * @param val Value to store. - * @param type Value type. - * @return Pointer. - */ - public long putOffHeap(long ptr, byte[] val, byte type) { - int size = val.length; - - assert size != 0; - - int allocated = ptr == 0 ? 0 : readInt(ptr); - - if (allocated != size) { - if (ptr != 0) - release(ptr, allocated + 5); - - ptr = allocate(size + 5); - - writeInt(ptr, size); - } - - writeByte(ptr + 4, type); - writeBytes(ptr + 5, val); - - return ptr; - } - - /** - * Releases off-heap memory allocated by {@link #putOffHeap} method. - * - * @param ptr Optional pointer returned by {@link #putOffHeap}. - */ - public void removeOffHeap(long ptr) { - if (ptr != 0) - release(ptr, readInt(ptr) + 5); - } - - /** - * Get value stored in offheap along with a value type. - * - * @param ptr Pointer to read. - * @return Stored byte array and "raw bytes" flag. - */ - public IgniteBiTuple get(long ptr) { - assert ptr != 0; - - int size = readInt(ptr); - - byte type = readByte(ptr + 4); - byte[] bytes = readBytes(ptr + 5, size); - - return new IgniteBiTuple<>(bytes, type); - } - /** * @param ptr1 First pointer. * @param ptr2 Second pointer. @@ -658,15 +533,6 @@ public void copyMemory(long srcPtr, long destPtr, long len) { GridUnsafe.copyOffheapOffheap(srcPtr, destPtr, len); } - /** - * Checks if direct memory allocation is limited to some value. - * - * @return {@code True} if memory allocation is limited. - */ - public boolean unlimited() { - return totalSize() == 0; - } - /** * @return Total size. */ @@ -674,18 +540,6 @@ public long totalSize() { return total; } - /** - * @return Free size. - */ - public long freeSize() { - if (total == 0) - return 0; - - long diff = total - allocated.get(); - - return diff < 0 ? 0 : diff; - } - /** * @return Allocated size. */ @@ -693,13 +547,6 @@ public long allocatedSize() { return allocated.get(); } - /** - * @return Size of memory allocated with {@link #allocateSystem(long, boolean)}. - */ - public long systemAllocatedSize() { - return sysAllocated.get(); - } - /** {@inheritDoc} */ @Override public String toString() { return S.toString(GridUnsafeMemory.class, this); diff --git a/modules/wire-compatibility-check/wire/enums/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation.json b/modules/wire-compatibility-check/wire/enums/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation.json new file mode 100644 index 0000000000000..e17d08090b831 --- /dev/null +++ b/modules/wire-compatibility-check/wire/enums/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation.json @@ -0,0 +1,8 @@ +{ + "class": "org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation", + "fields": [ + {"order": 0, "type": "org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation", "name": "CREATE"}, + {"order": 1, "type": "org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation", "name": "RESTORE"}, + {"order": 2, "type": "org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation", "name": "CHECK"} + ] +} diff --git a/modules/wire-compatibility-check/wire/idto/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperationStatus.json b/modules/wire-compatibility-check/wire/idto/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperationStatus.json new file mode 100644 index 0000000000000..0d052889397b2 --- /dev/null +++ b/modules/wire-compatibility-check/wire/idto/org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperationStatus.json @@ -0,0 +1,11 @@ +{ + "class": "org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperationStatus", + "fields": [ + {"order": 0, "type": "org.apache.ignite.internal.management.snapshot.SnapshotStatusTask$SnapshotOperation", "name": "op"}, + {"order": 1, "type": "java.lang.String", "name": "name"}, + {"order": 2, "type": "int", "name": "incIdx"}, + {"order": 3, "type": "java.lang.String", "name": "reqId"}, + {"order": 4, "type": "long", "name": "startTime"}, + {"order": 5, "type": "java.util.Map>", "name": "progress"} + ] +} From 617884596142ed0ae77ae7d975241c6e235987c0 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Thu, 29 Oct 2026 21:36:16 +0300 Subject: [PATCH 38/44] better test --- ...SnapshotCheckStatusRollingUpgradeTest.java | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java index 543cb07846ae0..f6ae09abc9d47 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java @@ -140,29 +140,32 @@ public void testClientOfTheNextVersionWithCreateInProgress() throws Exception { createCache(0); - // Blocks the finalization of the create operation to keep it in progress while the status is requested. + // The snapshot to check. It must be created before the messages are blocked. + createSnapshot(0, "testSnapshotToCheck"); + + // Blocks the finalization of the create operation and the check operation to keep them in progress while + // the status is requested. for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) { spi(grid(i)).blockMessages((node, msg) -> - msg instanceof SingleNodeMessage snm && snm.type() == END_SNAPSHOT.ordinal()); + msg instanceof SingleNodeMessage snm && + (snm.type() == END_SNAPSHOT.ordinal() || snm.type() == CHECK_SNAPSHOT_METAS.ordinal())); } + int chkNodeIdx = 1; + + IgniteInternalFuture chkFut = snp(chkNodeIdx).checkSnapshot("testSnapshotToCheck", null); IgniteFuture createFut = snp(0).createSnapshot(SNP_NAME); try { - assertTrue("Snapshot create operation has not started.", waitForCondition(() -> { - for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) { - if (snp(i).currentCreateRequest() == null) - return false; - } - - return true; - }, 10_000L)); + assertTrue(waitForCondition( + () -> snp(chkNodeIdx).isSnapshotCreating() && snp(chkNodeIdx).isSnapshotChecking("testSnapshotToCheck"), 10_000L)); Collection nodes = Collections.singleton(nodeId(0)); Collection statuses = snapshotStatusTaskWithClient(new VisorTaskArgument<>(nodes, new NoArg(), false)); + // The check operation is not reported by the cluster of the previous version. assertEquals(1, statuses.size()); SnapshotStatusTask.SnapshotOperationStatus status = F.first(statuses); @@ -177,6 +180,13 @@ public void testClientOfTheNextVersionWithCreateInProgress() throws Exception { } createFut.get(10_000L); + + // Ensure that the snapshot check process correctly finished too. + SnapshotPartitionsVerifyResult chkRes = chkFut.get(getTestTimeout()); + + assertTrue(F.isEmpty(chkRes.exceptions())); + assertTrue(F.isEmpty(chkRes.idleVerifyResult().exceptions())); + assertFalse(chkRes.idleVerifyResult().hasConflicts()); } /** */ @@ -299,7 +309,24 @@ private void createCache(int gridIdx) { /** */ private void createSnapshot(int gridIdx) { - snp(gridIdx).createSnapshot(SNP_NAME).get(getTestTimeout()); + createSnapshot(gridIdx, SNP_NAME); + } + + /** */ + private void createSnapshot(int gridIdx, String snpName) { + snp(gridIdx).createSnapshot(snpName).get(getTestTimeout()); + } + + /** @return {@code True} if the create operation of the snapshot {@code snpName} is in progress on all the server nodes. */ + private boolean createInProgress(String snpName) { + for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) { + SnapshotOperationRequest req = snp(i).currentCreateRequest(); + + if (req == null || !snpName.equals(req.snapshotName())) + return false; + } + + return true; } /** */ From 2d14d0e56b0fd5a5fd3e895922329c9212d2dd38 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 30 Oct 2026 15:44:35 +0300 Subject: [PATCH 39/44] manual review fixes --- .../ignite/util/GridCommandHandlerTest.java | 135 ++++++-------- .../snapshot/SnapshotStatusCommand.java | 23 +-- .../snapshot/SnapshotStatusTask.java | 54 ++---- .../snapshot/SnapshotCheckProcess.java | 2 +- .../snapshot/SnapshotMXBeanImpl.java | 3 +- ...SnapshotCheckStatusRollingUpgradeTest.java | 168 ++++++------------ 6 files changed, 144 insertions(+), 241 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 0902154f7d252..b06cd399e8595 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3669,7 +3669,7 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { startGrids(3).cluster().state(ClusterState.ACTIVE); - createCacheAndPreload(grid(1), 1000); + createCacheAndPreload(grid(1), 100); IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot(); @@ -3679,22 +3679,14 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { awaitPartitionMapExchange(); - TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); - TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); + TestRecordingCommunicationSpi cm = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); - // Block one of the process' first messages of snapshot restoring or snapshot checking. - F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm - && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() || sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()))); + cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm + && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal())); - // Snapshot restoration should get paused at the preceeding snapshot check. IgniteFutureImpl restoreFut = snpMgr.restoreSnapshot("test_snapshot", null, null, 0, true); - // Waiting for each node to send snapshot check single result. - for (var cm : F.asList(cm1, cm2)) { - assertTrue(waitForCondition(() -> cm.blockedMessages().stream().anyMatch(m -> - m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()), - getTestTimeout())); - } + assertTrue(waitForCondition(cm::hasBlockedMessages, getTestTimeout())); injectTestSystemOut(); @@ -3707,16 +3699,9 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { assertContains(log, out, "Check snapshot operation is in progress"); assertContains(log, out, "Snapshot name: test_snapshot"); - // Ensure that no snapshot restore started or finished. assertFalse("Snapshot future has finished", restoreFut.isDone()); - F.asList(cm1, cm2).forEach(cm -> { - assertTrue(cm.blockedMessages().stream().noneMatch(m -> - m.ioMessage().message() instanceof SingleNodeMessage sm - && sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal())); - - cm.stopBlock(); - }); + cm.stopBlock(); restoreFut.get(getTestTimeout()); } @@ -3753,12 +3738,12 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot(); - createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 1000, 32, null); + createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 100, 32, null); snpMgr.createSnapshot("testSnapshot0").get(getTestTimeout()); if (incremental) { try (IgniteDataStreamer streamer = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { - for (int i = 1000; i < 2000; i++) + for (int i = 100; i < 110; i++) streamer.addData(i, i); } @@ -3766,12 +3751,11 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment } if (twoSnapshots) { - createCacheAndPreload(grid(1), "cache2", 1000, 32, null); snpMgr.createSnapshot("testSnapshot1").get(getTestTimeout()); if (incremental) { - try (IgniteDataStreamer streamer = grid(0).dataStreamer("cache2")) { - for (int i = 1000; i < 2000; i++) + try (IgniteDataStreamer streamer = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 110; i < 120; i++) streamer.addData(i, i); } @@ -3779,30 +3763,26 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment } } - TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); - TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); + TestRecordingCommunicationSpi cm = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); - F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> - msg instanceof SingleNodeMessage sm && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()))); + cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm && + (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal())); - var checkFut0 = runAsync(() -> incremental + IgniteInternalFuture checkFut0 = runAsync(() -> incremental ? execute("--snapshot", "check", "testSnapshot0", "--increment", "1") : execute("--snapshot", "check", "testSnapshot0")); - var checkFut1 = twoSnapshots + IgniteInternalFuture checkFut1 = twoSnapshots ? runAsync(() -> incremental ? execute("--snapshot", "check", "testSnapshot1", "--increment", "1") : execute("--snapshot", "check", "testSnapshot1")) : null; - // Waiting for the nodes each to send snapshot check single result. - for (var cm : F.asList(cm1, cm2)) { - assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m -> - m.ioMessage().message() instanceof SingleNodeMessage sm - && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoSnapshots ? 2 : 1), - getTestTimeout() - )); - } + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoSnapshots ? 2 : 1), + getTestTimeout() + )); injectTestSystemOut(); @@ -3813,11 +3793,13 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment if (log.isInfoEnabled()) log.info("Test out:" + U.nl() + out); - String chkSnpE = "Check snapshot operation is in progress"; - var chkSnpESum = Arrays.stream(out.split(U.nl())) - .mapToInt(l -> (l.length() - l.replace(chkSnpE, "").length()) / chkSnpE.length()) + String checkSnpErr = "Check snapshot operation is in progress"; + + int errSum = Arrays.stream(out.split(U.nl())) + .mapToInt(l -> (l.length() - l.replace(checkSnpErr, "").length()) / checkSnpErr.length()) .sum(); - assertEquals(twoSnapshots ? 2 : 1, chkSnpESum); + + assertEquals(twoSnapshots ? 2 : 1, errSum); assertTrue(out.contains("Snapshot name: testSnapshot0")); @@ -3837,7 +3819,7 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment } } - F.asList(cm1, cm2).forEach(TestRecordingCommunicationSpi::stopBlock); + cm.stopBlock(); checkFut0.get(); @@ -3863,72 +3845,67 @@ private void doTestSnapshotsChecksStatusWithParallelCreate(boolean twoChecks) th IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot(); - createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 1000, 32, null); + createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 100, 32, null); snpMgr.createSnapshot("testSnapshot0").get(getTestTimeout()); if (twoChecks) { - createCacheAndPreload(grid(1), "cache2", 1000, 32, null); + createCacheAndPreload(grid(1), "cache2", 100, 32, null); snpMgr.createSnapshot("testSnapshot1").get(getTestTimeout()); } - createCacheAndPreload(grid(1), "cache3", 1000, 32, null); + createCacheAndPreload(grid(1), "cache3", 100, 32, null); - TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); - TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); + TestRecordingCommunicationSpi cm = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); - F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> + cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() - || sm.type() == END_SNAPSHOT.ordinal())) - ); + || sm.type() == END_SNAPSHOT.ordinal())); - var createFut = snpMgr.createSnapshot("testSnapshot2"); + IgniteFuture createFut = snpMgr.createSnapshot("testSnapshot2"); - var checkFut0 = runAsync(() -> execute("--snapshot", "check", "testSnapshot0")); - var checkFut1 = twoChecks ? runAsync(() -> execute("--snapshot", "check", "testSnapshot1")) : null; + IgniteInternalFuture checkFut0 = runAsync(() -> execute("--snapshot", "check", "testSnapshot0")); + IgniteInternalFuture checkFut1 = twoChecks ? runAsync(() -> execute("--snapshot", "check", "testSnapshot1")) : null; - // Waiting for the nodes each to send snapshot create single result. - for (var cm : F.asList(cm1, cm2)) { - assertTrue(waitForCondition(() -> cm.blockedMessages().stream().anyMatch(m -> - m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == END_SNAPSHOT.ordinal()), - getTestTimeout() - )); - } + // Wait for the creation gets blocked. + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().anyMatch(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == END_SNAPSHOT.ordinal()), + getTestTimeout() + )); - // Waiting for the nodes each to send snapshot check single result. - for (var cm : F.asList(cm1, cm2)) { - assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m -> - m.ioMessage().message() instanceof SingleNodeMessage sm - && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoChecks ? 2 : 1), - getTestTimeout() - )); - } + // Wait for the checks get blocked. + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoChecks ? 2 : 1), + getTestTimeout() + )); injectTestSystemOut(); assertEquals("Unexpected exit code", EXIT_CODE_OK, execute("--snapshot", "status")); - var out = testOut.toString(); - - if (log.isInfoEnabled()) - log.info("Test out:" + U.nl() + out); + String out = testOut.toString(); assertTrue(out.contains("Create snapshot operation is in progress")); assertTrue(out.contains("Snapshot name: testSnapshot2")); - String chkLogE = "Check snapshot operation is in progress"; - var chkLogSum = Arrays.stream(out.split(U.nl())) - .mapToInt(l -> (l.length() - l.replace(chkLogE, "").length()) / chkLogE.length()) + String checkSnpLogErr = "Check snapshot operation is in progress"; + + int chkLogSum = Arrays.stream(out.split(U.nl())) + .mapToInt(l -> (l.length() - l.replace(checkSnpLogErr, "").length()) / checkSnpLogErr.length()) .sum(); + assertEquals(twoChecks ? 2 : 1, chkLogSum); assertTrue(out.contains("Snapshot name: testSnapshot0")); + if (twoChecks) assertTrue(out.contains("Snapshot name: testSnapshot1")); - F.asList(cm1, cm2).forEach(TestRecordingCommunicationSpi::stopBlock); + cm.stopBlock(); createFut.get(); checkFut0.get(); + if (twoChecks) checkFut1.get(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index bc2a2e3b916f7..08f6ccdc165fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -71,7 +71,8 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand statuses = rawResult instanceof Collection ? (Collection)rawResult : null; @@ -83,7 +84,9 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand titles; /** */ - SnapshotTaskProgressDesc(List titles) { + private SnapshotTaskProgressDesc(List titles) { this.titles = Collections.unmodifiableList(titles); } /** @return Progress table columns titles. */ - List titles() { + private List titles() { return titles; } /** @return Progress table columns types. */ - List types() { + private List types() { List types = new ArrayList<>(); types.add(STRING); @@ -165,13 +168,13 @@ List types() { } /** @return Progress table data row. */ - abstract List buildRow(UUID nodeId, T5 progress); + protected abstract List buildRow(UUID nodeId, T5 progress); } /** */ private static class CreateFullSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { /** */ - CreateFullSnapshotTaskProgressDesc() { + private CreateFullSnapshotTaskProgressDesc() { super(F.asList("Node ID", "Processed, bytes", "Total, bytes", "Percent")); } @@ -192,7 +195,7 @@ private static class CreateFullSnapshotTaskProgressDesc extends SnapshotTaskProg /** */ private static class CreateIncrementalSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { /** */ - CreateIncrementalSnapshotTaskProgressDesc() { + private CreateIncrementalSnapshotTaskProgressDesc() { super(F.asList("Node ID", "Progress")); } @@ -205,7 +208,7 @@ private static class CreateIncrementalSnapshotTaskProgressDesc extends SnapshotT /** */ private static class RestoreFullSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { /** */ - RestoreFullSnapshotTaskProgressDesc() { + private RestoreFullSnapshotTaskProgressDesc() { super(F.asList("Node ID", "Processed, partitions", "Total, partitions", "Percent")); } @@ -226,7 +229,7 @@ private static class RestoreFullSnapshotTaskProgressDesc extends SnapshotTaskPro /** */ private static class RestoreIncrementalSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { /** */ - RestoreIncrementalSnapshotTaskProgressDesc() { + private RestoreIncrementalSnapshotTaskProgressDesc() { super(F.asList( "Node ID", "Processed, partitions", diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index bc631d6159b33..cda0f85bf21ff 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -36,7 +36,6 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperationRequest; import org.apache.ignite.internal.processors.metric.impl.MetricUtils; import org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeatureSet; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T5; @@ -71,8 +70,8 @@ public class SnapshotStatusTask extends VisorMultiNodeTask job(NoArg arg) { @@ -81,7 +80,8 @@ public class SnapshotStatusTask extends VisorMultiNodeTask jobNodes(VisorTaskArgument arg) { - cmdInitiatorFeatures = arg.initiatorFeatures(); + featureEnabled = ignite.context().rollingUpgrade().features().isActive(SNAPSHOT_CHECK_STATUS_FEATURE) && + arg.initiatorFeatures().contains(SNAPSHOT_CHECK_STATUS_FEATURE.id());; return nodeIds(ignite.cluster().forServers().nodes()); } @@ -97,11 +97,6 @@ public class SnapshotStatusTask extends VisorMultiNodeTask allStatuses = new ArrayList<>(results.size()); @@ -109,36 +104,19 @@ public class SnapshotStatusTask extends VisorMultiNodeTask jobMultipleStatuses = (Collection)rawResult; - - // Skip the check status for an old command initiator that doesn't know about it. - if (!initiatorSupports) { - jobMultipleStatuses = jobMultipleStatuses.stream().filter(s -> s.operation() != SnapshotOperation.CHECK).toList(); - - assert jobMultipleStatuses.isEmpty() || jobMultipleStatuses.size() == 1 : - "Snapshot creation and restoration are mutually exclusive; only one of them can run at a time."; - } - - allStatuses.addAll(jobMultipleStatuses); - } + if (rawResult instanceof SnapshotStatus singleStatus) + allStatuses.add(new SnapshotOperationStatus(singleStatus)); + else + allStatuses.addAll((Collection)rawResult); } - // Empty list notifies that this is the new result but is empty. Old empty is always null. - if (allStatuses.isEmpty()) - return initiatorSupports && featureEnabled ? (Serializable)Collections.emptyList() : null; - ArrayList taskResult = new ArrayList<>(); // First, find create or restore operation. Collection filtered = F.viewReadOnly(allStatuses, s -> s, s -> s.operation() != SnapshotOperation.CHECK); - // Found create or restore statuses. if (!filtered.isEmpty()) { // Merged create or restore. SnapshotOperationStatus merged = F.first(filtered); @@ -148,21 +126,25 @@ public class SnapshotStatusTask extends VisorMultiNodeTask s.reqId.equals(mergedRqId)); - // Merge nodes progress. + // Progress per node. Map> progress = new HashMap<>(); filtered.forEach(s -> progress.putAll(s.progress)); merged = new SnapshotOperationStatus(merged.op, merged.name, merged.incIdx, merged.reqId, merged.startTime, progress); - // If the initiator like thin client of 'control.sh' doesn't support the feature yet, we have to rewrap - // the first single result back to the old format. - if (!initiatorSupports) - return new SnapshotStatus(merged.op, merged.name, merged.incIdx, merged.reqId, merged.startTime, progress); - taskResult.add(merged); } + if (!featureEnabled) { + if (taskResult.isEmpty()) + return null; + + SnapshotOperationStatus s0 = taskResult.get(0); + + return new SnapshotStatus(s0.op, s0.name, s0.incIdx, s0.reqId, s0.startTime, s0.progress); + } + // Check operation statuses go last. filtered = F.viewReadOnly(allStatuses, s -> s, s -> s.operation() == SnapshotOperation.CHECK); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index ebe9ac1020a4d..84cf4a6d34618 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -793,7 +793,7 @@ private static final class SnapshotCheckContext { /** All the snapshot metadatas. */ @Nullable private Map> clusterMetas; - /** Common counter of total work units to process on current node, of all snapshot parts ({@link #checkedSnapshotParts}). */ + /** Common counter of total work units to process on current node. */ @GridToStringExclude private final AtomicInteger totalCounter = new AtomicInteger(-1); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java index fac5b21c9c39a..bfd507a2c4d44 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java @@ -37,8 +37,7 @@ */ public class SnapshotMXBeanImpl implements SnapshotMXBean { /** */ - public static final String NO_CREATE_OR_RESTORE_IN_PROGRESS_MSG = - "There is no create or restore snapshot operation in progress."; + public static final String NO_CREATE_OR_RESTORE_IN_PROGRESS_MSG = "There is no create or restore snapshot operation in progress."; /** Instance of snapshot cache shared manager. */ private final IgniteSnapshotManager mgr; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java index f6ae09abc9d47..c42fd32a117f6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java @@ -109,8 +109,7 @@ public class IgniteClusterSnapshotCheckStatusRollingUpgradeTest extends Abstract .setDefaultDataRegionConfiguration( new DataRegionConfiguration() .setPersistenceEnabled(true) - .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE) - ) + .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)) ); cfg.setWorkDirectory(new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath()); @@ -121,13 +120,18 @@ public class IgniteClusterSnapshotCheckStatusRollingUpgradeTest extends Abstract /** */ @Test public void testClientOfThePreviousVersion() throws Exception { - doTestClientOfVersion(MASTER_VER_REF, "2.18.0"); - } + for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) + startGrid(i, MASTER_VER_REF, false); - /** */ - @Test - public void testClientOfTheNextVersion() throws Exception { - doTestClientOfVersion("2.18.0", MASTER_VER_REF); + grid(0).cluster().state(ClusterState.ACTIVE); + + createCacheAndSnapshot(0); + + Collection nodes = Collections.singleton(nodeId(0)); + + VisorTaskArgument vtArg = withCoreVersion("2.18.0", () -> new VisorTaskArgument<>(nodes, new NoArg(), false)); + + validateSnapshotCheckStatus(false, vtArg); } /** */ @@ -140,11 +144,8 @@ public void testClientOfTheNextVersionWithCreateInProgress() throws Exception { createCache(0); - // The snapshot to check. It must be created before the messages are blocked. - createSnapshot(0, "testSnapshotToCheck"); + snp(0).createSnapshot("testSnapshotToCheck").get(getTestTimeout()); - // Blocks the finalization of the create operation and the check operation to keep them in progress while - // the status is requested. for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) { spi(grid(i)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && @@ -162,14 +163,9 @@ public void testClientOfTheNextVersionWithCreateInProgress() throws Exception { Collection nodes = Collections.singleton(nodeId(0)); - Collection statuses = + SnapshotStatusTask.SnapshotStatus status = snapshotStatusTaskWithClient(new VisorTaskArgument<>(nodes, new NoArg(), false)); - // The check operation is not reported by the cluster of the previous version. - assertEquals(1, statuses.size()); - - SnapshotStatusTask.SnapshotOperationStatus status = F.first(statuses); - assertEquals(SnapshotOperation.CREATE, status.operation()); assertEquals(SNP_NAME, status.name()); assertEquals(ALL_GRIDS - CLIENTS, status.progress().size()); @@ -180,29 +176,7 @@ public void testClientOfTheNextVersionWithCreateInProgress() throws Exception { } createFut.get(10_000L); - - // Ensure that the snapshot check process correctly finished too. - SnapshotPartitionsVerifyResult chkRes = chkFut.get(getTestTimeout()); - - assertTrue(F.isEmpty(chkRes.exceptions())); - assertTrue(F.isEmpty(chkRes.idleVerifyResult().exceptions())); - assertFalse(chkRes.idleVerifyResult().hasConflicts()); - } - - /** */ - private void doTestClientOfVersion(String clusterVer, String clientVer) throws Exception { - for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) - startGrid(i, clusterVer, false); - - grid(0).cluster().state(ClusterState.ACTIVE); - - createCacheAndSnapshot(0); - - Collection nodes = Collections.singleton(nodeId(0)); - - VisorTaskArgument vtArg = withCoreVersion(clientVer, () -> new VisorTaskArgument<>(nodes, new NoArg(), false)); - - validateSnapshotCheckStatusFeature(false, vtArg); + chkFut.get(10_000L); } /** */ @@ -217,7 +191,7 @@ public void testParallelRollingUpgradeInProgress() throws Exception { createCacheAndSnapshot(testNodeIx); - validateSnapshotCheckStatusFeature(false); + validateSnapshotCheckStatus(false); ru(grid(testNodeIx)).enableVersionUpgrade(); @@ -234,7 +208,7 @@ public void testParallelRollingUpgradeInProgress() throws Exception { assertTrue(spi(grid(testNodeIx)).waitForBlocked(1, getTestTimeout())); - validateSnapshotCheckStatusFeature(false); + validateSnapshotCheckStatus(false); spi(grid(testNodeIx)).stopBlock(); @@ -248,7 +222,7 @@ public void testParallelRollingUpgradeInProgress() throws Exception { assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout())); } - validateSnapshotCheckStatusFeature(true); + validateSnapshotCheckStatus(true); } /** */ @@ -261,7 +235,7 @@ public void testSnapshotStatusFeature() throws Exception { createCacheAndSnapshot(1); - validateSnapshotCheckStatusFeature(false); + validateSnapshotCheckStatus(false); ru(grid(0)).enableVersionUpgrade(); @@ -271,7 +245,7 @@ public void testSnapshotStatusFeature() throws Exception { upgradeNodeVersion(i, "2.18.0", MASTER_VER_REF); } - validateSnapshotCheckStatusFeature(false); + validateSnapshotCheckStatus(false); ru(grid(1)).finalizeClusterVersion(); @@ -281,56 +255,11 @@ public void testSnapshotStatusFeature() throws Exception { assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout())); } - validateSnapshotCheckStatusFeature(true); - } - - /** */ - private void createCacheAndSnapshot(int gridIdx) { - createCache(gridIdx); - - createSnapshot(gridIdx); - } - - /** Creates a replicated cache and fills it with data. */ - private void createCache(int gridIdx) { - int partsCnt = 4; - int keysCnt = partsCnt * 5; - - grid(gridIdx).createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) - .setCacheMode(CacheMode.REPLICATED) - .setAffinity(new RendezvousAffinityFunction().setPartitions(partsCnt)) - .setAtomicityMode(CacheAtomicityMode.ATOMIC)); - - try (IgniteDataStreamer ds = grid(gridIdx).dataStreamer(DEFAULT_CACHE_NAME)) { - for (int i = 0; i < keysCnt; i++) - ds.addData(i, i); - } - } - - /** */ - private void createSnapshot(int gridIdx) { - createSnapshot(gridIdx, SNP_NAME); - } - - /** */ - private void createSnapshot(int gridIdx, String snpName) { - snp(gridIdx).createSnapshot(snpName).get(getTestTimeout()); - } - - /** @return {@code True} if the create operation of the snapshot {@code snpName} is in progress on all the server nodes. */ - private boolean createInProgress(String snpName) { - for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) { - SnapshotOperationRequest req = snp(i).currentCreateRequest(); - - if (req == null || !snpName.equals(req.snapshotName())) - return false; - } - - return true; + validateSnapshotCheckStatus(true); } /** */ - private void validateSnapshotCheckStatusFeature(boolean supports, @Nullable VisorTaskArgument clientArg) throws Exception { + private void validateSnapshotCheckStatus(boolean supports, @Nullable VisorTaskArgument clientArg) throws Exception { int blockNodeIdx = 1; spi(grid(blockNodeIdx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && @@ -365,25 +294,12 @@ private void validateSnapshotCheckStatusFeature(boolean supports, @Nullable Viso spi(grid(blockNodeIdx)).stopBlock(); - assertTrue(waitForCondition(() -> !spi(grid(blockNodeIdx)).hasBlockedMessages(), getTestTimeout())); - - // Ensure that the snapshot check process correctly finished too. - SnapshotPartitionsVerifyResult chkRes = chkFut.get(getTestTimeout()); - - assertTrue(F.isEmpty(chkRes.exceptions())); - assertFalse(F.isEmpty(chkRes.metas())); - assertTrue(F.isEmpty(chkRes.idleVerifyResult().exceptions())); - assertFalse(chkRes.idleVerifyResult().hasConflicts()); - } - - /** */ - private void validateSnapshotCheckStatusFeature(boolean supports) throws Exception { - validateSnapshotCheckStatusFeature(supports, null); + chkFut.get(10_000L); } /** */ - private IgniteSnapshotManager snp(int gridIdx) { - return grid(gridIdx).context().cache().context().snapshotMgr(); + private void validateSnapshotCheckStatus(boolean supports) throws Exception { + validateSnapshotCheckStatus(supports, null); } /** */ @@ -396,13 +312,39 @@ private static T snapshotStatusTaskWithGrid(Ignite grid } /** */ - private T snapshotStatusTaskWithClient(VisorTaskArgument vtArg) throws Exception { - String taskName = SnapshotStatusTask.class.getName(); - + private static T snapshotStatusTaskWithClient(VisorTaskArgument vtArg) throws Exception { try (IgniteClient cli = Ignition.startClient(new ClientConfiguration().setAddresses(Config.SERVER))) { - VisorTaskResult vtRes = cli.compute().execute(taskName, vtArg); + VisorTaskResult vtRes = cli.compute().execute(SnapshotStatusTask.class.getName(), vtArg); return vtRes.result(); } } + + /** */ + private IgniteSnapshotManager snp(int gridIdx) { + return grid(gridIdx).context().cache().context().snapshotMgr(); + } + + /** */ + private void createCacheAndSnapshot(int gridIdx) { + createCache(gridIdx); + + snp(gridIdx).createSnapshot(SNP_NAME).get(getTestTimeout()); + } + + /** Creates a replicated cache and fills it with data. */ + private void createCache(int gridIdx) { + int partsCnt = 4; + int keysCnt = partsCnt * 5; + + grid(gridIdx).createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setCacheMode(CacheMode.REPLICATED) + .setAffinity(new RendezvousAffinityFunction().setPartitions(partsCnt)) + .setAtomicityMode(CacheAtomicityMode.ATOMIC)); + + try (IgniteDataStreamer ds = grid(gridIdx).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 0; i < keysCnt; i++) + ds.addData(i, i); + } + } } From 7c25ead41ca92f5dce9ea686d68ff4a28daa26cc Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 30 Oct 2026 15:44:35 +0300 Subject: [PATCH 40/44] manual review fixes --- .../ignite/util/GridCommandHandlerTest.java | 135 ++++++-------- .../snapshot/SnapshotStatusCommand.java | 23 +-- .../snapshot/SnapshotStatusTask.java | 54 ++---- .../snapshot/SnapshotCheckProcess.java | 2 +- .../snapshot/SnapshotMXBeanImpl.java | 3 +- ...SnapshotCheckStatusRollingUpgradeTest.java | 165 ++++++------------ 6 files changed, 140 insertions(+), 242 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 0902154f7d252..b06cd399e8595 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3669,7 +3669,7 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { startGrids(3).cluster().state(ClusterState.ACTIVE); - createCacheAndPreload(grid(1), 1000); + createCacheAndPreload(grid(1), 100); IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot(); @@ -3679,22 +3679,14 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { awaitPartitionMapExchange(); - TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); - TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); + TestRecordingCommunicationSpi cm = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); - // Block one of the process' first messages of snapshot restoring or snapshot checking. - F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm - && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() || sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal()))); + cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm + && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal())); - // Snapshot restoration should get paused at the preceeding snapshot check. IgniteFutureImpl restoreFut = snpMgr.restoreSnapshot("test_snapshot", null, null, 0, true); - // Waiting for each node to send snapshot check single result. - for (var cm : F.asList(cm1, cm2)) { - assertTrue(waitForCondition(() -> cm.blockedMessages().stream().anyMatch(m -> - m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()), - getTestTimeout())); - } + assertTrue(waitForCondition(cm::hasBlockedMessages, getTestTimeout())); injectTestSystemOut(); @@ -3707,16 +3699,9 @@ public void testRestoreSnapshotMetricsAtStart() throws Exception { assertContains(log, out, "Check snapshot operation is in progress"); assertContains(log, out, "Snapshot name: test_snapshot"); - // Ensure that no snapshot restore started or finished. assertFalse("Snapshot future has finished", restoreFut.isDone()); - F.asList(cm1, cm2).forEach(cm -> { - assertTrue(cm.blockedMessages().stream().noneMatch(m -> - m.ioMessage().message() instanceof SingleNodeMessage sm - && sm.type() == RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE.ordinal())); - - cm.stopBlock(); - }); + cm.stopBlock(); restoreFut.get(getTestTimeout()); } @@ -3753,12 +3738,12 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot(); - createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 1000, 32, null); + createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 100, 32, null); snpMgr.createSnapshot("testSnapshot0").get(getTestTimeout()); if (incremental) { try (IgniteDataStreamer streamer = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { - for (int i = 1000; i < 2000; i++) + for (int i = 100; i < 110; i++) streamer.addData(i, i); } @@ -3766,12 +3751,11 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment } if (twoSnapshots) { - createCacheAndPreload(grid(1), "cache2", 1000, 32, null); snpMgr.createSnapshot("testSnapshot1").get(getTestTimeout()); if (incremental) { - try (IgniteDataStreamer streamer = grid(0).dataStreamer("cache2")) { - for (int i = 1000; i < 2000; i++) + try (IgniteDataStreamer streamer = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 110; i < 120; i++) streamer.addData(i, i); } @@ -3779,30 +3763,26 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment } } - TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); - TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); + TestRecordingCommunicationSpi cm = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); - F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> - msg instanceof SingleNodeMessage sm && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()))); + cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm && + (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal())); - var checkFut0 = runAsync(() -> incremental + IgniteInternalFuture checkFut0 = runAsync(() -> incremental ? execute("--snapshot", "check", "testSnapshot0", "--increment", "1") : execute("--snapshot", "check", "testSnapshot0")); - var checkFut1 = twoSnapshots + IgniteInternalFuture checkFut1 = twoSnapshots ? runAsync(() -> incremental ? execute("--snapshot", "check", "testSnapshot1", "--increment", "1") : execute("--snapshot", "check", "testSnapshot1")) : null; - // Waiting for the nodes each to send snapshot check single result. - for (var cm : F.asList(cm1, cm2)) { - assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m -> - m.ioMessage().message() instanceof SingleNodeMessage sm - && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoSnapshots ? 2 : 1), - getTestTimeout() - )); - } + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoSnapshots ? 2 : 1), + getTestTimeout() + )); injectTestSystemOut(); @@ -3813,11 +3793,13 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment if (log.isInfoEnabled()) log.info("Test out:" + U.nl() + out); - String chkSnpE = "Check snapshot operation is in progress"; - var chkSnpESum = Arrays.stream(out.split(U.nl())) - .mapToInt(l -> (l.length() - l.replace(chkSnpE, "").length()) / chkSnpE.length()) + String checkSnpErr = "Check snapshot operation is in progress"; + + int errSum = Arrays.stream(out.split(U.nl())) + .mapToInt(l -> (l.length() - l.replace(checkSnpErr, "").length()) / checkSnpErr.length()) .sum(); - assertEquals(twoSnapshots ? 2 : 1, chkSnpESum); + + assertEquals(twoSnapshots ? 2 : 1, errSum); assertTrue(out.contains("Snapshot name: testSnapshot0")); @@ -3837,7 +3819,7 @@ private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean increment } } - F.asList(cm1, cm2).forEach(TestRecordingCommunicationSpi::stopBlock); + cm.stopBlock(); checkFut0.get(); @@ -3863,72 +3845,67 @@ private void doTestSnapshotsChecksStatusWithParallelCreate(boolean twoChecks) th IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot(); - createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 1000, 32, null); + createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 100, 32, null); snpMgr.createSnapshot("testSnapshot0").get(getTestTimeout()); if (twoChecks) { - createCacheAndPreload(grid(1), "cache2", 1000, 32, null); + createCacheAndPreload(grid(1), "cache2", 100, 32, null); snpMgr.createSnapshot("testSnapshot1").get(getTestTimeout()); } - createCacheAndPreload(grid(1), "cache3", 1000, 32, null); + createCacheAndPreload(grid(1), "cache3", 100, 32, null); - TestRecordingCommunicationSpi cm1 = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); - TestRecordingCommunicationSpi cm2 = ((TestRecordingCommunicationSpi)grid(2).configuration().getCommunicationSpi()); + TestRecordingCommunicationSpi cm = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi()); - F.asList(cm1, cm2).forEach(cm -> cm.blockMessages((node, msg) -> + cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage sm && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal() - || sm.type() == END_SNAPSHOT.ordinal())) - ); + || sm.type() == END_SNAPSHOT.ordinal())); - var createFut = snpMgr.createSnapshot("testSnapshot2"); + IgniteFuture createFut = snpMgr.createSnapshot("testSnapshot2"); - var checkFut0 = runAsync(() -> execute("--snapshot", "check", "testSnapshot0")); - var checkFut1 = twoChecks ? runAsync(() -> execute("--snapshot", "check", "testSnapshot1")) : null; + IgniteInternalFuture checkFut0 = runAsync(() -> execute("--snapshot", "check", "testSnapshot0")); + IgniteInternalFuture checkFut1 = twoChecks ? runAsync(() -> execute("--snapshot", "check", "testSnapshot1")) : null; - // Waiting for the nodes each to send snapshot create single result. - for (var cm : F.asList(cm1, cm2)) { - assertTrue(waitForCondition(() -> cm.blockedMessages().stream().anyMatch(m -> - m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == END_SNAPSHOT.ordinal()), - getTestTimeout() - )); - } + // Wait for the creation gets blocked. + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().anyMatch(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm && sm.type() == END_SNAPSHOT.ordinal()), + getTestTimeout() + )); - // Waiting for the nodes each to send snapshot check single result. - for (var cm : F.asList(cm1, cm2)) { - assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m -> - m.ioMessage().message() instanceof SingleNodeMessage sm - && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoChecks ? 2 : 1), - getTestTimeout() - )); - } + // Wait for the checks get blocked. + assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m -> + m.ioMessage().message() instanceof SingleNodeMessage sm + && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoChecks ? 2 : 1), + getTestTimeout() + )); injectTestSystemOut(); assertEquals("Unexpected exit code", EXIT_CODE_OK, execute("--snapshot", "status")); - var out = testOut.toString(); - - if (log.isInfoEnabled()) - log.info("Test out:" + U.nl() + out); + String out = testOut.toString(); assertTrue(out.contains("Create snapshot operation is in progress")); assertTrue(out.contains("Snapshot name: testSnapshot2")); - String chkLogE = "Check snapshot operation is in progress"; - var chkLogSum = Arrays.stream(out.split(U.nl())) - .mapToInt(l -> (l.length() - l.replace(chkLogE, "").length()) / chkLogE.length()) + String checkSnpLogErr = "Check snapshot operation is in progress"; + + int chkLogSum = Arrays.stream(out.split(U.nl())) + .mapToInt(l -> (l.length() - l.replace(checkSnpLogErr, "").length()) / checkSnpLogErr.length()) .sum(); + assertEquals(twoChecks ? 2 : 1, chkLogSum); assertTrue(out.contains("Snapshot name: testSnapshot0")); + if (twoChecks) assertTrue(out.contains("Snapshot name: testSnapshot1")); - F.asList(cm1, cm2).forEach(TestRecordingCommunicationSpi::stopBlock); + cm.stopBlock(); createFut.get(); checkFut0.get(); + if (twoChecks) checkFut1.get(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java index bc2a2e3b916f7..08f6ccdc165fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusCommand.java @@ -71,7 +71,8 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand statuses = rawResult instanceof Collection ? (Collection)rawResult : null; @@ -83,7 +84,9 @@ public class SnapshotStatusCommand extends AbstractSnapshotCommand titles; /** */ - SnapshotTaskProgressDesc(List titles) { + private SnapshotTaskProgressDesc(List titles) { this.titles = Collections.unmodifiableList(titles); } /** @return Progress table columns titles. */ - List titles() { + private List titles() { return titles; } /** @return Progress table columns types. */ - List types() { + private List types() { List types = new ArrayList<>(); types.add(STRING); @@ -165,13 +168,13 @@ List types() { } /** @return Progress table data row. */ - abstract List buildRow(UUID nodeId, T5 progress); + protected abstract List buildRow(UUID nodeId, T5 progress); } /** */ private static class CreateFullSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { /** */ - CreateFullSnapshotTaskProgressDesc() { + private CreateFullSnapshotTaskProgressDesc() { super(F.asList("Node ID", "Processed, bytes", "Total, bytes", "Percent")); } @@ -192,7 +195,7 @@ private static class CreateFullSnapshotTaskProgressDesc extends SnapshotTaskProg /** */ private static class CreateIncrementalSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { /** */ - CreateIncrementalSnapshotTaskProgressDesc() { + private CreateIncrementalSnapshotTaskProgressDesc() { super(F.asList("Node ID", "Progress")); } @@ -205,7 +208,7 @@ private static class CreateIncrementalSnapshotTaskProgressDesc extends SnapshotT /** */ private static class RestoreFullSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { /** */ - RestoreFullSnapshotTaskProgressDesc() { + private RestoreFullSnapshotTaskProgressDesc() { super(F.asList("Node ID", "Processed, partitions", "Total, partitions", "Percent")); } @@ -226,7 +229,7 @@ private static class RestoreFullSnapshotTaskProgressDesc extends SnapshotTaskPro /** */ private static class RestoreIncrementalSnapshotTaskProgressDesc extends SnapshotTaskProgressDesc { /** */ - RestoreIncrementalSnapshotTaskProgressDesc() { + private RestoreIncrementalSnapshotTaskProgressDesc() { super(F.asList( "Node ID", "Processed, partitions", diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index bc631d6159b33..cda0f85bf21ff 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -36,7 +36,6 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperationRequest; import org.apache.ignite.internal.processors.metric.impl.MetricUtils; import org.apache.ignite.internal.processors.rollingupgrade.feature.CoreFeatureRegistry; -import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeatureSet; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T5; @@ -71,8 +70,8 @@ public class SnapshotStatusTask extends VisorMultiNodeTask job(NoArg arg) { @@ -81,7 +80,8 @@ public class SnapshotStatusTask extends VisorMultiNodeTask jobNodes(VisorTaskArgument arg) { - cmdInitiatorFeatures = arg.initiatorFeatures(); + featureEnabled = ignite.context().rollingUpgrade().features().isActive(SNAPSHOT_CHECK_STATUS_FEATURE) && + arg.initiatorFeatures().contains(SNAPSHOT_CHECK_STATUS_FEATURE.id());; return nodeIds(ignite.cluster().forServers().nodes()); } @@ -97,11 +97,6 @@ public class SnapshotStatusTask extends VisorMultiNodeTask allStatuses = new ArrayList<>(results.size()); @@ -109,36 +104,19 @@ public class SnapshotStatusTask extends VisorMultiNodeTask jobMultipleStatuses = (Collection)rawResult; - - // Skip the check status for an old command initiator that doesn't know about it. - if (!initiatorSupports) { - jobMultipleStatuses = jobMultipleStatuses.stream().filter(s -> s.operation() != SnapshotOperation.CHECK).toList(); - - assert jobMultipleStatuses.isEmpty() || jobMultipleStatuses.size() == 1 : - "Snapshot creation and restoration are mutually exclusive; only one of them can run at a time."; - } - - allStatuses.addAll(jobMultipleStatuses); - } + if (rawResult instanceof SnapshotStatus singleStatus) + allStatuses.add(new SnapshotOperationStatus(singleStatus)); + else + allStatuses.addAll((Collection)rawResult); } - // Empty list notifies that this is the new result but is empty. Old empty is always null. - if (allStatuses.isEmpty()) - return initiatorSupports && featureEnabled ? (Serializable)Collections.emptyList() : null; - ArrayList taskResult = new ArrayList<>(); // First, find create or restore operation. Collection filtered = F.viewReadOnly(allStatuses, s -> s, s -> s.operation() != SnapshotOperation.CHECK); - // Found create or restore statuses. if (!filtered.isEmpty()) { // Merged create or restore. SnapshotOperationStatus merged = F.first(filtered); @@ -148,21 +126,25 @@ public class SnapshotStatusTask extends VisorMultiNodeTask s.reqId.equals(mergedRqId)); - // Merge nodes progress. + // Progress per node. Map> progress = new HashMap<>(); filtered.forEach(s -> progress.putAll(s.progress)); merged = new SnapshotOperationStatus(merged.op, merged.name, merged.incIdx, merged.reqId, merged.startTime, progress); - // If the initiator like thin client of 'control.sh' doesn't support the feature yet, we have to rewrap - // the first single result back to the old format. - if (!initiatorSupports) - return new SnapshotStatus(merged.op, merged.name, merged.incIdx, merged.reqId, merged.startTime, progress); - taskResult.add(merged); } + if (!featureEnabled) { + if (taskResult.isEmpty()) + return null; + + SnapshotOperationStatus s0 = taskResult.get(0); + + return new SnapshotStatus(s0.op, s0.name, s0.incIdx, s0.reqId, s0.startTime, s0.progress); + } + // Check operation statuses go last. filtered = F.viewReadOnly(allStatuses, s -> s, s -> s.operation() == SnapshotOperation.CHECK); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index ebe9ac1020a4d..84cf4a6d34618 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -793,7 +793,7 @@ private static final class SnapshotCheckContext { /** All the snapshot metadatas. */ @Nullable private Map> clusterMetas; - /** Common counter of total work units to process on current node, of all snapshot parts ({@link #checkedSnapshotParts}). */ + /** Common counter of total work units to process on current node. */ @GridToStringExclude private final AtomicInteger totalCounter = new AtomicInteger(-1); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java index fac5b21c9c39a..bfd507a2c4d44 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMXBeanImpl.java @@ -37,8 +37,7 @@ */ public class SnapshotMXBeanImpl implements SnapshotMXBean { /** */ - public static final String NO_CREATE_OR_RESTORE_IN_PROGRESS_MSG = - "There is no create or restore snapshot operation in progress."; + public static final String NO_CREATE_OR_RESTORE_IN_PROGRESS_MSG = "There is no create or restore snapshot operation in progress."; /** Instance of snapshot cache shared manager. */ private final IgniteSnapshotManager mgr; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java index f6ae09abc9d47..5326f1776a8cb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java @@ -109,8 +109,7 @@ public class IgniteClusterSnapshotCheckStatusRollingUpgradeTest extends Abstract .setDefaultDataRegionConfiguration( new DataRegionConfiguration() .setPersistenceEnabled(true) - .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE) - ) + .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)) ); cfg.setWorkDirectory(new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath()); @@ -120,31 +119,27 @@ public class IgniteClusterSnapshotCheckStatusRollingUpgradeTest extends Abstract /** */ @Test - public void testClientOfThePreviousVersion() throws Exception { - doTestClientOfVersion(MASTER_VER_REF, "2.18.0"); + public void testClientOfThePreviousVersionWithCreateInProgress() throws Exception { + doTestDifferentClusterAndClientVersions(MASTER_VER_REF, "2.18.0"); } /** */ @Test - public void testClientOfTheNextVersion() throws Exception { - doTestClientOfVersion("2.18.0", MASTER_VER_REF); + public void testClientOfTheNextVersionWithCreateInProgress() throws Exception { + doTestDifferentClusterAndClientVersions("2.18.0", MASTER_VER_REF); } /** */ - @Test - public void testClientOfTheNextVersionWithCreateInProgress() throws Exception { + private void doTestDifferentClusterAndClientVersions(String clusterVer, String clientVer) throws Exception { for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) - startGrid(i, "2.18.0", false); + startGrid(i, clusterVer, false); grid(0).cluster().state(ClusterState.ACTIVE); createCache(0); - // The snapshot to check. It must be created before the messages are blocked. - createSnapshot(0, "testSnapshotToCheck"); + snp(0).createSnapshot("testSnapshotToCheck").get(getTestTimeout()); - // Blocks the finalization of the create operation and the check operation to keep them in progress while - // the status is requested. for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) { spi(grid(i)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && @@ -162,13 +157,9 @@ public void testClientOfTheNextVersionWithCreateInProgress() throws Exception { Collection nodes = Collections.singleton(nodeId(0)); - Collection statuses = - snapshotStatusTaskWithClient(new VisorTaskArgument<>(nodes, new NoArg(), false)); - - // The check operation is not reported by the cluster of the previous version. - assertEquals(1, statuses.size()); + VisorTaskArgument vtArg = withCoreVersion(clientVer, () -> new VisorTaskArgument<>(nodes, new NoArg(), false)); - SnapshotStatusTask.SnapshotOperationStatus status = F.first(statuses); + SnapshotStatusTask.SnapshotStatus status = snapshotStatusTaskWithClient(vtArg); assertEquals(SnapshotOperation.CREATE, status.operation()); assertEquals(SNP_NAME, status.name()); @@ -180,29 +171,7 @@ public void testClientOfTheNextVersionWithCreateInProgress() throws Exception { } createFut.get(10_000L); - - // Ensure that the snapshot check process correctly finished too. - SnapshotPartitionsVerifyResult chkRes = chkFut.get(getTestTimeout()); - - assertTrue(F.isEmpty(chkRes.exceptions())); - assertTrue(F.isEmpty(chkRes.idleVerifyResult().exceptions())); - assertFalse(chkRes.idleVerifyResult().hasConflicts()); - } - - /** */ - private void doTestClientOfVersion(String clusterVer, String clientVer) throws Exception { - for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) - startGrid(i, clusterVer, false); - - grid(0).cluster().state(ClusterState.ACTIVE); - - createCacheAndSnapshot(0); - - Collection nodes = Collections.singleton(nodeId(0)); - - VisorTaskArgument vtArg = withCoreVersion(clientVer, () -> new VisorTaskArgument<>(nodes, new NoArg(), false)); - - validateSnapshotCheckStatusFeature(false, vtArg); + chkFut.get(10_000L); } /** */ @@ -217,7 +186,7 @@ public void testParallelRollingUpgradeInProgress() throws Exception { createCacheAndSnapshot(testNodeIx); - validateSnapshotCheckStatusFeature(false); + validateSnapshotCheckStatus(false); ru(grid(testNodeIx)).enableVersionUpgrade(); @@ -234,7 +203,7 @@ public void testParallelRollingUpgradeInProgress() throws Exception { assertTrue(spi(grid(testNodeIx)).waitForBlocked(1, getTestTimeout())); - validateSnapshotCheckStatusFeature(false); + validateSnapshotCheckStatus(false); spi(grid(testNodeIx)).stopBlock(); @@ -248,7 +217,7 @@ public void testParallelRollingUpgradeInProgress() throws Exception { assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout())); } - validateSnapshotCheckStatusFeature(true); + validateSnapshotCheckStatus(true); } /** */ @@ -261,7 +230,7 @@ public void testSnapshotStatusFeature() throws Exception { createCacheAndSnapshot(1); - validateSnapshotCheckStatusFeature(false); + validateSnapshotCheckStatus(false); ru(grid(0)).enableVersionUpgrade(); @@ -271,7 +240,7 @@ public void testSnapshotStatusFeature() throws Exception { upgradeNodeVersion(i, "2.18.0", MASTER_VER_REF); } - validateSnapshotCheckStatusFeature(false); + validateSnapshotCheckStatus(false); ru(grid(1)).finalizeClusterVersion(); @@ -281,56 +250,11 @@ public void testSnapshotStatusFeature() throws Exception { assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout())); } - validateSnapshotCheckStatusFeature(true); - } - - /** */ - private void createCacheAndSnapshot(int gridIdx) { - createCache(gridIdx); - - createSnapshot(gridIdx); - } - - /** Creates a replicated cache and fills it with data. */ - private void createCache(int gridIdx) { - int partsCnt = 4; - int keysCnt = partsCnt * 5; - - grid(gridIdx).createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) - .setCacheMode(CacheMode.REPLICATED) - .setAffinity(new RendezvousAffinityFunction().setPartitions(partsCnt)) - .setAtomicityMode(CacheAtomicityMode.ATOMIC)); - - try (IgniteDataStreamer ds = grid(gridIdx).dataStreamer(DEFAULT_CACHE_NAME)) { - for (int i = 0; i < keysCnt; i++) - ds.addData(i, i); - } - } - - /** */ - private void createSnapshot(int gridIdx) { - createSnapshot(gridIdx, SNP_NAME); + validateSnapshotCheckStatus(true); } /** */ - private void createSnapshot(int gridIdx, String snpName) { - snp(gridIdx).createSnapshot(snpName).get(getTestTimeout()); - } - - /** @return {@code True} if the create operation of the snapshot {@code snpName} is in progress on all the server nodes. */ - private boolean createInProgress(String snpName) { - for (int i = 0; i < ALL_GRIDS - CLIENTS; i++) { - SnapshotOperationRequest req = snp(i).currentCreateRequest(); - - if (req == null || !snpName.equals(req.snapshotName())) - return false; - } - - return true; - } - - /** */ - private void validateSnapshotCheckStatusFeature(boolean supports, @Nullable VisorTaskArgument clientArg) throws Exception { + private void validateSnapshotCheckStatus(boolean supports, @Nullable VisorTaskArgument clientArg) throws Exception { int blockNodeIdx = 1; spi(grid(blockNodeIdx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && @@ -365,25 +289,12 @@ private void validateSnapshotCheckStatusFeature(boolean supports, @Nullable Viso spi(grid(blockNodeIdx)).stopBlock(); - assertTrue(waitForCondition(() -> !spi(grid(blockNodeIdx)).hasBlockedMessages(), getTestTimeout())); - - // Ensure that the snapshot check process correctly finished too. - SnapshotPartitionsVerifyResult chkRes = chkFut.get(getTestTimeout()); - - assertTrue(F.isEmpty(chkRes.exceptions())); - assertFalse(F.isEmpty(chkRes.metas())); - assertTrue(F.isEmpty(chkRes.idleVerifyResult().exceptions())); - assertFalse(chkRes.idleVerifyResult().hasConflicts()); - } - - /** */ - private void validateSnapshotCheckStatusFeature(boolean supports) throws Exception { - validateSnapshotCheckStatusFeature(supports, null); + chkFut.get(10_000L); } /** */ - private IgniteSnapshotManager snp(int gridIdx) { - return grid(gridIdx).context().cache().context().snapshotMgr(); + private void validateSnapshotCheckStatus(boolean supports) throws Exception { + validateSnapshotCheckStatus(supports, null); } /** */ @@ -396,13 +307,39 @@ private static T snapshotStatusTaskWithGrid(Ignite grid } /** */ - private T snapshotStatusTaskWithClient(VisorTaskArgument vtArg) throws Exception { - String taskName = SnapshotStatusTask.class.getName(); - + private static T snapshotStatusTaskWithClient(VisorTaskArgument vtArg) throws Exception { try (IgniteClient cli = Ignition.startClient(new ClientConfiguration().setAddresses(Config.SERVER))) { - VisorTaskResult vtRes = cli.compute().execute(taskName, vtArg); + VisorTaskResult vtRes = cli.compute().execute(SnapshotStatusTask.class.getName(), vtArg); return vtRes.result(); } } + + /** */ + private IgniteSnapshotManager snp(int gridIdx) { + return grid(gridIdx).context().cache().context().snapshotMgr(); + } + + /** */ + private void createCacheAndSnapshot(int gridIdx) { + createCache(gridIdx); + + snp(gridIdx).createSnapshot(SNP_NAME).get(getTestTimeout()); + } + + /** Creates a replicated cache and fills it with data. */ + private void createCache(int gridIdx) { + int partsCnt = 4; + int keysCnt = partsCnt * 5; + + grid(gridIdx).createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setCacheMode(CacheMode.REPLICATED) + .setAffinity(new RendezvousAffinityFunction().setPartitions(partsCnt)) + .setAtomicityMode(CacheAtomicityMode.ATOMIC)); + + try (IgniteDataStreamer ds = grid(gridIdx).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 0; i < keysCnt; i++) + ds.addData(i, i); + } + } } From 3e270e0d540a0a27b2b092ff05c1d72f56ba2cc9 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 30 Oct 2026 16:00:11 +0300 Subject: [PATCH 41/44] renaming --- .../IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java index 5326f1776a8cb..870a9ac5aa04d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java @@ -176,7 +176,7 @@ private void doTestDifferentClusterAndClientVersions(String clusterVer, String c /** */ @Test - public void testParallelRollingUpgradeInProgress() throws Exception { + public void testRollingUpgradeInProgress() throws Exception { for (int i = 0; i < ALL_GRIDS; i++) startGrid(i, "2.18.0", i >= ALL_GRIDS - CLIENTS); From e3d7c2f45cb2ada35cf2f5d65b6bfb373931fa7f Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Sat, 31 Oct 2026 12:33:14 +0300 Subject: [PATCH 42/44] fixes --- docs/_docs/monitoring-metrics/new-metrics.adoc | 3 ++- docs/_docs/snapshots/snapshots.adoc | 2 +- .../cache/persistence/snapshot/IgniteSnapshotManager.java | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/_docs/monitoring-metrics/new-metrics.adoc b/docs/_docs/monitoring-metrics/new-metrics.adoc index b055dc4069544..1d63e6fcd3349 100644 --- a/docs/_docs/monitoring-metrics/new-metrics.adoc +++ b/docs/_docs/monitoring-metrics/new-metrics.adoc @@ -684,13 +684,13 @@ Register name: `snapshot-restore` [cols="2,1,3",opts="header"] |=== |Name | Type | Description +|requestId | string | The request ID of the last running cluster snapshot restore operation on this node. |endTime | long | The system time when the restore operation of a cluster snapshot on this node ended. |error | string | Error message of the last running cluster snapshot restore operation on this node. |incrementIndex | integer | The index of incremental snapshot of the last snapshot restore operation on this node. |processedPartitions | integer | The number of processed partitions on this node. |processedWalEntries | long | The number of processed entries from incremental snapshot on this node. |processedWalSegments | integer | The number of processed WAL segments in the incremental snapshot on this node. -|requestId | string | The request ID of the last running cluster snapshot restore operation on this node. |snapshotName | string | The snapshot name of the last running cluster snapshot restore operation on this node. |startTime | long | The system time of the start of the cluster snapshot restore operation on this node. |totalPartitions | integer | The total number of partitions to be restored on this node. @@ -706,6 +706,7 @@ Register name: `snapshot-check.{snapshot_name}` [cols="2,1,3",opts="header"] |=== |Name | Type | Description +|requestId | string | The request ID of the last running cluster snapshot check operation on this node. |checkPartitions | boolean | Shows whether full validation of snapshot partitions is enabled. |incrementIndex | integer | The index of incremental snapshot of the snapshot check operation. |processedPartitions | integer | The number of checked partitions on current node. diff --git a/docs/_docs/snapshots/snapshots.adoc b/docs/_docs/snapshots/snapshots.adoc index 0f9378e4cfd61..ccb6f664443a3 100644 --- a/docs/_docs/snapshots/snapshots.adoc +++ b/docs/_docs/snapshots/snapshots.adoc @@ -371,7 +371,7 @@ If the metadata isn't found or cannot be read, snapshot isn't deleted. == Getting Snapshot Operation Status -The status of the create/restore/check snapshot operations in the cluster can be obtained using the `control.sh|bat` script. +The status of the create/restore snapshot operations in the cluster can be obtained using the `control.sh|bat` script. [tabs] -- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 11afe3182fb56..cd9bb196d7497 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -1731,6 +1731,8 @@ private IgniteInternalFuture cancelSnapshot0(String name) { } /** + * TODO: support cancellation of the check operation https://issues.apache.org/jira/browse/IGNITE-29240 + * * @param reqId Snapshot operation request ID. * @return Future which will be completed when cancel operation finished. */ @@ -4477,6 +4479,7 @@ public CancelSnapshotCallable(UUID reqId, String snpName) { /** {@inheritDoc} */ @Override public Boolean call() { + // TODO: support cancellation of the check operation https://issues.apache.org/jira/browse/IGNITE-29240 if (reqId != null) return ignite.context().cache().context().snapshotMgr().cancelLocalSnapshotOperations(reqId); else { From 6610d8b966c97ee5fdec8ed5444c618b5662a6a7 Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Sat, 31 Oct 2026 13:04:00 +0300 Subject: [PATCH 43/44] fixes, same rqId --- .../_docs/monitoring-metrics/new-metrics.adoc | 1 + .../snapshot/SnapshotStatusTask.java | 8 ++++---- .../snapshot/IgniteSnapshotManager.java | 20 ++++++++++++++++--- .../snapshot/SnapshotCheckProcess.java | 10 ++++++---- .../snapshot/SnapshotRestoreProcess.java | 2 +- 5 files changed, 29 insertions(+), 12 deletions(-) diff --git a/docs/_docs/monitoring-metrics/new-metrics.adoc b/docs/_docs/monitoring-metrics/new-metrics.adoc index 1d63e6fcd3349..97ad14458aa8c 100644 --- a/docs/_docs/monitoring-metrics/new-metrics.adoc +++ b/docs/_docs/monitoring-metrics/new-metrics.adoc @@ -707,6 +707,7 @@ Register name: `snapshot-check.{snapshot_name}` |=== |Name | Type | Description |requestId | string | The request ID of the last running cluster snapshot check operation on this node. +If snapshot is checked as part oth the restoration operation, it has the same request ID. |checkPartitions | boolean | Shows whether full validation of snapshot partitions is enabled. |incrementIndex | integer | The index of incremental snapshot of the snapshot check operation. |processedPartitions | integer | The number of checked partitions on current node. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java index cda0f85bf21ff..afec07774361b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotStatusTask.java @@ -339,10 +339,10 @@ public static class SnapshotStatus implements Serializable { private final long startTime; /** Progress of operation on nodes. */ - Map> progress; + private final Map> progress; /** */ - private SnapshotStatus( + public SnapshotStatus( SnapshotOperation op, String name, int incIdx, @@ -355,7 +355,7 @@ private SnapshotStatus( this.incIdx = incIdx; this.requestId = requestId; this.startTime = startTime; - this.progress = progress; + this.progress = Collections.unmodifiableMap(progress); } /** @return Operation type. */ @@ -385,7 +385,7 @@ public long startTime() { /** @return Progress of operation on nodes. */ public Map> progress() { - return Collections.unmodifiableMap(progress); + return progress; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index cd9bb196d7497..46a8a28262cbd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -1878,6 +1878,18 @@ public IgniteInternalFuture checkSnapshot(String }); } + /** {@link #checkSnapshot(String, String, Collection, boolean, int, boolean, UUID)} with default {@code reqId} == {@code null}. */ + public IgniteInternalFuture checkSnapshot( + String name, + @Nullable String snpPath, + @Nullable Collection grps, + boolean includeCustomHandlers, + int incIdx, + boolean check + ) { + return checkSnapshot(name, snpPath, grps, includeCustomHandlers, incIdx, check, null); + } + /** * The check snapshot procedure performs compute operation over the whole cluster to verify the snapshot * entirety and partitions consistency. The result future will be completed with an exception if this @@ -1890,16 +1902,18 @@ public IgniteInternalFuture checkSnapshot(String * handlers, otherwise only system consistency check will be performed. * @param incIdx Incremental snapshot index. * @param check If {@code true} check snapshot integrity. + * @param reqId Optional request ID to use. If {@code null}, own random one is generated. * @return Future with the result of execution snapshot partitions verify task, which besides calculating partition * hashes of {@link IdleVerifyResult} also contains the snapshot metadata distribution across the cluster. */ - public IgniteInternalFuture checkSnapshot( + IgniteInternalFuture checkSnapshot( String name, @Nullable String snpPath, @Nullable Collection grps, boolean includeCustomHandlers, int incIdx, - boolean check + boolean check, + @Nullable UUID reqId ) { A.notNullOrEmpty(name, "Snapshot name cannot be null or empty."); A.ensure(U.alphanumericUnderscore(name), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_"); @@ -1913,7 +1927,7 @@ public IgniteInternalFuture checkSnapshot( } IgniteInternalFuture res = checkSnpProc.start( - name, snpPath, grps, check, incIdx, includeCustomHandlers); + name, snpPath, grps, check, incIdx, includeCustomHandlers, reqId); res.listen(lsnr -> { if (log.isInfoEnabled()) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index 84cf4a6d34618..aabbdd226e23f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -661,6 +661,7 @@ private List assingMetas(Map start( String snpName, @@ -668,16 +669,17 @@ public IgniteInternalFuture start( @Nullable Collection grpNames, boolean fullCheck, int incIdx, - boolean allRestoreHandlers + boolean allRestoreHandlers, + @Nullable UUID reqId ) { assert !F.isEmpty(snpName); - UUID reqId = UUID.randomUUID(); + UUID reqId0 = reqId != null ? reqId : UUID.randomUUID(); Set requiredNodes = new HashSet<>(F.viewReadOnly(kctx.discovery().discoCache().aliveBaselineNodes(), node2id())); SnapshotCheckProcessRequest req = new SnapshotCheckProcessRequest( - reqId, + reqId0, requiredNodes, snpName, snpPath, @@ -689,7 +691,7 @@ public IgniteInternalFuture start( GridFutureAdapter clusterOpFut = new GridFutureAdapter<>(); - clusterOpFut.listen(fut -> clusterOpFuts.remove(reqId)); + clusterOpFut.listen(fut -> clusterOpFuts.remove(reqId0)); clusterOpFuts.put(reqId, clusterOpFut); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 2643a5575f6d4..31f0abc9c9fad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -322,7 +322,7 @@ public IgniteFutureImpl start( snpMgr.recordSnapshotEvent(snpName, msg, EventType.EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED); - snpMgr.checkSnapshot(snpName, snpPath, cacheGrpNames, incIdx < 1, incIdx, check).listen(f -> { + snpMgr.checkSnapshot(snpName, snpPath, cacheGrpNames, incIdx < 1, incIdx, check, fut.rqId).listen(f -> { if (f.error() != null) { finishProcess(fut0.rqId, f.error()); From 4504ec0e2d99674dbb48ef2e0d5994617fe0340c Mon Sep 17 00:00:00 2001 From: Steshin Vladimir Date: Sat, 31 Oct 2026 19:03:19 +0300 Subject: [PATCH 44/44] fixes in progress --- .../_docs/monitoring-metrics/new-metrics.adoc | 2 +- .../ignite/util/GridCommandHandlerTest.java | 10 +-- ...SnapshotCheckStatusRollingUpgradeTest.java | 76 ++----------------- 3 files changed, 10 insertions(+), 78 deletions(-) diff --git a/docs/_docs/monitoring-metrics/new-metrics.adoc b/docs/_docs/monitoring-metrics/new-metrics.adoc index 97ad14458aa8c..acefac9d4fc4d 100644 --- a/docs/_docs/monitoring-metrics/new-metrics.adoc +++ b/docs/_docs/monitoring-metrics/new-metrics.adoc @@ -707,7 +707,7 @@ Register name: `snapshot-check.{snapshot_name}` |=== |Name | Type | Description |requestId | string | The request ID of the last running cluster snapshot check operation on this node. -If snapshot is checked as part oth the restoration operation, it has the same request ID. +If snapshot is checked as part of the restoration operation, it has the same request ID. |checkPartitions | boolean | Shows whether full validation of snapshot partitions is enabled. |incrementIndex | integer | The index of incremental snapshot of the snapshot check operation. |processedPartitions | integer | The number of checked partitions on current node. diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index b06cd399e8595..199e714ff1336 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3662,11 +3662,9 @@ public void testSnapshotRestoreCancelAndStatus() throws Exception { assertNull(ig.cache(DEFAULT_CACHE_NAME)); } - /** Tests that snapshot metrics aren't empty when being restored snapshot waits for the check process. */ + /** Tests that snapshot status isn't empty when being restored snapshot waits for the check process. */ @Test - public void testRestoreSnapshotMetricsAtStart() throws Exception { - listeningLog = new ListeningTestLogger(log); - + public void testRestoreSnapshotStatusAtStart() throws Exception { startGrids(3).cluster().state(ClusterState.ACTIVE); createCacheAndPreload(grid(1), 100); @@ -3890,11 +3888,11 @@ private void doTestSnapshotsChecksStatusWithParallelCreate(boolean twoChecks) th String checkSnpLogErr = "Check snapshot operation is in progress"; - int chkLogSum = Arrays.stream(out.split(U.nl())) + int entrySum = Arrays.stream(out.split(U.nl())) .mapToInt(l -> (l.length() - l.replace(checkSnpLogErr, "").length()) / checkSnpLogErr.length()) .sum(); - assertEquals(twoChecks ? 2 : 1, chkLogSum); + assertEquals(twoChecks ? 2 : 1, entrySum); assertTrue(out.contains("Snapshot name: testSnapshot0")); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java index 870a9ac5aa04d..2a39a83069050 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckStatusRollingUpgradeTest.java @@ -52,7 +52,6 @@ import org.apache.ignite.internal.visor.VisorTaskArgument; import org.apache.ignite.internal.visor.VisorTaskResult; import org.apache.ignite.lang.IgniteFuture; -import org.jetbrains.annotations.Nullable; import org.junit.Test; import static java.nio.file.Files.newDirectoryStream; @@ -60,8 +59,6 @@ import static org.apache.ignite.internal.processors.rollingupgrade.feature.ManagementApiVersionValidationTest.withCoreVersion; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_METAS; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.END_SNAPSHOT; -import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RU_PREPARE_VERSION_FINALIZATION; -import static org.apache.ignite.testframework.GridTestUtils.runAsync; import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; /** */ @@ -174,52 +171,6 @@ private void doTestDifferentClusterAndClientVersions(String clusterVer, String c chkFut.get(10_000L); } - /** */ - @Test - public void testRollingUpgradeInProgress() throws Exception { - for (int i = 0; i < ALL_GRIDS; i++) - startGrid(i, "2.18.0", i >= ALL_GRIDS - CLIENTS); - - grid(0).cluster().active(true); - - int testNodeIx = ALL_GRIDS - CLIENTS - 1; - - createCacheAndSnapshot(testNodeIx); - - validateSnapshotCheckStatus(false); - - ru(grid(testNodeIx)).enableVersionUpgrade(); - - for (int i = 0; i < ALL_GRIDS; i++) { - assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); - - upgradeNodeVersion(i, "2.18.0", MASTER_VER_REF); - } - - spi(grid(testNodeIx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && - snm.type() == RU_PREPARE_VERSION_FINALIZATION.ordinal()); - - IgniteInternalFuture finalizeFut = runAsync(() -> ru(testNodeIx).finalizeClusterVersion()); - - assertTrue(spi(grid(testNodeIx)).waitForBlocked(1, getTestTimeout())); - - validateSnapshotCheckStatus(false); - - spi(grid(testNodeIx)).stopBlock(); - - assertFalse(spi(grid(testNodeIx)).hasBlockedMessages()); - - finalizeFut.get(getTestTimeout()); - - for (int i = 0; i < ALL_GRIDS; i++) { - int i0 = i; - - assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout())); - } - - validateSnapshotCheckStatus(true); - } - /** */ @Test public void testSnapshotStatusFeature() throws Exception { @@ -230,10 +181,10 @@ public void testSnapshotStatusFeature() throws Exception { createCacheAndSnapshot(1); - validateSnapshotCheckStatus(false); - ru(grid(0)).enableVersionUpgrade(); + validateSnapshotCheckStatus(false); + for (int i = 0; i < ALL_GRIDS; i++) { assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); @@ -254,7 +205,7 @@ public void testSnapshotStatusFeature() throws Exception { } /** */ - private void validateSnapshotCheckStatus(boolean supports, @Nullable VisorTaskArgument clientArg) throws Exception { + private void validateSnapshotCheckStatus(boolean supports) throws Exception { int blockNodeIdx = 1; spi(grid(blockNodeIdx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && @@ -264,8 +215,8 @@ private void validateSnapshotCheckStatus(boolean supports, @Nullable VisorTaskAr assertTrue(waitForCondition(() -> spi(grid(blockNodeIdx)).hasBlockedMessages(), getTestTimeout())); - if (clientArg != null) { - Serializable rawRes = snapshotStatusTaskWithClient(clientArg); + for (Ignite ig : G.allGrids()) { + Serializable rawRes = snapshotStatusTaskWithGrid(ig); if (supports) { assertTrue(rawRes instanceof Collection cll && !cll.isEmpty() && @@ -274,29 +225,12 @@ private void validateSnapshotCheckStatus(boolean supports, @Nullable VisorTaskAr else assertTrue(rawRes == null); } - else { - for (Ignite ig : G.allGrids()) { - Serializable rawRes = snapshotStatusTaskWithGrid(ig); - - if (supports) { - assertTrue(rawRes instanceof Collection cll && !cll.isEmpty() && - F.first(cll) instanceof SnapshotStatusTask.SnapshotOperationStatus); - } - else - assertTrue(rawRes == null); - } - } spi(grid(blockNodeIdx)).stopBlock(); chkFut.get(10_000L); } - /** */ - private void validateSnapshotCheckStatus(boolean supports) throws Exception { - validateSnapshotCheckStatus(supports, null); - } - /** */ private static T snapshotStatusTaskWithGrid(Ignite grid) throws Exception { Collection nodes = grid.cluster().nodes().stream().map(ClusterNode::id).toList();