IGNITE-19034 VisorSnapshotStatusTask returns empty result for restoring snapshot until snapshot check is completed

docs/_docs/monitoring-metrics/new-metrics.adoc

@@ -684,13 +684,13 @@ Register name: `snapshot-restore`
 [cols="2,1,3",opts="header"]
 |===
 |Name | Type | Description
+|requestId | string | The request ID of the last running cluster snapshot restore operation on this node.
 |endTime | long | The system time when the restore operation of a cluster snapshot on this node ended.
 |error | string | Error message of the last running cluster snapshot restore operation on this node.
 |incrementIndex | integer | The index of incremental snapshot of the last snapshot restore operation on this node.
 |processedPartitions | integer | The number of processed partitions on this node.
 |processedWalEntries | long | The number of processed entries from incremental snapshot on this node.
 |processedWalSegments | integer | The number of processed WAL segments in the incremental snapshot on this node.
-|requestId | string | The request ID of the last running cluster snapshot restore operation on this node.
 |snapshotName | string | The snapshot name of the last running cluster snapshot restore operation on this node.
 |startTime | long | The system time of the start of the cluster snapshot restore operation on this node.
 |totalPartitions | integer | The total number of partitions to be restored on this node.
@@ -706,6 +706,8 @@ Register name: `snapshot-check.{snapshot_name}`
 [cols="2,1,3",opts="header"]
 |===
 |Name | Type | Description
+|requestId | string | The request ID of the last running cluster snapshot check operation on this node.
+If snapshot is checked as part of the restoration operation, it has the same request ID.
 |checkPartitions | boolean | Shows whether full validation of snapshot partitions is enabled.
 |incrementIndex | integer | The index of incremental snapshot of the snapshot check operation.
 |processedPartitions | integer | The number of checked partitions on current node.

docs/_docs/snapshots/snapshots.adoc

@@ -371,35 +371,33 @@ If the metadata isn't found or cannot be read, snapshot isn't deleted.
 
 == Getting Snapshot Operation Status
 
-The status of the create/restore snapshot operations in the cluster can be obtained using the `control.sh|bat` script or JMX interface:
+The status of the create/restore snapshot operations in the cluster can be obtained using the `control.sh|bat` script.
 
 [tabs]
 --
 tab:Unix[]
 [source,shell]
 ----
-# Get the status of the snapshot operation.
+# Get the status of snapshot operation.
 control.sh --snapshot status
 ----
 
 tab:Windows[]
 [source,shell]
 ----
-# Get the status of the snapshot operation.
+# Get the status of snapshot operation.
 control.bat --snapshot status
 ----
-
-tab:JMX[]
-You can also get the current snapshot status via the `SnapshotMXBean` interface:
-[source,java]
-----
-SnapshotMXBean mxBean = ...;
-
-// The status of a current snapshot operation in the cluster.
-String status = mxBean.status();
-----
 --
 
+[NOTE]
+====
+Multiple snapshot check operations can run concurrently with any other snapshot operation of another snapshot.
+Only one check can be running for the same snapshot at a time. Create and restore operations cannot run in parallel —
+only one of them can be active at a time. The status is printed for each running snapshot operation (if multiple) separately,
+one after another.
+====
+
 == Cancelling Snapshot Operation
 
 To abort create/restore snapshot operation you need to obtain an `operation request ID`.

modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java

@@ -85,6 +85,7 @@
 import org.apache.ignite.internal.management.cache.FindAndDeleteGarbageInPersistenceTaskResult;
 import org.apache.ignite.internal.management.cache.IdleVerifyDumpTask;
 import org.apache.ignite.internal.management.cache.VerifyBackupPartitionsTask;
+import org.apache.ignite.internal.management.snapshot.SnapshotStatusCommand;
 import org.apache.ignite.internal.management.tx.TxInfo;
 import org.apache.ignite.internal.management.tx.TxTaskResult;
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
@@ -121,6 +122,7 @@
 import org.apache.ignite.internal.util.distributed.DistributedProcess;
 import org.apache.ignite.internal.util.distributed.SingleNodeMessage;
 import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
+import org.apache.ignite.internal.util.future.IgniteFutureImpl;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.lang.GridFunc;
 import org.apache.ignite.internal.util.typedef.F;
@@ -182,6 +184,8 @@
 import static org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor.DEFAULT_TARGET_FOLDER;
 import static org.apache.ignite.internal.processors.job.GridJobProcessor.JOBS_VIEW;
 import static org.apache.ignite.internal.processors.task.GridTaskProcessor.TASKS_VIEW;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_PARTS;
+import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.END_SNAPSHOT;
 import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE;
 import static org.apache.ignite.testframework.GridTestUtils.assertContains;
 import static org.apache.ignite.testframework.GridTestUtils.assertNotContains;
@@ -3653,11 +3657,257 @@ public void testSnapshotRestoreCancelAndStatus() throws Exception {
         assertTrue(ctxDisposed);
 
         assertEquals(EXIT_CODE_OK, execute("--snapshot", "status"));
-        assertContains(log, testOut.toString(), "There is no create or restore snapshot operation in progress.");
+        assertContains(log, testOut.toString(), SnapshotStatusCommand.NO_SNP_OP_IN_PROGRESS_MSG);
 
         assertNull(ig.cache(DEFAULT_CACHE_NAME));
     }
 
+    /** Tests that snapshot status isn't empty when being restored snapshot waits for the check process. */
+    @Test
+    public void testRestoreSnapshotStatusAtStart() throws Exception {
+        startGrids(3).cluster().state(ClusterState.ACTIVE);
+
+        createCacheAndPreload(grid(1), 100);
+
+        IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot();
+
+        snpMgr.createSnapshot("test_snapshot").get(getTestTimeout());
+
+        grid(0).destroyCache(DEFAULT_CACHE_NAME);
+
+        awaitPartitionMapExchange();
+
+        TestRecordingCommunicationSpi cm = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi());
+
+        cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage<?> sm
+            && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()));
+
+        IgniteFutureImpl<Void> restoreFut = snpMgr.restoreSnapshot("test_snapshot", null, null, 0, true);
+
+        assertTrue(waitForCondition(cm::hasBlockedMessages, getTestTimeout()));
+
+        injectTestSystemOut();
+
+        int code = execute("--snapshot", "status");
+
+        assertEquals("Unexpected exit code", EXIT_CODE_OK, code);
+
+        String out = testOut.toString();
+
+        assertContains(log, out, "Check snapshot operation is in progress");
+        assertContains(log, out, "Snapshot name: test_snapshot");
+
+        assertFalse("Snapshot future has finished", restoreFut.isDone());
+
+        cm.stopBlock();
+
+        restoreFut.get(getTestTimeout());
+    }
+
+    /** */
+    @Test
+    public void testOneSnapshotCheckStatus() throws Exception {
+        doTestSnapshotsChecksStatus(false, false);
+    }
+
+    /** */
+    @Test
+    public void testOneIncrementalSnapshotCheckStatus() throws Exception {
+        doTestSnapshotsChecksStatus(false, true);
+    }
+
+    /** */
+    @Test
+    public void testTwoSnapshotsChecksStatus() throws Exception {
+        doTestSnapshotsChecksStatus(true, false);
+    }
+
+    /** */
+    @Test
+    public void testTwoIncrementalsSnapshotsChecksStatus() throws Exception {
+        doTestSnapshotsChecksStatus(true, true);
+    }
+
+    /** */
+    private void doTestSnapshotsChecksStatus(boolean twoSnapshots, boolean incremental) throws Exception {
+        walCompactionEnabled(incremental);
+
+        startGrids(3).cluster().state(ClusterState.ACTIVE);
+
+        IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot();
+
+        createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 100, 32, null);
+        snpMgr.createSnapshot("testSnapshot0").get(getTestTimeout());
+
+        if (incremental) {
+            try (IgniteDataStreamer<Object, Object> streamer = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) {
+                for (int i = 100; i < 110; i++)
+                    streamer.addData(i, i);
+            }
+
+            snpMgr.createIncrementalSnapshot("testSnapshot0").get(getTestTimeout());
+        }
+
+        if (twoSnapshots) {
+            snpMgr.createSnapshot("testSnapshot1").get(getTestTimeout());
+
+            if (incremental) {
+                try (IgniteDataStreamer<Object, Object> streamer = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) {
+                    for (int i = 110; i < 120; i++)
+                        streamer.addData(i, i);
+                }
+
+                snpMgr.createIncrementalSnapshot("testSnapshot1").get(getTestTimeout());
+            }
+        }
+
+        TestRecordingCommunicationSpi cm = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi());
+
+        cm.blockMessages((node, msg) -> msg instanceof SingleNodeMessage<?> sm &&
+            (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()));
+
+        IgniteInternalFuture<?> checkFut0 = runAsync(() -> incremental
+            ? execute("--snapshot", "check", "testSnapshot0", "--increment", "1")
+            : execute("--snapshot", "check", "testSnapshot0"));
+
+        IgniteInternalFuture<?> checkFut1 = twoSnapshots
+            ? runAsync(() -> incremental
+                             ? execute("--snapshot", "check", "testSnapshot1", "--increment", "1")
+                             : execute("--snapshot", "check", "testSnapshot1"))
+            : null;
+
+        assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m ->
+                m.ioMessage().message() instanceof SingleNodeMessage<?> sm
+                    && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoSnapshots ? 2 : 1),
+            getTestTimeout()
+        ));
+
+        injectTestSystemOut();
+
+        assertEquals("Unexpected exit code", EXIT_CODE_OK, execute("--snapshot", "status"));
+
+        String out = testOut.toString();
+
+        if (log.isInfoEnabled())
+            log.info("Test out:" + U.nl() + out);
+
+        String checkSnpErr = "Check snapshot operation is in progress";
+
+        int errSum = Arrays.stream(out.split(U.nl()))
+            .mapToInt(l -> (l.length() - l.replace(checkSnpErr, "").length()) / checkSnpErr.length())
+            .sum();
+
+        assertEquals(twoSnapshots ? 2 : 1, errSum);
+
+        assertTrue(out.contains("Snapshot name: testSnapshot0"));
+
+        if (incremental)
+            assertTrue(out.contains("Increment index: 1"));
+
+        if (twoSnapshots) {
+            assertTrue(out.contains("Snapshot name: testSnapshot1"));
+
+            if (incremental) {
+                // Number of 'Increment index: 1' entries.
+                var sum = Arrays.stream(out.split(U.nl()))
+                    .mapToInt(l -> (l.length() - l.replace("Increment index: 1", "").length()) / "Increment index: 1".length())
+                    .sum();
+
+                assertEquals(2, sum);
+            }
+        }
+
+        cm.stopBlock();
+
+        checkFut0.get();
+
+        if (twoSnapshots)
+            checkFut1.get();
+    }
+
+    /** */
+    @Test
+    public void testSnapshotCheckStatusWithParallelCreate() throws Exception {
+        doTestSnapshotsChecksStatusWithParallelCreate(false);
+    }
+
+    /** */
+    @Test
+    public void testTwoSnapshotsChecksStatusWithParallelCreate() throws Exception {
+        doTestSnapshotsChecksStatusWithParallelCreate(true);
+    }
+
+    /** */
+    private void doTestSnapshotsChecksStatusWithParallelCreate(boolean twoChecks) throws Exception {
+        startGrids(3).cluster().state(ClusterState.ACTIVE);
+
+        IgniteSnapshotManager snpMgr = (IgniteSnapshotManager)grid(0).snapshot();
+
+        createCacheAndPreload(grid(1), DEFAULT_CACHE_NAME, 100, 32, null);
+        snpMgr.createSnapshot("testSnapshot0").get(getTestTimeout());
+
+        if (twoChecks) {
+            createCacheAndPreload(grid(1), "cache2", 100, 32, null);
+            snpMgr.createSnapshot("testSnapshot1").get(getTestTimeout());
+        }
+
+        createCacheAndPreload(grid(1), "cache3", 100, 32, null);
+
+        TestRecordingCommunicationSpi cm = ((TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi());
+
+        cm.blockMessages((node, msg) ->
+            msg instanceof SingleNodeMessage<?> sm && (sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()
+                || sm.type() == END_SNAPSHOT.ordinal()));
+
+        IgniteFuture<?> createFut = snpMgr.createSnapshot("testSnapshot2");
+
+        IgniteInternalFuture<?> checkFut0 = runAsync(() -> execute("--snapshot", "check", "testSnapshot0"));
+        IgniteInternalFuture<?> checkFut1 = twoChecks ? runAsync(() -> execute("--snapshot", "check", "testSnapshot1")) : null;
+
+        // Wait for the creation gets blocked.
+        assertTrue(waitForCondition(() -> cm.blockedMessages().stream().anyMatch(m ->
+                m.ioMessage().message() instanceof SingleNodeMessage<?> sm && sm.type() == END_SNAPSHOT.ordinal()),
+            getTestTimeout()
+        ));
+
+        // Wait for the checks get blocked.
+        assertTrue(waitForCondition(() -> cm.blockedMessages().stream().filter(m ->
+                m.ioMessage().message() instanceof SingleNodeMessage<?> sm
+                    && sm.type() == CHECK_SNAPSHOT_PARTS.ordinal()).count() == (twoChecks ? 2 : 1),
+            getTestTimeout()
+        ));
+
+        injectTestSystemOut();
+
+        assertEquals("Unexpected exit code", EXIT_CODE_OK, execute("--snapshot", "status"));
+
+        String out = testOut.toString();
+
+        assertTrue(out.contains("Create snapshot operation is in progress"));
+        assertTrue(out.contains("Snapshot name: testSnapshot2"));
+
+        String checkSnpLogErr = "Check snapshot operation is in progress";
+
+        int entrySum = Arrays.stream(out.split(U.nl()))
+                    .mapToInt(l -> (l.length() - l.replace(checkSnpLogErr, "").length()) / checkSnpLogErr.length())
+                    .sum();
+
+        assertEquals(twoChecks ? 2 : 1, entrySum);
+
+        assertTrue(out.contains("Snapshot name: testSnapshot0"));
+
+        if (twoChecks)
+            assertTrue(out.contains("Snapshot name: testSnapshot1"));
+
+        cm.stopBlock();
+
+        createFut.get();
+        checkFut0.get();
+
+        if (twoChecks)
+            checkFut1.get();
+    }
+
     /** @throws Exception If fails. */
     @Test
     public void testSnapshotStatusInMemory() throws Exception {
@@ -3785,7 +4035,7 @@ private void checkSnapshotStatus(boolean isCreating, boolean isRestoring, boolea
         assertEquals(out, EXIT_CODE_OK, status);
 
         if (!isCreating && !isRestoring) {
-            assertContains(log, out, "There is no create or restore snapshot operation in progress.");
+            assertContains(log, out, SnapshotStatusCommand.NO_SNP_OP_IN_PROGRESS_MSG);
 
             return;
         }