IGNITE-29173 Add kill lock command and locks control after jobs and services

docs/_docs/distributed-locks.adoc

@@ -52,6 +52,18 @@ In Ignite, locks are supported only for the `TRANSACTIONAL` atomicity mode, whic
 `CacheConfiguration.atomicityMode` parameter.
 ====
 
+== Lock Ownership
+
+Explicit locks are bound to the thread that acquired them: the lock must be released by the same thread. Locks acquired by a compute job,
+by the `Service.execute()` method, or by a service method invoked remotely or from a thin client, and not released by the time the method
+returns are released forcibly and a warning is logged, since otherwise they would be inherited by the next task executed by the same pool
+thread or never released at all. Locks acquired by a service method invoked locally through a service proxy belong to the calling thread
+and are not released.
+Thus, a lock can't be held across job continuations (`ComputeJobContinuation.holdcc()`) or failovers.
+
+If a lock is released externally (for example, it is killed with the link:sql-reference/operational-commands#kill-lock[KILL LOCK] command),
+`Lock.unlock()` throws `IllegalStateException` to signal that the code protected by the lock may have been executed without the lock held.
+
 == Locks and Transactions
 
 Explicit locks are not transactional and cannot not be used from within transactions (exception will be thrown).

docs/_docs/sql-reference/operational-commands.adoc

@@ -424,3 +424,36 @@ control.bat --kill CLIENT connection_id [--node-id node_id]
 Value differs on nodes for the same client. You can always find parameter values with the link:monitoring-metrics/system-views#client_connections[CLIENT_CONNECTIONS] view.
 
 * `node_id` - corresponds to the node_id to drop connection from.
+
+== KILL LOCK
+
+The `KILL LOCK` command allows you to cancel an explicit cache lock acquired via the `Cache.lock(key)` API.
+
+Note that killing a lock may affect the execution of the application code that relies on the lock, so the command must be used with caution.
+
+The command releases the lock on all its keys. If the lock is not acquired yet (the thread is still waiting for it), the lock request is
+cancelled and the waiting `Lock.lock()` call fails with `CacheException`. Other threads holding or waiting for the same keys are not affected.
+The subsequent `Lock.unlock()` call of the thread that has acquired the killed lock fails with `IllegalStateException`.
+
+[tabs]
+--
+
+tab:Unix[]
+[source,bash]
+----
+./control.sh --kill LOCK xid [--node-id node_id] [--yes]
+----
+
+tab:Windows[]
+[source,bash]
+----
+control.bat --kill LOCK xid [--node-id node_id] [--yes]
+----
+
+--
+
+=== Parameters
+
+* `xid` - the lock id (mandatory) that can be retrieved via the link:monitoring-metrics/system-views#cache_explicit_locks[CACHE_EXPLICIT_LOCKS] view.
+* `node_id` - the node id (optional). The id of the node where the lock is acquired. If specified, the request is sent only to this node. Otherwise, all nodes are checked.
+

modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java

@@ -75,6 +75,8 @@
 import org.apache.ignite.internal.management.kill.KillAllCommand;
 import org.apache.ignite.internal.management.kill.KillAllCommandArg;
 import org.apache.ignite.internal.management.kill.KillCommand;
+import org.apache.ignite.internal.management.kill.KillLockCommand;
+import org.apache.ignite.internal.management.kill.KillLockCommandArg;
 import org.apache.ignite.internal.management.meta.MetaCommand;
 import org.apache.ignite.internal.management.meta.MetaRemoveCommand;
 import org.apache.ignite.internal.management.meta.MetaUpdateCommand;
@@ -94,6 +96,7 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.ListeningTestLogger;
 import org.apache.ignite.testframework.LogListener;
@@ -557,6 +560,8 @@ else if (cmd.getClass() == MetaRemoveCommand.class)
             cmdText = F.concat(cmdText, "--typeId", "1");
         else if (cmd.getClass() == KillAllCommand.class)
             cmdText = F.concat(cmdText, "SQL");
+        else if (cmd.getClass() == KillLockCommand.class)
+            cmdText = F.concat(cmdText, IgniteUuid.randomUuid().toString());
 
         args = parseArgs(asList(cmdText));
 
@@ -729,6 +734,22 @@ public void testKillArguments() {
         assertParseArgsThrows("String representation of \"java.util.UUID\" is exepected", IllegalArgumentException.class,
             "--kill", "continuous", UUID.randomUUID().toString(), "not_a_uuid");
 
+        // Lock command format errors.
+        assertParseArgsThrows("Argument xid required.", "--kill", "lock");
+
+        assertParseArgsThrows("Invalid UUID string: not_a_uuid", IllegalArgumentException.class,
+            "--kill", "lock", "not_a_uuid");
+
+        // Lock command parse success.
+        IgniteUuid xid = IgniteUuid.randomUuid();
+        UUID nodeId = UUID.randomUUID();
+
+        KillLockCommandArg lockArg = (KillLockCommandArg)parseArgs(asList(
+            "--kill", "lock", xid.toString(), "--node-id", nodeId.toString())).commandArg();
+
+        assertEquals(xid, lockArg.xid());
+        assertEquals(nodeId, lockArg.nodeId());
+
         // Kill all command format errors.
         assertParseArgsThrows("Argument target required.", "--kill", "all");
         assertParseArgsThrows("Can't parse value 'unknown'", "--kill", "all", "unknown");

modules/control-utility/src/test/java/org/apache/ignite/util/KillCommandsControlShTest.java

@@ -22,6 +22,7 @@
 import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
 import java.util.function.Consumer;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheAtomicityMode;
@@ -38,6 +39,7 @@
 import org.apache.ignite.internal.util.typedef.T3;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.systemview.view.CacheExplicitLockView;
 import org.apache.ignite.spi.systemview.view.ComputeJobView;
 import org.apache.ignite.spi.systemview.view.SystemView;
 import org.apache.ignite.testframework.GridTestUtils;
@@ -46,10 +48,12 @@
 import org.junit.Test;
 
 import static java.util.Arrays.asList;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_CONNECTION_FAILED;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_INVALID_ARGUMENTS;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR;
+import static org.apache.ignite.internal.processors.cache.GridCacheMvccManager.CACHE_EXPLICIT_LOCKS_VIEW;
 import static org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.doSnapshotCancellationTest;
 import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.SNAPSHOT_METRICS;
 import static org.apache.ignite.internal.processors.job.GridJobProcessor.JOBS_VIEW;
@@ -165,6 +169,50 @@ public void testCancelSQLQuery() {
         });
     }
 
+    /** @throws Exception If failed. */
+    @Test
+    public void testKillLock() throws Exception {
+        int key = 0;
+
+        Lock lock = client.cache(DEFAULT_CACHE_NAME).lock(key);
+
+        // Explicit lock is bound to the thread and stays held after the thread finishes.
+        long threadId = GridTestUtils.runAsync(() -> {
+            lock.lock();
+
+            return Thread.currentThread().getId();
+        }).get();
+
+        CacheExplicitLockView view = F.find(client.context().systemView().view(CACHE_EXPLICIT_LOCKS_VIEW), null,
+            v -> v.threadId() == threadId);
+
+        assertNotNull(view);
+
+        IgniteUuid xid = view.xid();
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--kill", "lock", xid.toString()));
+
+        Lock srvLock = srvs.get(0).cache(DEFAULT_CACHE_NAME).lock(key);
+
+        assertTrue(srvLock.tryLock(getTestTimeout(), MILLISECONDS));
+
+        srvLock.unlock();
+
+        assertContains(log, testOut.toString(),
+            "Lock with xid " + xid + " has been killed on node " + client.localNode().id() + '.');
+
+        String notFoundMsg = "Lock with xid " + xid + " is not found.";
+
+        assertNotContains(log, testOut.toString(), notFoundMsg);
+
+        // The lock is already killed.
+        assertEquals(EXIT_CODE_OK, execute("--kill", "lock", xid.toString()));
+
+        assertContains(log, testOut.toString(), notFoundMsg);
+    }
+
     /** @throws Exception If failed. */
     @Test
     public void testCancelContinuousQuery() throws Exception {

modules/core/src/main/java/org/apache/ignite/internal/management/kill/KillCommand.java

@@ -33,7 +33,8 @@ public KillCommand() {
             new KillContinuousCommand(),
             new KillClientCommand(),
             new KillSnapshotCommand(),
-            new KillConsistencyCommand()
+            new KillConsistencyCommand(),
+            new KillLockCommand()
         );
     }
 }

modules/core/src/main/java/org/apache/ignite/internal/management/kill/KillLockCommand.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.management.kill;
+
+import java.util.Collection;
+import java.util.UUID;
+import java.util.function.Consumer;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.management.api.CommandUtils;
+import org.apache.ignite.internal.management.api.ComputeCommand;
+
+/**
+ * Kill explicit lock command.
+ */
+public class KillLockCommand implements ComputeCommand<KillLockCommandArg, UUID> {
+    /** {@inheritDoc} */
+    @Override public String description() {
+        return "Kill explicit lock by xid";
+    }
+
+    /** {@inheritDoc} */
+    @Override public Class<KillLockCommandArg> argClass() {
+        return KillLockCommandArg.class;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Class<KillLockTask> taskClass() {
+        return KillLockTask.class;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<ClusterNode> nodes(Collection<ClusterNode> nodes, KillLockCommandArg arg) {
+        return CommandUtils.nodeOrAll(arg.nodeId(), nodes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void printResult(KillLockCommandArg arg, UUID res, Consumer<String> printer) {
+        if (res == null) {
+            printer.accept("Lock with xid " + arg.xid() + " is not found" +
+                (arg.nodeId() == null ? "" : " on node " + arg.nodeId()) + '.');
+        }
+        else
+            printer.accept("Lock with xid " + arg.xid() + " has been killed on node " + res + '.');
+    }
+
+    /** {@inheritDoc} */
+    @Override public String confirmationPrompt(KillLockCommandArg arg) {
+        return "Warning: the command will kill explicit lock with xid " + arg.xid()
+            + ". Killing a lock may affect the execution of application code that relies on it.";
+    }
+}

modules/core/src/main/java/org/apache/ignite/internal/management/kill/KillLockCommandArg.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.management.kill;
+
+import java.util.UUID;
+import org.apache.ignite.internal.Order;
+import org.apache.ignite.internal.dto.IgniteDataTransferObject;
+import org.apache.ignite.internal.management.api.Argument;
+import org.apache.ignite.internal.management.api.CliConfirmArgument;
+import org.apache.ignite.internal.management.api.Positional;
+import org.apache.ignite.lang.IgniteUuid;
+
+/**
+ * Arguments for --kill lock command.
+ */
+@CliConfirmArgument
+public class KillLockCommandArg extends IgniteDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0;
+
+    /** */
+    @Order(0)
+    @Positional
+    @Argument(description = "Lock identifier (xid from CACHE_EXPLICIT_LOCKS view)")
+    IgniteUuid xid;
+
+    /** */
+    @Order(1)
+    @Argument(description = "Node id the lock is acquired on", optional = true)
+    UUID nodeId;
+
+    /** */
+    public IgniteUuid xid() {
+        return xid;
+    }
+
+    /** */
+    public void xid(IgniteUuid xid) {
+        this.xid = xid;
+    }
+
+    /** */
+    public UUID nodeId() {
+        return nodeId;
+    }
+
+    /** */
+    public void nodeId(UUID nodeId) {
+        this.nodeId = nodeId;
+    }
+}