Automatically expire stale Pending node entries

CHANGELOG.md

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 [7.0.14]: https://github.com/microsoft/CCF/releases/tag/ccf-7.0.14
 
+### Added
+
+- Pending node entries are now automatically removed when they stop retrying joins for the configurable `pending_node_timeout` delay (1 hour by default). Set it to `0s` to disable automatic removal (#8173).
+
 ### Fixed
 
 - If the view changed while a transaction was committing, the transaction could apply its writes to the local key-value store and then fail to replicate, leaving state that never reached consensus. The transaction's view is now validated atomically with the allocation of its version, so it is rejected before any map is modified, and `ccf::kv::CommitResult::FAIL_NO_REPLICATE` no longer implies a locally applied write (#8242).

doc/host_config_schema/host_config.json

@@ -575,6 +575,11 @@
       "description": "This section includes configuration for periodic cleanup of old files (snapshots, ledger chunks)",
       "additionalProperties": false
     },
+    "pending_node_timeout": {
+      "type": "string",
+      "default": "1h",
+      "description": "Delay after which the primary automatically removes a Pending node that has stopped sending join requests. Cleanup runs at least once per minute, so removal may occur up to one minute after this delay. Set to 0s to disable automatic removal."
+    },
     "identity_history_fetch": {
       "type": "object",
       "properties": {

doc/operations/start_network.rst

@@ -62,6 +62,8 @@ If the network has not yet been opened by members (see :ref:`governance/open_net
 
 The ``Pending`` joining node automatically polls the service (interval configurable via ``join.retry_timeout`` configuration entry) until the members have successfully transitioned the node to the ``Trusted`` state. It is only then that the joining node transitions to the ``PartOfNetwork`` state and starts updating its ledger.
 
+The primary automatically removes a ``Pending`` node that has stopped sending join requests for longer than the ``pending_node_timeout`` configuration entry (1 hour by default). Cleanup runs at least once per minute. Set ``pending_node_timeout`` to ``0s`` to disable automatic removal.
+
 .. tip:: After the node has been trusted by members, operators should poll the :http:GET:`/node/state` endpoint on the newly added node, using the node's self-signed certificate as TLS CA, until the ``{"state": "PartOfNetwork"}`` is reported. This status confirms that the replication of the ledger has started on this node.
 
 .. note:: To accelerate the joining procedure, it is possible for new nodes to join from a snapshot. More information on snapshots :ref:`here <operations/ledger_snapshot:Join or Recover From Snapshot>`.

include/ccf/node/startup_config.h

@@ -29,6 +29,8 @@ namespace ccf
 
     ccf::ds::SizeString historical_cache_soft_limit = {"512MB"};
 
+    ccf::ds::TimeString pending_node_timeout = {"1h"};
+
     ccf::consensus::Configuration consensus = {};
     ccf::NodeInfoNetwork network;
 

include/ccf/service/node_info.h

@@ -72,6 +72,10 @@ namespace ccf
      * compatibility.
      */
     bool retired_committed = false;
+
+    /** Time when this Pending node last sent a join request, in milliseconds
+     * since the Unix epoch. Optional for backward compatibility. */
+    std::optional<int64_t> pending_last_seen = std::nullopt;
   };
   DECLARE_JSON_TYPE_WITH_BASE_AND_OPTIONAL_FIELDS(NodeInfo, NodeInfoNetwork);
   DECLARE_JSON_REQUIRED_FIELDS(
@@ -84,7 +88,8 @@ namespace ccf
     certificate_signing_request,
     public_key,
     node_data,
-    retired_committed);
+    retired_committed,
+    pending_last_seen);
 }
 
 FMT_BEGIN_NAMESPACE

src/common/configuration.h

@@ -145,6 +145,7 @@ namespace ccf
     attestation,
     snapshots,
     files_cleanup,
+    pending_node_timeout,
     node_to_node_message_limit,
     historical_cache_soft_limit,
     identity_history_fetch);

src/node/node_state.h

@@ -44,6 +44,7 @@
 #include "node/local_sealing.h"
 #include "node/node_inbound_message.h"
 #include "node/node_to_node_channel_manager.h"
+#include "node/pending_node_cleanup.h"
 #include "node/recovery_decision_protocol.h"
 #include "node/recovery_snapshot_ledger.h"
 #include "node/signature_cache_subsystem.h"
@@ -485,6 +486,7 @@ namespace ccf
     // JWT key auto-refresh
     //
     std::shared_ptr<JwtKeyAutoRefresh> jwt_key_auto_refresh;
+    std::shared_ptr<PendingNodeCleanup> pending_node_cleanup;
 
     std::unique_ptr<StartupSnapshotInfo> startup_snapshot_info = nullptr;
     // Set to the snapshot seqno when a node starts from one and remembered for
@@ -3599,6 +3601,12 @@ namespace ccf
         commit_callbacks,
         public_only);
 
+      pending_node_cleanup = std::make_shared<PendingNodeCleanup>(
+        node_client,
+        consensus,
+        std::chrono::milliseconds(config.pending_node_timeout));
+      pending_node_cleanup->start();
+
       network.tables->set_consensus(consensus);
       network.tables->set_snapshotter(snapshotter);
 

src/node/pending_node_cleanup.h

@@ -0,0 +1,92 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "kv/kv_types.h"
+#include "node/node_client.h"
+#include "tasks/basic_task.h"
+#include "tasks/task_system.h"
+
+#include <algorithm>
+#include <chrono>
+#include <memory>
+
+namespace ccf
+{
+  class PendingNodeCleanup
+    : public std::enable_shared_from_this<PendingNodeCleanup>
+  {
+  private:
+    static constexpr auto max_cleanup_interval = std::chrono::minutes(1);
+
+    std::shared_ptr<NodeClient> node_client;
+    std::shared_ptr<ccf::kv::Consensus> consensus;
+    std::chrono::milliseconds cleanup_interval;
+    ccf::tasks::Task periodic_cleanup_task;
+
+    void send_cleanup_request()
+    {
+      ::http::Request request(
+        fmt::format(
+          "/{}/{}",
+          ccf::get_actor_prefix(ccf::ActorsType::nodes),
+          "network/nodes/remove_expired_pending"),
+        HTTP_POST);
+      request.set_header(http::headers::CONTENT_LENGTH, "0");
+
+      node_client->make_request(request);
+    }
+
+  public:
+    PendingNodeCleanup(
+      std::shared_ptr<NodeClient> node_client_,
+      std::shared_ptr<ccf::kv::Consensus> consensus_,
+      std::chrono::milliseconds pending_node_timeout) :
+      node_client(std::move(node_client_)),
+      consensus(std::move(consensus_)),
+      cleanup_interval(std::min(
+        pending_node_timeout,
+        std::chrono::duration_cast<std::chrono::milliseconds>(
+          max_cleanup_interval)))
+    {}
+
+    ~PendingNodeCleanup()
+    {
+      stop();
+    }
+
+    void start()
+    {
+      if (cleanup_interval <= std::chrono::milliseconds::zero())
+      {
+        return;
+      }
+
+      const auto self = weak_from_this();
+      periodic_cleanup_task = ccf::tasks::make_basic_task([self]() {
+        const auto self_sp = self.lock();
+        if (self_sp == nullptr)
+        {
+          return;
+        }
+
+        if (self_sp->consensus->can_replicate())
+        {
+          self_sp->send_cleanup_request();
+        }
+      });
+
+      ccf::tasks::add_periodic_task(
+        periodic_cleanup_task, cleanup_interval, cleanup_interval);
+    }
+
+    void stop()
+    {
+      if (periodic_cleanup_task != nullptr)
+      {
+        periodic_cleanup_task->cancel_task();
+        periodic_cleanup_task = nullptr;
+      }
+    }
+  };
+}