Automatically expire stale Pending node entries

CHANGELOG.md

@@ -5,12 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [7.0.14]
+## [7.0.15]
 
-[7.0.14]: https://github.com/microsoft/CCF/releases/tag/ccf-7.0.14
+[7.0.15]: https://github.com/microsoft/CCF/releases/tag/ccf-7.0.15
 
 ### Added
 
+- Pending node entries are now automatically removed when they stop retrying joins for the configurable `pending_node_timeout` delay (1 hour by default). Set it to `0s` to disable automatic removal (#8173).
 - COSE Sign1 verification now accepts the fully-specified ECDSA algorithm identifiers introduced by [RFC 9864](https://www.rfc-editor.org/rfc/rfc9864.html): `ESP256` (-9), `ESP384` (-51) and `ESP512` (-52), in addition to the deprecated `ES256` (-7), `ES384` (-35) and `ES512` (-36) they replace. `ESn` and `ESPn` are treated as equivalent for the curve they denote, which CCF already requires to match the verification key. Signatures produced by CCF continue to use the `ES` identifiers (#8267).
 
 ### Fixed

doc/host_config_schema/host_config.json

@@ -575,6 +575,11 @@
       "description": "This section includes configuration for periodic cleanup of old files (snapshots, ledger chunks)",
       "additionalProperties": false
     },
+    "pending_node_timeout": {
+      "type": "string",
+      "default": "1h",
+      "description": "Delay after which the primary automatically removes a Pending node that has stopped sending join requests. Cleanup runs at least once per minute, so removal may occur up to one minute after this delay. Set to 0s to disable automatic removal."
+    },
     "identity_history_fetch": {
       "type": "object",
       "properties": {

doc/operations/start_network.rst

@@ -62,6 +62,8 @@ If the network has not yet been opened by members (see :ref:`governance/open_net
 
 The ``Pending`` joining node automatically polls the service (interval configurable via ``join.retry_timeout`` configuration entry) until the members have successfully transitioned the node to the ``Trusted`` state. It is only then that the joining node transitions to the ``PartOfNetwork`` state and starts updating its ledger.
 
+The primary automatically removes a ``Pending`` node that has stopped sending join requests for longer than the ``pending_node_timeout`` configuration entry (1 hour by default). Cleanup runs at least once per minute. Set ``pending_node_timeout`` to ``0s`` to disable automatic removal.
+
 .. tip:: After the node has been trusted by members, operators should poll the :http:GET:`/node/state` endpoint on the newly added node, using the node's self-signed certificate as TLS CA, until the ``{"state": "PartOfNetwork"}`` is reported. This status confirms that the replication of the ledger has started on this node.
 
 .. note:: To accelerate the joining procedure, it is possible for new nodes to join from a snapshot. More information on snapshots :ref:`here <operations/ledger_snapshot:Join or Recover From Snapshot>`.

include/ccf/node/startup_config.h

@@ -29,6 +29,8 @@ namespace ccf
 
     ccf::ds::SizeString historical_cache_soft_limit = {"512MB"};
 
+    ccf::ds::TimeString pending_node_timeout = {"1h"};
+
     ccf::consensus::Configuration consensus = {};
     ccf::NodeInfoNetwork network;
 

include/ccf/service/node_info.h

@@ -72,6 +72,10 @@ namespace ccf
      * compatibility.
      */
     bool retired_committed = false;
+
+    /** Time when this Pending node last sent a join request, in milliseconds
+     * since the Unix epoch. Optional for backward compatibility. */
+    std::optional<int64_t> pending_last_seen = std::nullopt;
   };
   DECLARE_JSON_TYPE_WITH_BASE_AND_OPTIONAL_FIELDS(NodeInfo, NodeInfoNetwork);
   DECLARE_JSON_REQUIRED_FIELDS(
@@ -84,7 +88,8 @@ namespace ccf
     certificate_signing_request,
     public_key,
     node_data,
-    retired_committed);
+    retired_committed,
+    pending_last_seen);
 }
 
 FMT_BEGIN_NAMESPACE

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ccf"
-version = "7.0.14"
+version = "7.0.15"
 authors = [
   { name="CCF Team", email="CCF-Sec@microsoft.com" },
 ]

src/common/configuration.h

@@ -145,6 +145,7 @@ namespace ccf
     attestation,
     snapshots,
     files_cleanup,
+    pending_node_timeout,
     node_to_node_message_limit,
     historical_cache_soft_limit,
     identity_history_fetch);

src/node/node_state.h

@@ -44,6 +44,7 @@
 #include "node/local_sealing.h"
 #include "node/node_inbound_message.h"
 #include "node/node_to_node_channel_manager.h"
+#include "node/pending_node_cleanup.h"
 #include "node/recovery_decision_protocol.h"
 #include "node/recovery_snapshot_ledger.h"
 #include "node/signature_cache_subsystem.h"
@@ -485,6 +486,7 @@ namespace ccf
     // JWT key auto-refresh
     //
     std::shared_ptr<JwtKeyAutoRefresh> jwt_key_auto_refresh;
+    std::shared_ptr<PendingNodeCleanup> pending_node_cleanup;
 
     std::unique_ptr<StartupSnapshotInfo> startup_snapshot_info = nullptr;
     // Set to the snapshot seqno when a node starts from one and remembered for
@@ -3598,6 +3600,12 @@ namespace ccf
         commit_callbacks,
         public_only);
 
+      pending_node_cleanup = std::make_shared<PendingNodeCleanup>(
+        node_client,
+        consensus,
+        std::chrono::milliseconds(config.pending_node_timeout));
+      pending_node_cleanup->start();
+
       network.tables->set_consensus(consensus);
       network.tables->set_snapshotter(snapshotter);
 

src/node/pending_node_cleanup.h

@@ -0,0 +1,92 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "kv/kv_types.h"
+#include "node/node_client.h"
+#include "tasks/basic_task.h"
+#include "tasks/task_system.h"
+
+#include <algorithm>
+#include <chrono>
+#include <memory>
+
+namespace ccf
+{
+  class PendingNodeCleanup
+    : public std::enable_shared_from_this<PendingNodeCleanup>
+  {
+  private:
+    static constexpr auto max_cleanup_interval = std::chrono::minutes(1);
+
+    std::shared_ptr<NodeClient> node_client;
+    std::shared_ptr<ccf::kv::Consensus> consensus;
+    std::chrono::milliseconds cleanup_interval;
+    ccf::tasks::Task periodic_cleanup_task;
+
+    void send_cleanup_request()
+    {
+      ::http::Request request(
+        fmt::format(
+          "/{}/{}",
+          ccf::get_actor_prefix(ccf::ActorsType::nodes),
+          "network/nodes/remove_expired_pending"),
+        HTTP_POST);
+      request.set_header(http::headers::CONTENT_LENGTH, "0");
+
+      node_client->make_request(request);
+    }
+
+  public:
+    PendingNodeCleanup(
+      std::shared_ptr<NodeClient> node_client_,
+      std::shared_ptr<ccf::kv::Consensus> consensus_,
+      std::chrono::milliseconds pending_node_timeout) :
+      node_client(std::move(node_client_)),
+      consensus(std::move(consensus_)),
+      cleanup_interval(std::min(
+        pending_node_timeout,
+        std::chrono::duration_cast<std::chrono::milliseconds>(
+          max_cleanup_interval)))
+    {}
+
+    ~PendingNodeCleanup()
+    {
+      stop();
+    }
+
+    void start()
+    {
+      if (cleanup_interval <= std::chrono::milliseconds::zero())
+      {
+        return;
+      }
+
+      const auto self = weak_from_this();
+      periodic_cleanup_task = ccf::tasks::make_basic_task([self]() {
+        const auto self_sp = self.lock();
+        if (self_sp == nullptr)
+        {
+          return;
+        }
+
+        if (self_sp->consensus->can_replicate())
+        {
+          self_sp->send_cleanup_request();
+        }
+      });
+
+      ccf::tasks::add_periodic_task(
+        periodic_cleanup_task, cleanup_interval, cleanup_interval);
+    }
+
+    void stop()
+    {
+      if (periodic_cleanup_task != nullptr)
+      {
+        periodic_cleanup_task->cancel_task();
+        periodic_cleanup_task = nullptr;
+      }
+    }
+  };
+}