New RPCConnectionManager: Single unified manager, no ringbuffer, OpenSSL-owned sockets

CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [7.0.13]
+
+[7.0.13]: https://github.com/microsoft/CCF/releases/tag/ccf-7.0.13
+
+### Changed
+
+- TLS is now terminated by OpenSSL directly on the socket, rather than being relayed over the ringbuffer and decrypted through a memory BIO. The session interfaces in `include/ccf/node/session.h` and `include/ccf/research/custom_protocol_subsystem_interface.h` have changed shape accordingly: a session now receives and emits plaintext, taking ownership of the inbound buffer, and writes its output through a `ccf::SessionWriter` rather than a `tls::Context` (#8117).
+
 ## [7.0.12]
 
 [7.0.12]: https://github.com/microsoft/CCF/releases/tag/ccf-7.0.12

CMakeLists.txt

@@ -706,10 +706,10 @@ if(BUILD_TESTS)
     )
 
     add_unit_test(
-      rpc_connections_test
-      ${CMAKE_CURRENT_SOURCE_DIR}/src/host/test/rpc_connections.cpp
+      openssl_server_test
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/host/test/openssl_server_test.cpp
     )
-    target_link_libraries(rpc_connections_test PRIVATE uv)
+    target_link_libraries(openssl_server_test PRIVATE ccf_tasks http_parser uv)
 
     add_unit_test(
       raft_test
@@ -870,7 +870,10 @@ if(BUILD_TESTS)
       ${CMAKE_CURRENT_SOURCE_DIR}/src/node/test/node_info_json.cpp
     )
 
-    add_unit_test(tls_test ${CMAKE_CURRENT_SOURCE_DIR}/src/tls/test/main.cpp)
+    add_unit_test(
+      tls_test
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/clients/tls/test/main.cpp
+    )
     target_link_libraries(tls_test PRIVATE ${CMAKE_THREAD_LIBS_INIT})
 
     add_unit_test(

doc/contribute/onboarding.rst

@@ -63,12 +63,13 @@ Note that this diagram deliberately does not represent host-to-enclave communica
 .. mermaid::
 
     flowchart TB
-        Client[HTTPS/1.1 Client <a href='../build_apps/auth/index.html'>auth</a>] -- TLS 1.2 or 1.3 --> TLSSession
-        TLSSession[TLS Session <a href='https://github.com/microsoft/CCF/blob/main/src/enclave/tls_session.h'>src</a>] -- PlainText --> HTTPSession
+        Client[HTTPS/1.1 Client <a href='../build_apps/auth/index.html'>auth</a>] -- TLS 1.2 or 1.3 --> TLSConnection
+        TLSConnection[TLS Connection <a href='https://github.com/microsoft/CCF/blob/main/src/host/tls/openssl_server.h'>src</a>] -- PlainText --> HTTPSession
+        HTTPSession -- PlainText --> SessionWriter[Session Writer <a href='https://github.com/microsoft/CCF/blob/main/src/enclave/session_writer.h'>src</a>]
+        SessionWriter --> TLSConnection
         HTTPSession[HTTP Session <a href='https://github.com/microsoft/CCF/blob/main/src/http/http_session.h'>src</a>] -- Request --> Endpoint[Application Endpoint <a href='../build_apps/api.html#application-endpoint-registration'>doc</a>]
         Endpoint -- Response --> HTTPSession
-        HTTPSession --> TLSSession
-        TLSSession --> Client
+        TLSConnection --> Client
         Endpoint -- WriteSet --> Store[Store <a href='../build_apps/kv/index.html'>doc</a>]
         Store -- LedgerEntry --> Ledger[Ledger <a href='../architecture/ledger.html'>doc</a>]
         Ledger -- LedgerEntry --> Disk

include/ccf/node/session.h

@@ -3,7 +3,7 @@
 #pragma once
 
 #include <cstdint>
-#include <span>
+#include <vector>
 
 namespace ccf
 {
@@ -12,8 +12,55 @@ namespace ccf
   public:
     virtual ~Session() = default;
 
-    virtual void handle_incoming_data(std::span<const uint8_t> data) = 0;
+    // Inbound bytes for this session. Ownership is transferred, so that the
+    // buffer the transport has already built is moved through the session
+    // rather than copied again.
+    virtual void handle_incoming_data(std::vector<uint8_t>&& data) = 0;
     virtual void send_data(std::vector<uint8_t>&& data) = 0;
     virtual void close_session() = 0;
   };
+
+  // Abstract output sink injected into Sessions: a Session hands its outbound
+  // bytes (and connection-teardown requests) to a SessionWriter, which is
+  // implemented by the RPC transport.
+  //
+  // IMPORTANT: Sessions may invoke these methods from worker threads, so
+  // implementations MUST be thread-safe and must marshal any socket operations
+  // onto their I/O thread if required.
+  class SessionWriter
+  {
+  public:
+    virtual ~SessionWriter() = default;
+
+    // `id` throughout is the connection identifier the session was created
+    // with (ccf::tls::ConnID).
+
+    // Queue bytes to be written to the socket associated with `id`. Ownership
+    // is transferred, so that a response which may be arbitrarily large is
+    // moved through to the transport rather than copied again.
+    //
+    // Fire-and-forget: there is currently no backpressure signal.
+    //
+    // FUTURE: to surface genuine TCP-layer backpressure, an implementation
+    // should report when a connection's pending-write queue exceeds a watermark
+    // (tracking per-connection queued bytes) and return a writable/would-block
+    // status here.
+    virtual void write_outbound(int64_t id, std::vector<uint8_t>&& data) = 0;
+
+    // Tear down the connection: stop the underlying socket and drop the
+    // session.
+    virtual void close_socket(int64_t id) = 0;
+
+    // Report that `bytes` of previously delivered inbound data have now been
+    // processed. The transport uses this to decide when it may read more: it
+    // stops reading once the node is holding more unprocessed inbound data
+    // than it is willing to, and resumes as sessions catch up. Without it a
+    // client could make the node queue work faster than it retires it, for as
+    // long as it liked.
+    //
+    // A session which does not report is not penalised beyond its own
+    // connection - the transport releases whatever is still outstanding when
+    // the connection closes - so this defaults to a no-op.
+    virtual void inbound_consumed(int64_t /*id*/, size_t /*bytes*/) {}
+  };
 }

include/ccf/node/startup_config.h

@@ -29,6 +29,12 @@ namespace ccf
 
     ccf::ds::SizeString historical_cache_soft_limit = {"512MB"};
 
+    // How long an idle RPC (client TLS) connection is kept before it is closed.
+    // std::nullopt disables idle closure (connections are never closed for
+    // being idle).
+    std::optional<ccf::ds::TimeString> idle_connection_timeout =
+      ccf::ds::TimeString("60s");
+
     ccf::consensus::Configuration consensus = {};
     ccf::NodeInfoNetwork network;
 

include/ccf/research/custom_protocol_subsystem_interface.h

@@ -16,15 +16,14 @@ namespace ccf
 {
   namespace tls
   {
-    class Context;
     using ConnID = int64_t;
   }
 
   class CustomProtocolSubsystemInterface : public AbstractNodeSubSystem
   {
   public:
     using CreateSessionFn = std::function<std::shared_ptr<Session>(
-      ccf::tls::ConnID, const std::unique_ptr<tls::Context>&&)>;
+      ccf::tls::ConnID, ccf::SessionWriter&)>;
 
     ~CustomProtocolSubsystemInterface() override = default;
 
@@ -41,7 +40,7 @@ namespace ccf
     virtual std::shared_ptr<Session> create_session(
       const std::string& protocol_name,
       ccf::tls::ConnID conn_id,
-      const std::unique_ptr<tls::Context>&& ctx) = 0;
+      ccf::SessionWriter& writer) = 0;
 
     struct Essentials
     {

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ccf"
-version = "7.0.12"
+version = "7.0.13"
 authors = [
   { name="CCF Team", email="CCF-Sec@microsoft.com" },
 ]

src/clients/rpc_tls_client.h

@@ -8,6 +8,7 @@
 #include "tls_client.h"
 
 #define FMT_HEADER_ONLY
+#include <deque>
 #include <fmt/format.h>
 #include <nlohmann/json.hpp>
 #include <optional>
@@ -90,7 +91,7 @@ namespace client
       return call_raw(prep.encoded);
     }
 
-    std::optional<Response> last_response;
+    std::deque<Response> pending_responses;
 
   public:
     using TlsClient::TlsClient;
@@ -206,20 +207,20 @@ namespace client
 
     Response read_response()
     {
-      last_response = std::nullopt;
-
-      while (!last_response.has_value())
+      while (pending_responses.empty())
       {
         const auto next = read_all();
         parser.execute(next.data(), next.size());
       }
 
-      return std::move(last_response.value());
+      auto response = std::move(pending_responses.front());
+      pending_responses.pop_front();
+      return response;
     }
 
     std::optional<Response> read_response_non_blocking()
     {
-      if (bytes_available())
+      if (!pending_responses.empty() || bytes_available())
       {
         return read_response();
       }
@@ -232,8 +233,8 @@ namespace client
       ccf::http::HeaderMap&& headers,
       std::vector<uint8_t>&& body) override
     {
-      last_response = {
-        next_recv_id++, status, std::move(headers), std::move(body)};
+      pending_responses.push_back(
+        {next_recv_id++, status, std::move(headers), std::move(body)});
     }
 
     void set_prefix(const std::string& prefix_)

src/clients/tls/README.md

@@ -0,0 +1,14 @@
+# TLS certificate helpers (client tooling only)
+
+These helpers configure an OpenSSL client with a trusted root and an owned
+identity:
+
+- `CA` parses one or more root certificates and populates an `SSL_CTX`'s
+  trusted certificate store.
+- `Cert` holds a client's own certificate and private key, and applies them
+  (and the peer verification mode) to an `SSL_CTX` or `SSL`.
+
+Their only consumer is `TlsClient` in the parent directory, used by the C++
+test and perf clients. **No node code uses them.** The node's inbound TLS is
+handled by `src/host/tls`, which builds its own `SSL_CTX`, and its outbound
+requests go through libcurl.

src/tls/cert.h → src/clients/tls/cert.h

@@ -3,9 +3,9 @@
 #pragma once
 
 #include "ccf/crypto/openssl/openssl_wrappers.h"
+#include "clients/tls/ca.h"
 #include "crypto/openssl/ec_key_pair.h"
 #include "ds/internal_logger.h"
-#include "tls/ca.h"
 
 #include <cstring>
 #include <memory>

src/clients/tls/test/main.cpp

@@ -0,0 +1,129 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+
+// Unit tests for the certificate helpers used by the C++ test clients, which
+// configure an OpenSSL context and connection with a trusted root and an owned
+// identity. The TLS transport itself is tested in
+// src/host/test/openssl_server_test.cpp.
+
+#include "ccf/crypto/ec_key_pair.h"
+#include "ccf/crypto/verifier.h"
+#include "ccf/ds/x509_time_fmt.h"
+#include "clients/tls/ca.h"
+#include "clients/tls/cert.h"
+#include "crypto/certs.h"
+
+#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+#include <chrono>
+#include <doctest/doctest.h>
+#include <memory>
+#include <optional>
+#include <string>
+
+namespace
+{
+  constexpr size_t certificate_validity_period_days = 365;
+
+  std::string valid_from_yesterday()
+  {
+    using namespace std::literals;
+    return ccf::ds::to_x509_time_string(std::chrono::system_clock::now() - 24h);
+  }
+
+  ccf::crypto::Pem generate_self_signed_cert(
+    const ccf::crypto::ECKeyPairPtr& kp, const std::string& name)
+  {
+    return ccf::crypto::create_self_signed_cert(
+      kp, name, {}, valid_from_yesterday(), certificate_validity_period_days);
+  }
+
+  struct NetworkCA
+  {
+    ccf::crypto::ECKeyPairPtr kp;
+    ccf::crypto::Pem cert;
+  };
+
+  /// Get self-signed CA certificate.
+  NetworkCA get_ca()
+  {
+    auto kp = ccf::crypto::make_ec_key_pair();
+    return {kp, generate_self_signed_cert(kp, "CN=issuer")};
+  }
+
+  /// Creates a ::tls::Cert endorsed by the given CA.
+  std::unique_ptr<::tls::Cert> get_dummy_cert(
+    NetworkCA& net_ca, const std::string& name, bool auth_required = true)
+  {
+    auto ca = std::make_unique<::tls::CA>(net_ca.cert.str());
+
+    // Create a signing request and sign with the CA
+    auto kp = ccf::crypto::make_ec_key_pair();
+    auto crt = ccf::crypto::create_endorsed_cert(
+      kp,
+      "CN=" + name,
+      {},
+      valid_from_yesterday(),
+      certificate_validity_period_days,
+      net_ca.kp->private_key_pem(),
+      net_ca.cert);
+
+    // Verify node certificate with the CA's certificate
+    auto v = ccf::crypto::make_verifier(crt);
+    REQUIRE(v->verify_certificate({&net_ca.cert}));
+
+    return std::make_unique<::tls::Cert>(
+      std::move(ca), crt, kp->private_key_pem(), std::nullopt, auth_required);
+  }
+}
+
+TEST_CASE("CA configures trusted certificate store")
+{
+  auto ca = get_ca();
+  ::tls::CA trusted_ca(ca.cert.str(), true);
+  ccf::crypto::OpenSSL::Unique_SSL_CTX ctx(TLS_method());
+
+  trusted_ca.configure_trusted_cert_store(ctx);
+
+  auto* store = SSL_CTX_get_cert_store(ctx);
+  REQUIRE(store != nullptr);
+  auto* params = X509_STORE_get0_param(store);
+  REQUIRE(params != nullptr);
+  REQUIRE(
+    (X509_VERIFY_PARAM_get_flags(params) & X509_V_FLAG_PARTIAL_CHAIN) != 0);
+}
+
+TEST_CASE("Cert configures TLS verification and own certificate")
+{
+  auto ca = get_ca();
+
+  SUBCASE("auth_required requires a peer certificate")
+  {
+    auto cert = get_dummy_cert(ca, "server");
+    ccf::crypto::OpenSSL::Unique_SSL_CTX ctx(TLS_method());
+
+    cert->configure_context(ctx);
+    ccf::crypto::OpenSSL::Unique_SSL ssl(ctx);
+    cert->configure_connection(ssl);
+
+    constexpr auto expected_verify_mode =
+      SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT;
+    REQUIRE(SSL_CTX_get_verify_mode(ctx) == expected_verify_mode);
+    REQUIRE(SSL_get_verify_mode(ssl) == expected_verify_mode);
+    REQUIRE(SSL_CTX_get0_certificate(ctx) != nullptr);
+    REQUIRE(SSL_get_certificate(ssl) != nullptr);
+  }
+
+  SUBCASE("without auth_required a peer certificate is requested, not required")
+  {
+    auto cert = get_dummy_cert(ca, "server", false);
+    ccf::crypto::OpenSSL::Unique_SSL_CTX ctx(TLS_method());
+
+    cert->configure_context(ctx);
+    ccf::crypto::OpenSSL::Unique_SSL ssl(ctx);
+    cert->configure_connection(ssl);
+
+    // The connection inherits the context's verification mode
+    REQUIRE((SSL_get_verify_mode(ssl) & SSL_VERIFY_PEER) != 0);
+    REQUIRE((SSL_get_verify_mode(ssl) & SSL_VERIFY_FAIL_IF_NO_PEER_CERT) == 0);
+  }
+}

src/clients/tls_client.h

@@ -3,9 +3,9 @@
 #pragma once
 
 #include "ccf/crypto/openssl/openssl_wrappers.h"
+#include "clients/tls/ca.h"
+#include "clients/tls/cert.h"
 #include "ds/internal_logger.h"
-#include "tls/ca.h"
-#include "tls/cert.h"
 
 #include <cstdint>
 #include <cstring>