Harden C API and RPC boundaries against fault-injected failures

include/dsn/cpp/serverlet.h

@@ -71,12 +71,17 @@ namespace dsn
 
         void operator () (const TResponse& resp)
         {
-            if (_response != nullptr)
+            if (_response == nullptr)
             {
-                ::dsn::marshall(_response, resp);
-                auto err = dsn_rpc_reply(_response);
-                dassert(err == ERR_OK, "dsn_rpc_reply failed: %s", error_code(err).to_string());
+                derror("rpc_replier got null response");
+                return;
             }
+
+            auto err = ::dsn::try_marshall(_response, resp);
+            dassert(err == ERR_OK, "marshall response failed: %s", err.to_string());
+
+            auto reply_err = dsn_rpc_reply(_response);
+            dassert(reply_err == ERR_OK, "dsn_rpc_reply failed: %s", error_code(reply_err).to_string());
         }
 
         bool is_empty() const
@@ -326,9 +331,12 @@ namespace dsn
     {
         auto msg = dsn_msg_create_response(request);
         dassert(msg != nullptr, "dsn_msg_create_response failed");
-        ::dsn::marshall(msg, resp);
-        auto err = dsn_rpc_reply(msg);
-        dassert(err == ERR_OK, "dsn_rpc_reply failed: %s", error_code(err).to_string());
+
+        auto err = ::dsn::try_marshall(msg, resp);
+        dassert(err == ERR_OK, "marshall response failed: %s", err.to_string());
+
+        auto reply_err = dsn_rpc_reply(msg);
+        dassert(reply_err == ERR_OK, "dsn_rpc_reply failed: %s", error_code(reply_err).to_string());
     }
     /*@}*/
 } // end namespace

include/dsn/cpp/test_utils.h

@@ -56,6 +56,8 @@ DEFINE_TASK_CODE_RPC(RPC_TEST_HASH1, TASK_PRIORITY_COMMON, THREAD_POOL_TEST_SERV
 DEFINE_TASK_CODE_RPC(RPC_TEST_HASH2, TASK_PRIORITY_COMMON, THREAD_POOL_TEST_SERVER)
 DEFINE_TASK_CODE_RPC(RPC_TEST_HASH3, TASK_PRIORITY_COMMON, THREAD_POOL_TEST_SERVER)
 DEFINE_TASK_CODE_RPC(RPC_TEST_HASH4, TASK_PRIORITY_COMMON, THREAD_POOL_TEST_SERVER)
+DEFINE_TASK_CODE_RPC(RPC_TEST_HASH5, TASK_PRIORITY_COMMON, THREAD_POOL_TEST_SERVER)
+DEFINE_TASK_CODE_RPC(RPC_TEST_HASH6, TASK_PRIORITY_COMMON, THREAD_POOL_TEST_SERVER)
 DEFINE_TASK_CODE_RPC(RPC_TEST_STRING_COMMAND, TASK_PRIORITY_COMMON, THREAD_POOL_TEST_SERVER)
 
 DEFINE_TASK_CODE_AIO(LPC_AIO_TEST, TASK_PRIORITY_COMMON, THREAD_POOL_DEFAULT)
@@ -124,6 +126,8 @@ class test_client :
             register_async_rpc_handler(RPC_TEST_HASH2, "rpc.test.hash2", &test_client::on_rpc_test);
             register_async_rpc_handler(RPC_TEST_HASH3, "rpc.test.hash3", &test_client::on_rpc_test);
             register_async_rpc_handler(RPC_TEST_HASH4, "rpc.test.hash4", &test_client::on_rpc_test);
+            register_async_rpc_handler(RPC_TEST_HASH5, "rpc.test.hash5", &test_client::on_rpc_test);
+            register_async_rpc_handler(RPC_TEST_HASH6, "rpc.test.hash6", &test_client::on_rpc_test);
 
             register_rpc_handler(RPC_TEST_STRING_COMMAND, "rpc.test.string.command", &test_client::on_rpc_string_test);
         }

include/dsn/tool-api/message_parser.h

@@ -53,11 +53,14 @@ namespace dsn
             : _buffer_occupied(0), _buffer_block_size(buffer_block_size) {}
         ~message_reader() {}
 
-        // called before read to extend read buffer
+        // called before read to extend read buffer; returns nullptr if the buffer cannot be prepared.
         DSN_API char* read_buffer_ptr(unsigned int read_next);
 
         // get remaining buffer capacity
-        unsigned int read_buffer_capacity() const { return _buffer.length() - _buffer_occupied; }
+        unsigned int read_buffer_capacity() const
+        {
+            return _buffer.length() >= _buffer_occupied ? _buffer.length() - _buffer_occupied : 0;
+        }
 
         // called after read to mark data occupied
         void mark_read(unsigned int read_length) { _buffer_occupied += read_length; }

include/dsn/tool-api/rpc_message.h

@@ -151,11 +151,11 @@ namespace dsn
         //
         // routines for buffer management
         //        
-        DSN_API void write_next(void** ptr, size_t* size, size_t min_size);
-        DSN_API void write_commit(size_t size);
+        DSN_API bool write_next(void** ptr, size_t* size, size_t min_size);
+        DSN_API bool write_commit(size_t size);
         DSN_API void write_append(const blob& data);
         DSN_API bool read_next(void** ptr, size_t* size);
-        DSN_API void read_commit(size_t size);
+        DSN_API bool read_commit(size_t size);
         size_t body_size() { return (size_t)header->body_length; }
         DSN_API void* rw_ptr(size_t offset_begin);
 

include/dsn/utility/factory_store.h

@@ -183,8 +183,6 @@ namespace dsn {
                 fprintf(stderr, "\t\t%s (type: %s)\n", it->c_str(), entry.type == PROVIDER_TYPE_MAIN ? "provider" : "aspect");
             }
             fprintf(stderr, "\tPlease specify the correct factory name in your tool_app or in configuration file\n");
-
-            std::abort();
         }
 
     private:

src/core/src/address.cpp

@@ -64,6 +64,7 @@
 # include <dsn/tool-api/task.h>
 # include "group_address.h"
 # include "uri_address.h"
+# include "c_api_guard.h"
 
 namespace dsn
 {
@@ -99,6 +100,7 @@ static bool net_init()
 // name to ip etc.
 DSN_API uint32_t dsn_ipv4_from_host(const char* name)
 {
+    DSN_C_GUARD_BEGIN
     if ((name == nullptr) || (name[0] == '\0'))
     {
         derror("dsn_ipv4_from_host got null or empty name");
@@ -153,6 +155,7 @@ DSN_API uint32_t dsn_ipv4_from_host(const char* name)
 
     // converts from network byte order to host byte order
     return (uint32_t)ntohl(addr.sin_addr.s_addr);
+    DSN_C_GUARD_END(0)
 }
 
 static bool interface_has_prefix(const char* network_interface, const char* prefix)
@@ -201,6 +204,7 @@ static bool is_default_interface(const struct ifaddrs* ifa)
 // an address that is inconsistent with localhost-based tests.
 DSN_API uint32_t dsn_ipv4_local(const char* network_interface)
 {
+    DSN_C_GUARD_BEGIN
     uint32_t ret = 0;
 
 # if !defined(_WIN32)
@@ -283,6 +287,7 @@ DSN_API uint32_t dsn_ipv4_local(const char* network_interface)
 # endif
 
     return ret;
+    DSN_C_GUARD_END(0)
 }
 
 DSN_API const char*   dsn_address_to_string(dsn_address_t addr)
@@ -393,6 +398,7 @@ DSN_API dsn_address_t dsn_address_build_uri(
 
 DSN_API dsn_group_t dsn_group_build(const char* name) // must be paired with release later
 {
+    DSN_C_GUARD_BEGIN
     if (name == nullptr || name[0] == '\0')
     {
         derror("dsn_group_build got null or empty name");
@@ -401,6 +407,7 @@ DSN_API dsn_group_t dsn_group_build(const char* name) // must be paired with rel
 
     auto g = new ::dsn::rpc_group_address(name);
     return g;
+    DSN_C_GUARD_END(nullptr)
 }
 
 DSN_API int dsn_group_count(dsn_group_t g)
@@ -417,6 +424,7 @@ DSN_API int dsn_group_count(dsn_group_t g)
 
 DSN_API bool dsn_group_add(dsn_group_t g, dsn_address_t ep)
 {
+    DSN_C_GUARD_BEGIN
     if (g == nullptr)
     {
         derror("dsn_group_add got null group");
@@ -426,6 +434,7 @@ DSN_API bool dsn_group_add(dsn_group_t g, dsn_address_t ep)
     auto grp = (::dsn::rpc_group_address*)(g);
     ::dsn::rpc_address addr(ep);
     return grp->add(addr);
+    DSN_C_GUARD_END(false)
 }
 
 DSN_API void dsn_group_set_leader(dsn_group_t g, dsn_address_t ep)
@@ -517,6 +526,7 @@ DSN_API dsn_address_t dsn_group_forward_leader(dsn_group_t g)
 
 DSN_API bool dsn_group_remove(dsn_group_t g, dsn_address_t ep)
 {
+    DSN_C_GUARD_BEGIN
     if (g == nullptr)
     {
         derror("dsn_group_remove got null group");
@@ -526,6 +536,7 @@ DSN_API bool dsn_group_remove(dsn_group_t g, dsn_address_t ep)
     auto grp = (::dsn::rpc_group_address*)(g);
     ::dsn::rpc_address addr(ep);
     return grp->remove(addr);
+    DSN_C_GUARD_END(false)
 }
 
 DSN_API void dsn_group_destroy(dsn_group_t g)
@@ -542,13 +553,15 @@ DSN_API void dsn_group_destroy(dsn_group_t g)
 
 DSN_API dsn_uri_t dsn_uri_build(const char* url) // must be paired with destroy later
 {
+    DSN_C_GUARD_BEGIN
     if (url == nullptr || url[0] == '\0')
     {
         derror("dsn_uri_build got null or empty url");
         return nullptr;
     }
 
     return (dsn_uri_t)new ::dsn::rpc_uri_address(url);
+    DSN_C_GUARD_END(nullptr)
 }
 
 DSN_API void dsn_uri_destroy(dsn_uri_t uri)

src/core/src/app_manager.cpp

@@ -36,9 +36,11 @@
 # include "service_engine.h"
 # include "rpc_engine.h"
 # include <dsn/utility/singleton_store.h>
+# include "c_api_guard.h"
 
 DSN_API bool dsn_register_app(dsn_app* app_type)
 {
+    DSN_C_GUARD_BEGIN
     if (app_type == nullptr)
     {
         derror("dsn_register_app got null app_type");
@@ -72,10 +74,12 @@ DSN_API bool dsn_register_app(dsn_app* app_type)
         delete app;
     }
     return r;
+    DSN_C_GUARD_END(false)
 }
 
 DSN_API bool dsn_get_app_callbacks(const char* name, /* out */ dsn_app_callbacks* callbacks)
 {
+    DSN_C_GUARD_BEGIN
     if (name == nullptr || name[0] == '\0')
     {
         derror("dsn_get_app_callbacks got null or empty name");
@@ -100,6 +104,7 @@ DSN_API bool dsn_get_app_callbacks(const char* name, /* out */ dsn_app_callbacks
         dwarn("application model '%s' is not found, make sure it is registered", name);
         return false;
     }
+    DSN_C_GUARD_END(false)
 }
 
 DSN_API dsn_error_t dsn_hosted_app_create(
@@ -110,6 +115,7 @@ DSN_API dsn_error_t dsn_hosted_app_create(
     /*out*/void** app_context_for_callbacks
     )
 {
+    DSN_C_GUARD_BEGIN
     if (type == nullptr || type[0] == '\0')
     {
         derror("dsn_hosted_app_create got null or empty type");
@@ -152,10 +158,12 @@ DSN_API dsn_error_t dsn_hosted_app_create(
     return node->get_l2_handler()
         .create_app(type, gpid, data_dir, app_context_for_downcalls, app_context_for_callbacks)
         .get();
+    DSN_C_GUARD_END(::dsn::ERR_UNKNOWN.get())
 }
 
 DSN_API dsn_error_t dsn_hosted_app_start(void* app_context, int argc, char** argv)
 {
+    DSN_C_GUARD_BEGIN
     if (app_context == nullptr)
     {
         derror("dsn_hosted_app_start got null app_context");
@@ -191,10 +199,12 @@ DSN_API dsn_error_t dsn_hosted_app_start(void* app_context, int argc, char** arg
     }
 
     return node->get_l2_handler().start_app(app_context, argc, argv).get();
+    DSN_C_GUARD_END(::dsn::ERR_UNKNOWN.get())
 }
 
 DSN_API dsn_error_t dsn_hosted_app_destroy(void* app_context, bool cleanup)
 {
+    DSN_C_GUARD_BEGIN
     if (app_context == nullptr)
     {
         derror("dsn_hosted_app_destroy got null app_context");
@@ -209,10 +219,12 @@ DSN_API dsn_error_t dsn_hosted_app_destroy(void* app_context, bool cleanup)
     }
 
     return node->get_l2_handler().destroy_app(app_context, cleanup).get();
+    DSN_C_GUARD_END(::dsn::ERR_UNKNOWN.get())
 }
 
 DSN_API dsn_error_t dsn_hosted_app_commit_rpc_request(void* app_context, dsn_message_t msg, bool exec_inline)
 {
+    DSN_C_GUARD_BEGIN
     if (app_context == nullptr)
     {
         derror("dsn_hosted_app_commit_rpc_request got null app_context");
@@ -255,6 +267,7 @@ DSN_API dsn_error_t dsn_hosted_app_commit_rpc_request(void* app_context, dsn_mes
     }
 
     return ::dsn::ERR_OK.get();
+    DSN_C_GUARD_END(::dsn::ERR_UNKNOWN.get())
 }
 
 

src/core/src/c_api_guard.h

@@ -0,0 +1,109 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015 Microsoft Corporation
+ * 
+ * -=- Robust Distributed System Nucleus (rDSN) -=- 
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/*
+ * Description:
+ *     no-throw boundary guard for the rDSN C API (DSN_API) entry points.
+ *
+ *     The rDSN public API is a C ABI. A C++ exception that unwinds across the
+ *     C ABI boundary is undefined behavior, so every DSN_API entry point that
+ *     may invoke throwing C++ code (allocation, STL containers, serialization,
+ *     calls into the engines) must translate an escaping exception into the
+ *     API's documented failure value (dsn_error_t / bool / nullptr / 0)
+ *     instead of letting it propagate.
+ *
+ *     Usage:
+ *
+ *         DSN_API dsn_message_t dsn_msg_create_request(...)
+ *         {
+ *             DSN_C_GUARD_BEGIN
+ *             ... body that may throw ...
+ *             return msg;
+ *             DSN_C_GUARD_END(nullptr)   // fail value for this API
+ *         }
+ *
+ *         DSN_API void dsn_some_void_api(...)
+ *         {
+ *             DSN_C_GUARD_BEGIN
+ *             ... body that may throw ...
+ *             DSN_C_GUARD_END_VOID()
+ *         }
+ *
+ *     Two catch clauses are intentional: catch(const std::exception&) reports
+ *     the diagnostic via ex.what(); catch(...) is still required because the C
+ *     ABI boundary must also stop non-std exceptions (e.g. thrown ints, or
+ *     foreign exception objects) from unwinding into the C caller.
+ *
+ *     This guard does NOT catch dsn_coredump()/abort() (e.g. from dassert):
+ *     those are deliberate fatal-invariant aborts, not exceptions, and are
+ *     handled separately by converting recoverable aborts into error returns.
+ *
+ * Revision history:
+ *     2026-xx-xx, first version
+ */
+
+# pragma once
+
+# include <exception>
+# include <dsn/c/api_utilities.h>
+
+// open the guarded region; pair with DSN_C_GUARD_END / DSN_C_GUARD_END_VOID
+# define DSN_C_GUARD_BEGIN try {
+
+// close the guarded region for an API that returns a value; on an escaping
+// C++ exception, logs the diagnostic and returns (fail_ret). The trailing
+// return after the handlers is intentional: it makes (fail_ret) the function's
+// last statement so -Wreturn-type (-Werror) never fires regardless of how the
+// guarded body is structured, and it is the value returned on the exception
+// path.
+# define DSN_C_GUARD_END(fail_ret)                                              \
+    }                                                                           \
+    catch (const std::exception &ex)                                            \
+    {                                                                           \
+        derror("%s: C API call aborted by C++ exception: %s",                   \
+               __FUNCTION__, ex.what());                                        \
+    }                                                                           \
+    catch (...)                                                                 \
+    {                                                                           \
+        derror("%s: C API call aborted by unknown (non-std) C++ exception",     \
+               __FUNCTION__);                                                   \
+    }                                                                           \
+    return (fail_ret);
+
+// close the guarded region for an API that returns void; on an escaping C++
+// exception, logs the diagnostic and returns normally.
+# define DSN_C_GUARD_END_VOID()                                                 \
+    }                                                                           \
+    catch (const std::exception &ex)                                            \
+    {                                                                           \
+        derror("%s: C API call aborted by C++ exception: %s",                   \
+               __FUNCTION__, ex.what());                                        \
+    }                                                                           \
+    catch (...)                                                                 \
+    {                                                                           \
+        derror("%s: C API call aborted by unknown (non-std) C++ exception",     \
+               __FUNCTION__);                                                   \
+    }