From 1e0ad9e6facd7e1e51048b0cb75bffa775aef833 Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 10:37:55 +0800 Subject: [PATCH 01/17] nfs: convert file-copy I/O aborts to error-channel handling The NFS file-copy plugin (used by replica learning) aborted the whole host via dassert on recoverable file I/O failures. Convert these to use the existing error/completion channels, and make the local-write pump iterate instead of recurse so a persistent error cannot grow the stack. nfs_server_impl.cpp: - on_get_file_size (all-files branch): on file_size() failure, set ERR_FILE_OPERATION_FAILED and break instead of aborting; resp.error already carries the failure to the client. - close_file GC: on dsn_file_close failure, dwarn instead of aborting; erase the handle from the map before closing. nfs_client_impl.cpp: - continue_write: on create_directory / dsn_file_open failure, finalize the user request via handle_completion(ERR_FILE_OPERATION_FAILED) instead of aborting. - continue_write: iterate instead of recurse on per-file failures to keep stack depth O(1) under persistent errors. - handle_completion: on dsn_file_close failure, dwarn instead of aborting. --- src/plugins/tools.nfs/nfs_client_impl.cpp | 130 ++++++++++++---------- src/plugins/tools.nfs/nfs_server_impl.cpp | 15 ++- 2 files changed, 83 insertions(+), 62 deletions(-) diff --git a/src/plugins/tools.nfs/nfs_client_impl.cpp b/src/plugins/tools.nfs/nfs_client_impl.cpp index e29bf6471..36da5c70b 100644 --- a/src/plugins/tools.nfs/nfs_client_impl.cpp +++ b/src/plugins/tools.nfs/nfs_client_impl.cpp @@ -279,83 +279,92 @@ namespace dsn { return; } - // get write - dsn::ref_ptr reqc; + // Loop rather than recurse on per-file failures: a persistent error + // (e.g. a full disk that fails every transfer) would otherwise make + // continue_write() invoke itself once per failed request and could + // grow the call stack without bound. while (true) { + // get write + dsn::ref_ptr reqc; + while (true) { - zauto_lock l(_local_writes_lock); - if (!_local_writes.empty()) { - reqc = _local_writes.front(); - _local_writes.pop(); + zauto_lock l(_local_writes_lock); + if (!_local_writes.empty()) + { + reqc = _local_writes.front(); + _local_writes.pop(); + } + else + { + reqc = nullptr; + break; + } } - else + { - reqc = nullptr; - break; - } + zauto_lock l(reqc->lock); + if (reqc->is_valid) + break; + } } + if (nullptr == reqc) { - zauto_lock l(reqc->lock); - if (reqc->is_valid) - break; + --_concurrent_local_write_count; + return; } - } - if (nullptr == reqc) - { - --_concurrent_local_write_count; - return; - } + // real write + std::string file_path = dsn::utils::filesystem::path_combine(reqc->copy_req.dst_dir, reqc->file_ctx->file_name); + std::string path = dsn::utils::filesystem::remove_file_name(file_path.c_str()); + if (!dsn::utils::filesystem::create_directory(path)) + { + derror("create directory %s failed", path.c_str()); + error_code err = ERR_FILE_OPERATION_FAILED; + handle_completion(reqc->file_ctx->user_req, err); + continue; + } - // real write - std::string file_path = dsn::utils::filesystem::path_combine(reqc->copy_req.dst_dir, reqc->file_ctx->file_name); - std::string path = dsn::utils::filesystem::remove_file_name(file_path.c_str()); - if (!dsn::utils::filesystem::create_directory(path)) - { - dassert(false, "Fail to create directory %s.", path.c_str()); - } + dsn_handle_t hfile = reqc->file_ctx->file.load(); + if (!hfile) + { + zauto_lock l(reqc->file_ctx->user_req->user_req_lock); + hfile = reqc->file_ctx->file.load(); + if (!hfile) + { + hfile = dsn_file_open(file_path.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + reqc->file_ctx->file = hfile; + } + } - dsn_handle_t hfile = reqc->file_ctx->file.load(); - if (!hfile) - { - zauto_lock l(reqc->file_ctx->user_req->user_req_lock); - hfile = reqc->file_ctx->file.load(); if (!hfile) { - hfile = dsn_file_open(file_path.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); - reqc->file_ctx->file = hfile; + derror("file open %s failed", file_path.c_str()); + error_code err = ERR_FILE_OPERATION_FAILED; + handle_completion(reqc->file_ctx->user_req, err); + continue; } - } - if (!hfile) - { - derror("file open %s failed", file_path.c_str()); - error_code err = ERR_FILE_OPERATION_FAILED; - handle_completion(reqc->file_ctx->user_req, err); - --_concurrent_local_write_count; - continue_write(); + { + zauto_lock l(reqc->lock); + auto& reqc_save = *reqc.get(); + reqc_save.local_write_task = file::write( + hfile, + reqc_save.response.file_content.data(), + reqc_save.response.size, + reqc_save.response.offset, + LPC_NFS_WRITE, + this, + [this, reqc_cap = std::move(reqc)] (error_code err, int sz) + { + local_write_callback(err, sz, std::move(reqc_cap)); + } + ); + } return; } - - { - zauto_lock l(reqc->lock); - auto& reqc_save = *reqc.get(); - reqc_save.local_write_task = file::write( - hfile, - reqc_save.response.file_content.data(), - reqc_save.response.size, - reqc_save.response.offset, - LPC_NFS_WRITE, - this, - [this, reqc_cap = std::move(reqc)] (error_code err, int sz) - { - local_write_callback(err, sz, std::move(reqc_cap)); - } - ); - } } void nfs_client_impl::local_write_callback(error_code err, size_t sz, dsn::ref_ptr reqc) @@ -443,7 +452,10 @@ namespace dsn { if (f.second->file) { auto err2 = dsn_file_close(f.second->file); - dassert(err2 == ERR_OK, "dsn_file_close failed, err = %s", dsn_error_to_string(err2)); + if (err2 != ERR_OK) + { + dwarn("dsn_file_close failed, err = %s", dsn_error_to_string(err2)); + } f.second->file = nullptr; diff --git a/src/plugins/tools.nfs/nfs_server_impl.cpp b/src/plugins/tools.nfs/nfs_server_impl.cpp index 9aef2a41f..c646fa143 100644 --- a/src/plugins/tools.nfs/nfs_server_impl.cpp +++ b/src/plugins/tools.nfs/nfs_server_impl.cpp @@ -173,7 +173,9 @@ namespace dsn { int64_t sz; if (!dsn::utils::filesystem::file_size(fpath, sz)) { - dassert(false, "Fail to get file size of %s.", fpath.c_str()); + derror("get file size of %s failed", fpath.c_str()); + err = ERR_FILE_OPERATION_FAILED; + break; } resp.size_list.push_back((uint64_t)sz); @@ -222,11 +224,18 @@ namespace dsn { if (fptr->file_access_count == 0 && dsn_now_ms() - fptr->last_access_time > (uint64_t)_opts.file_close_expire_time_ms) { - dinfo("nfs: close file handle %s", it->first.c_str()); + std::string file_name = it->first; + dinfo("nfs: close file handle %s", file_name.c_str()); it = _handles_map.erase(it); ::dsn::error_code err = dsn_file_close(fptr->file_handle); - dassert(err == ERR_OK, "dsn_file_close failed, err = %s", err.to_string()); + if (err != ERR_OK) + { + dwarn("nfs: close file handle %s failed, err = %s", + file_name.c_str(), + err.to_string()); + } + delete fptr; } else From a5a2a7d8a5f9a1612d944a53ebefa87f0d46b2c0 Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 10:57:37 +0800 Subject: [PATCH 02/17] nfs: reject get_file_size response with mismatched list lengths end_get_file_size() iterates resp.size_list but indexes the parallel resp.file_list with the same index. The response is deserialized from a remote, untrusted peer; if it arrives with size_list longer than file_list, resp.file_list[i] is an out-of-bounds std::vector::operator[] read (constructing a std::string from garbage), corrupting memory or crashing the host. Validate that the two parallel arrays have equal length right after the existing resp.error check, and fail the request through the existing nfs_task channel with ERR_INVALID_DATA, matching the adjacent early-return error paths. A well-behaved server fills both lists in lockstep, so normal behavior is unchanged. --- src/plugins/tools.nfs/nfs_client_impl.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/plugins/tools.nfs/nfs_client_impl.cpp b/src/plugins/tools.nfs/nfs_client_impl.cpp index 36da5c70b..afbbeb77d 100644 --- a/src/plugins/tools.nfs/nfs_client_impl.cpp +++ b/src/plugins/tools.nfs/nfs_client_impl.cpp @@ -93,6 +93,22 @@ namespace dsn { return; } + // size_list and file_list are parallel arrays in the (remote, untrusted) + // response, but the loop below iterates size_list while indexing file_list. + // A malformed or corrupted response with mismatched lengths would otherwise + // read file_list out of bounds. Validate the lengths match and fail the + // request through the existing nfs_task error channel. + if (resp.size_list.size() != resp.file_list.size()) + { + derror("invalid get file size response: size_list (%d) and file_list (%d) " + "have mismatched lengths", + (int)resp.size_list.size(), + (int)resp.file_list.size()); + ureq->nfs_task->enqueue(ERR_INVALID_DATA, 0); + delete ureq; + return; + } + for (size_t i = 0; i < resp.size_list.size(); i++) // file list { file_context *filec; From 10023c9e6e51afcf51a57a6979bbef3617f8df2b Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 11:26:45 +0800 Subject: [PATCH 03/17] core: stop directory-walk abort on recoverable nftw failures remove_directory()'s nftw callback aborted the whole process via dassert when nftw reported a typeflag other than FTW_F/FTW_DP. nftw delivers FTW_NS when a per-entry stat() fails (transient I/O error, permission problem, or a concurrent removal race) and FTW_DNR for an unreadable directory; both are recoverable and remove_directory()/remove_path() already return a bool. Replace the dassert with a dwarn + FTW_STOP so the walk stops and the caller returns false instead of crashing the host. The Windows custom walk only ever passes FTW_F/FTW_DP, so this is a no-op there. --- src/dev/cpp/file_utils.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/dev/cpp/file_utils.cpp b/src/dev/cpp/file_utils.cpp index 23ce9135b..03a89ad0a 100644 --- a/src/dev/cpp/file_utils.cpp +++ b/src/dev/cpp/file_utils.cpp @@ -542,8 +542,18 @@ namespace dsn { { bool succ; - dassert((typeflag == FTW_F) || (typeflag == FTW_DP), - "Invalid typeflag = %d.", typeflag); + // nftw reports FTW_NS when stat() fails on an entry (a transient + // I/O error, a permission problem, or a concurrent removal) and + // FTW_DNR for an unreadable directory. These are recoverable, so + // stop the walk and let remove_directory()/remove_path() return + // false through their existing bool channel instead of aborting + // the whole process. + if ((typeflag != FTW_F) && (typeflag != FTW_DP)) + { + dwarn("remove path %s failed: unexpected file tree walk typeflag = %d", + fpath, typeflag); + return FTW_STOP; + } #ifdef _WIN32 if (typeflag != FTW_F) { From 7dcf8dc083bf89bd41e23b7700ae20f6d030e0b4 Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 11:45:38 +0800 Subject: [PATCH 04/17] nfs: allow zero-byte files in copy request size validation on_copy rejected request.size <= 0, but size == 0 is legitimate: the client emits exactly one copy request per file, and an empty source file yields a single request with size 0. Empty files are listed by on_get_file_size without filtering, so any directory containing even one empty file failed the entire multi-file transfer / replica-learn with ERR_INVALID_PARAMETERS. Reject only request.size < 0 so a zero length flows through the original open + zero-length file::read path (restoring pre-hardening behavior) while keeping the oversized-request heap-overflow guard intact. --- src/plugins/tools.nfs/nfs_server_impl.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/plugins/tools.nfs/nfs_server_impl.cpp b/src/plugins/tools.nfs/nfs_server_impl.cpp index c646fa143..0e2c68666 100644 --- a/src/plugins/tools.nfs/nfs_server_impl.cpp +++ b/src/plugins/tools.nfs/nfs_server_impl.cpp @@ -44,11 +44,14 @@ namespace dsn { //dinfo(">>> on call RPC_COPY end, exec RPC_NFS_COPY"); // request.size is supplied by the (untrusted) client but the read below targets a - // buffer of exactly nfs_copy_block_bytes. Reject out-of-range sizes to avoid a heap - // buffer overflow when reading the file content into the fixed-size block buffer. - if (request.size <= 0 || static_cast(request.size) > _opts.nfs_copy_block_bytes) + // buffer of exactly nfs_copy_block_bytes. Reject negative or oversized lengths to + // avoid a heap buffer overflow when reading the file content into the fixed-size + // block buffer. A zero length is legal and must be preserved: an empty source file + // produces a single zero-byte copy request, which has to succeed so the destination + // empty file still gets created (otherwise one empty file fails the whole transfer). + if (request.size < 0 || static_cast(request.size) > _opts.nfs_copy_block_bytes) { - derror("nfs: invalid copy request size %d (allowed range is (0, %u]) for file %s", + derror("nfs: invalid copy request size %d (allowed range is [0, %u]) for file %s", request.size, _opts.nfs_copy_block_bytes, request.file_name.c_str()); ::dsn::service::copy_response resp; resp.error = ERR_INVALID_PARAMETERS; From 1df74a753b1d24bd2271a638e4a913710cf18ba2 Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 14:12:40 +0800 Subject: [PATCH 05/17] core: convert disk-engine short-write abort to error-channel handling disk_file::on_write_completed asserted that the provider always wrote at least the requested number of bytes, aborting the process on a short write (which can happen on a full disk). Report ERR_FILE_OPERATION_FAILED for the affected task and the remaining batched tasks through the aio callback channel instead, and stop consuming the (now invalid) written size so the trailing size==0 invariant still holds. --- src/core/src/disk_engine.cpp | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/core/src/disk_engine.cpp b/src/core/src/disk_engine.cpp index 9b3bf0ef5..4f0e5f752 100644 --- a/src/core/src/disk_engine.cpp +++ b/src/core/src/disk_engine.cpp @@ -147,14 +147,25 @@ aio_task* disk_file::on_write_completed(aio_task* wk, void* ctx, error_code err, if (err == ERR_OK) { size_t this_size = (size_t)wk->aio()->buffer_size; - dassert(size >= this_size, - "written buffer size does not equal to input buffer's size: %d vs %d", - (int)size, - (int)this_size - ); - - wk->enqueue(err, this_size); - size -= this_size; + if (size < this_size) + { + // Short write from the underlying provider (e.g. the disk is full): + // fewer bytes were written than this task requested. Report a + // file-operation failure for this task and the remaining batched + // tasks (whose data was not written either) through the aio callback + // channel instead of aborting the whole process. + derror("disk write completed with fewer bytes than requested: %d vs %d", + (int)size, + (int)this_size + ); + wk->enqueue(ERR_FILE_OPERATION_FAILED, size); + size = 0; + } + else + { + wk->enqueue(err, this_size); + size -= this_size; + } } else { From 72eed3940fbef89b204174d31f1baeec9c985439 Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 14:12:45 +0800 Subject: [PATCH 06/17] core: avoid io_destroy abort in native aio provider destructor ~native_linux_aio_provider asserted io_destroy() returned 0, which aborts during shutdown if the kernel AIO context was already released or is otherwise invalid (e.g. EINVAL). A destructor must not abort: log the failure with derror (matching the other I/O error sites in this file) and let shutdown proceed. --- src/plugins/tools.common/native_aio_provider.linux.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/plugins/tools.common/native_aio_provider.linux.cpp b/src/plugins/tools.common/native_aio_provider.linux.cpp index a1c7846dc..15f415e1e 100644 --- a/src/plugins/tools.common/native_aio_provider.linux.cpp +++ b/src/plugins/tools.common/native_aio_provider.linux.cpp @@ -59,7 +59,12 @@ namespace dsn { native_linux_aio_provider::~native_linux_aio_provider() { auto ret = io_destroy(_ctx); - dassert(ret == 0, "io_destroy error, ret = %d", ret); + if (ret != 0) + { + // A destructor must not abort. io_destroy can fail (e.g. EINVAL on an + // already-released context); log and continue so shutdown proceeds. + derror("io_destroy error, ret = %d", ret); + } } void native_linux_aio_provider::start(io_modifer& ctx) From f1ccc6e5047d58b34275ca20b4ccfe25748dd834 Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 14:12:56 +0800 Subject: [PATCH 07/17] nfs: fix empty-file copy EOF failure and short-read uninitialized data Empty (0-byte) source files made the whole file transfer fail. A zero-length aio read/write completes with ERR_HANDLE_EOF (not ERR_OK), so the single size==0 copy request an empty file produces was reported as a failure -- the prior size-guard relaxation alone was not enough. Fix both sides to avoid issuing zero-length aio: the server short-circuits a size==0 copy request with an empty ERR_OK reply (no open/read), and the client finalizes the segment inline as success (the destination is already created by its O_CREAT open) via the loop, not recursion. Short reads also shipped uninitialized memory: internal_read_callback ignored the actual bytes read and set resp.size to the requested size while sending the full make_shared_array block buffer, so on a short read the client wrote uninitialized heap into the destination -- file corruption and a heap-memory disclosure into replicated data. Fail the copy block via ERR_FILE_OPERATION_FAILED when fewer bytes than requested are read. Add core.aio_zero_length pinning the zero-length read/write == ERR_HANDLE_EOF contract these fixes depend on. --- src/core/src/aio.test.cpp | 43 +++++++++++++++++++++++ src/plugins/tools.nfs/nfs_client_impl.cpp | 30 ++++++++++++++++ src/plugins/tools.nfs/nfs_server_impl.cpp | 31 ++++++++++++++++ 3 files changed, 104 insertions(+) diff --git a/src/core/src/aio.test.cpp b/src/core/src/aio.test.cpp index 6c00e9165..6c5fcb0c9 100644 --- a/src/core/src/aio.test.cpp +++ b/src/core/src/aio.test.cpp @@ -253,3 +253,46 @@ TEST(core, operation_failed) EXPECT_TRUE(utils::filesystem::remove_path(tmp_file)); } + +// Pins the zero-length aio contract that the NFS file-copy plugin depends on: +// a zero-length write/read completes with ERR_HANDLE_EOF (not ERR_OK) and count 0. +// Because of this, nfs_service_impl::on_copy / nfs_client_impl::continue_write must +// special-case empty (size==0) files instead of issuing a zero-length read/write, +// which would otherwise be reported to the copy caller as a failure. +TEST(core, aio_zero_length) +{ + // if in dsn_mimic_app() and disk_io_mode == IOE_PER_QUEUE + if (task::get_current_disk() == nullptr) return; + + ASSERT_TRUE(::dsn::utils::test::prepare_test_tmp_dir("dsn.core.aio_zero")); + const std::string tmp_file = + ::dsn::utils::test::test_tmp_path("dsn.core.aio_zero", "zero_test_file"); + + ::dsn::error_code werr; + size_t wcount = 0xdead; + ::dsn::error_code rerr; + size_t rcount = 0xdead; + char buffer[8] = {}; + + // zero-length write to a freshly created file + auto fp = dsn_file_open(tmp_file.c_str(), O_RDWR | O_CREAT | O_BINARY, 0666); + ASSERT_TRUE(fp != nullptr); + auto t = ::dsn::file::write(fp, buffer, 0, 0, LPC_AIO_TEST, nullptr, + [&werr, &wcount](::dsn::error_code e, size_t n) { werr = e; wcount = n; }, 0); + t->wait(); + EXPECT_TRUE(werr == ERR_HANDLE_EOF); + EXPECT_EQ(0u, wcount); + dsn_file_close(fp); + + // zero-length read from the (now empty) file + auto fp2 = dsn_file_open(tmp_file.c_str(), O_RDONLY | O_BINARY, 0); + ASSERT_TRUE(fp2 != nullptr); + t = ::dsn::file::read(fp2, buffer, 0, 0, LPC_AIO_TEST, nullptr, + [&rerr, &rcount](::dsn::error_code e, size_t n) { rerr = e; rcount = n; }, 0); + t->wait(); + EXPECT_TRUE(rerr == ERR_HANDLE_EOF); + EXPECT_EQ(0u, rcount); + dsn_file_close(fp2); + + EXPECT_TRUE(utils::filesystem::remove_path(tmp_file)); +} diff --git a/src/plugins/tools.nfs/nfs_client_impl.cpp b/src/plugins/tools.nfs/nfs_client_impl.cpp index afbbeb77d..e28e4c910 100644 --- a/src/plugins/tools.nfs/nfs_client_impl.cpp +++ b/src/plugins/tools.nfs/nfs_client_impl.cpp @@ -363,6 +363,36 @@ namespace dsn { continue; } + // An empty source file yields a copy request with response.size == 0. The + // destination file has just been created (or already exists) via the + // O_RDWR | O_CREAT open above, so there is nothing to write: a zero-length + // file::write would complete with ERR_HANDLE_EOF and be reported as a + // failure. Finalize this segment as a success inline -- mirroring the + // success bookkeeping in local_write_callback -- and continue the loop + // (no recursion) instead of issuing the zero-length write. + if (reqc->response.size == 0) + { + reqc->response.file_content = blob(); + + bool completed = false; + { + zauto_lock l(reqc->file_ctx->user_req->user_req_lock); + if (++reqc->file_ctx->finished_segments == (int)reqc->file_ctx->copy_requests.size()) + { + if (++reqc->file_ctx->user_req->finished_files == (int)reqc->file_ctx->user_req->file_context_map.size()) + { + completed = true; + } + } + } + + if (completed) + { + handle_completion(reqc->file_ctx->user_req, ERR_OK); + } + continue; + } + { zauto_lock l(reqc->lock); auto& reqc_save = *reqc.get(); diff --git a/src/plugins/tools.nfs/nfs_server_impl.cpp b/src/plugins/tools.nfs/nfs_server_impl.cpp index 0e2c68666..670071a53 100644 --- a/src/plugins/tools.nfs/nfs_server_impl.cpp +++ b/src/plugins/tools.nfs/nfs_server_impl.cpp @@ -59,6 +59,22 @@ namespace dsn { return; } + // An empty source file produces exactly one zero-byte copy request. A + // zero-length file::read completes with ERR_HANDLE_EOF (not ERR_OK), which the + // client treats as a copy failure, so the destination empty file would never + // be created and one empty file would fail the whole transfer. There is no + // content to ship, so reply success directly without opening or reading the + // file; the client creates the empty destination on its own O_CREAT open. + if (request.size == 0) + { + ::dsn::service::copy_response resp; + resp.error = ERR_OK; + resp.offset = request.offset; + resp.size = 0; + reply(resp); + return; + } + std::string file_path = dsn::utils::filesystem::path_combine(request.source_dir, request.file_name); dsn_handle_t hfile; @@ -137,6 +153,21 @@ namespace dsn { } } + // A short read (ERR_OK but fewer bytes than requested) means the source file + // no longer holds the full requested range -- it was truncated, raced with a + // writer, or is corrupt. cp.bb is allocated with make_shared_array, i.e. + // uninitialized heap, and is sent in full with resp.size = cp.size (the + // requested size). Reporting success here would make the client write cp.size + // bytes whose tail [sz, cp.size) was never filled by the read, corrupting the + // destination file and disclosing uninitialized heap memory into the replicated + // data. Fail this copy block through the existing response error channel instead. + if (err == ERR_OK && sz != cp.size) + { + derror("nfs: short read on file %s at offset %" PRId64 ": read %u of %u bytes", + cp.file_path.c_str(), cp.offset, (uint32_t)sz, (uint32_t)cp.size); + err = ERR_FILE_OPERATION_FAILED; + } + ::dsn::service::copy_response resp; resp.error = err; resp.file_content = cp.bb; From 5dc32fb94a5f126a0181ee80b1bfcad21ac5a073 Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 14:13:01 +0800 Subject: [PATCH 08/17] Sync dist service submodule Point rDSN.dist.service at the error-handling-follow-up branch (bf631e7), which hardens replica log replay and checkpoint copy against corrupt/untrusted input. --- src/plugins_ext/rDSN.dist.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins_ext/rDSN.dist.service b/src/plugins_ext/rDSN.dist.service index 82ec2cd5b..bf631e7ad 160000 --- a/src/plugins_ext/rDSN.dist.service +++ b/src/plugins_ext/rDSN.dist.service @@ -1 +1 @@ -Subproject commit 82ec2cd5b43120837f438edee286371ae0bb94c0 +Subproject commit bf631e7adccbbb92187045dab660587922149534 From ab03183548f401ec338f051dd3a144eb6bc17b3e Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 14:45:22 +0800 Subject: [PATCH 09/17] Sync dist service submodule (harden log_file::open_read file_size) Advance the rDSN.dist.service submodule pointer from bf631e7 to 9f490ec, which converts the log_file read-mode constructor's file_size() dassert abort into a graceful ERR_FILE_OPERATION_FAILED rejection via the existing log_file::open_read() error channel. --- src/plugins_ext/rDSN.dist.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins_ext/rDSN.dist.service b/src/plugins_ext/rDSN.dist.service index bf631e7ad..9f490ecf6 160000 --- a/src/plugins_ext/rDSN.dist.service +++ b/src/plugins_ext/rDSN.dist.service @@ -1 +1 @@ -Subproject commit bf631e7adccbbb92187045dab660587922149534 +Subproject commit 9f490ecf61715ce8cf0190223d2ba7862e322c1b From 1032bd63c1aa0471fceeaa55e63e9257a019e1d2 Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 15:44:54 +0800 Subject: [PATCH 10/17] Sync dist service submodule (harden mutation_log::mark_new_offset) Point rDSN.dist.service to 18ad52f, which converts the abort-on-recoverable- log-file-creation-failure in mutation_log::mark_new_offset into the existing _io_error_callback failover path and guards write_pending_mutations against a null current log file. --- src/plugins_ext/rDSN.dist.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins_ext/rDSN.dist.service b/src/plugins_ext/rDSN.dist.service index 9f490ecf6..18ad52f34 160000 --- a/src/plugins_ext/rDSN.dist.service +++ b/src/plugins_ext/rDSN.dist.service @@ -1 +1 @@ -Subproject commit 9f490ecf61715ce8cf0190223d2ba7862e322c1b +Subproject commit 18ad52f34a2a8d5feec4775a3415eda4d495f780 From 1266f4864a1246899ead27ac62f54bfaf85d5dd4 Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 18:22:34 +0800 Subject: [PATCH 11/17] Sync dist service submodule (harden log_file::flush/close) Advance the rDSN.dist.service pointer to 9d2d9dc, which makes log_file::flush() and log_file::close() report recoverable disk-syscall failures through their existing error channels instead of aborting the replica server via dassert. --- src/plugins_ext/rDSN.dist.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins_ext/rDSN.dist.service b/src/plugins_ext/rDSN.dist.service index 18ad52f34..9d2d9dc8b 160000 --- a/src/plugins_ext/rDSN.dist.service +++ b/src/plugins_ext/rDSN.dist.service @@ -1 +1 @@ -Subproject commit 18ad52f34a2a8d5feec4775a3415eda4d495f780 +Subproject commit 9d2d9dc8bb3e474f50bc1ff1c6537377984d9475 From e7edd194f12be0bc6481e420cd14b7764b9b7173 Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 19:18:46 +0800 Subject: [PATCH 12/17] Sync dist service submodule (mutation::read_from returns nullptr) Point to rDSN.dist.service b93df50, which makes mutation::read_from return nullptr instead of throwing and converts its callers to nullptr checks. --- src/plugins_ext/rDSN.dist.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins_ext/rDSN.dist.service b/src/plugins_ext/rDSN.dist.service index 9d2d9dc8b..b93df5076 160000 --- a/src/plugins_ext/rDSN.dist.service +++ b/src/plugins_ext/rDSN.dist.service @@ -1 +1 @@ -Subproject commit 9d2d9dc8bb3e474f50bc1ff1c6537377984d9475 +Subproject commit b93df50769a4e4f08e79a80b720f12d5292fbb5e From a3c25cf5a08d008413b0ac1da338d579487b150a Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 19:43:03 +0800 Subject: [PATCH 13/17] Sync dist service submodule (harden restore_from_local_storage) --- src/plugins_ext/rDSN.dist.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins_ext/rDSN.dist.service b/src/plugins_ext/rDSN.dist.service index b93df5076..9f9e82dc2 160000 --- a/src/plugins_ext/rDSN.dist.service +++ b/src/plugins_ext/rDSN.dist.service @@ -1 +1 @@ -Subproject commit b93df50769a4e4f08e79a80b720f12d5292fbb5e +Subproject commit 9f9e82dc2b208c44cd26dc4ed19d3a6506d1c213 From 905c8c0c0de67c215fa67c71f5da456300894460 Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 19:54:36 +0800 Subject: [PATCH 14/17] Validate NFS copy_response.size against file_content length on client The NFS client issued file::write(response.file_content.data(), response.size, ...) using the size and buffer from the remote server's copy_response without cross-checking them. A response whose declared size is negative or larger than the actual file_content buffer would read past the end of that buffer, corrupting the destination file with adjacent heap memory (disclosed into the replicated data) or crashing. This is the client-side counterpart of the size validation on_copy already performs on the request. end_copy now rejects such responses through the existing handle_completion(ERR_INVALID_DATA) channel before the local write. --- src/plugins/tools.nfs/nfs_client_impl.cpp | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/plugins/tools.nfs/nfs_client_impl.cpp b/src/plugins/tools.nfs/nfs_client_impl.cpp index e28e4c910..6722e1e2e 100644 --- a/src/plugins/tools.nfs/nfs_client_impl.cpp +++ b/src/plugins/tools.nfs/nfs_client_impl.cpp @@ -250,7 +250,28 @@ namespace dsn { handle_completion(reqc->file_ctx->user_req, err); return; } - + + // response.size and response.file_content are independent fields filled in by the + // (untrusted) remote server. The local write issues + // file::write(response.file_content.data(), response.size, ...), so a response whose + // declared size is negative or larger than the actual file_content buffer would make + // that write read past the end of the buffer -- corrupting the destination file with + // adjacent heap memory (and disclosing it into the replicated data) or crashing. This + // is the client-side counterpart of the size validation on_copy already performs on + // the request. A zero size is legal (empty-file segment) and is handled specially in + // continue_write. On the honest path file_content is the full block buffer and size is + // the valid prefix, so size <= file_content.length() always holds. + if (resp.size < 0 || static_cast(resp.size) > resp.file_content.length()) + { + derror("nfs: invalid copy response for file %s: declared size %d exceeds " + "content buffer length %u", + reqc->file_ctx->file_name.c_str(), + resp.size, + (uint32_t)resp.file_content.length()); + handle_completion(reqc->file_ctx->user_req, ERR_INVALID_DATA); + return; + } + reqc->response = resp; reqc->response.error.end_tracking(); // always ERR_OK reqc->is_ready_for_write = true; From 314d5c2c0f60f6e3323c2dcc3603b523df62138f Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 20:41:21 +0800 Subject: [PATCH 15/17] Sync dist service submodule (harden dump_file::read_next_buffer) --- src/plugins_ext/rDSN.dist.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins_ext/rDSN.dist.service b/src/plugins_ext/rDSN.dist.service index 9f9e82dc2..264cba18a 160000 --- a/src/plugins_ext/rDSN.dist.service +++ b/src/plugins_ext/rDSN.dist.service @@ -1 +1 @@ -Subproject commit 9f9e82dc2b208c44cd26dc4ed19d3a6506d1c213 +Subproject commit 264cba18a4cae2cf42d51ce1d7072816d9210612 From c4aee3b7682a23435f7b1837ad90535be4d3b0b8 Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 20:41:21 +0800 Subject: [PATCH 16/17] Validate source_dir prefix length in NFS on_get_file_size on_get_file_size returned each file path relative to source_dir via path.substr(request.source_dir.length()), using the raw untrusted source_dir length as the position into a path that get_subfiles/path_combine had already normalized (consecutive separators collapsed, trailing separator stripped). A peer sending a non-normalized source_dir (e.g. with redundant slashes) makes the normalized path shorter than the prefix length, so substr() throws std::out_of_range. The RPC dispatch chain has no try/catch, so the exception unwinds off the worker thread and aborts the NFS service, which runs on every replica node. Guard both loops: if request.source_dir.length() exceeds the path length the source_dir cannot be a valid prefix, so reject the request with ERR_INVALID_PARAMETERS instead of throwing. The guard only triggers for a non-normalized source_dir; valid requests are unaffected. --- src/plugins/tools.nfs/nfs_server_impl.cpp | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/plugins/tools.nfs/nfs_server_impl.cpp b/src/plugins/tools.nfs/nfs_server_impl.cpp index 670071a53..7beb3309b 100644 --- a/src/plugins/tools.nfs/nfs_server_impl.cpp +++ b/src/plugins/tools.nfs/nfs_server_impl.cpp @@ -212,6 +212,20 @@ namespace dsn { break; } + // fpath is built from the normalized source folder, while the + // strip length below uses the raw (untrusted) request.source_dir. + // A non-normalized source_dir (e.g. redundant or trailing '/') + // from the peer can be longer than the normalized fpath, which + // would make substr() throw std::out_of_range and abort the + // server. Reject such a request instead of crashing. + if (request.source_dir.length() > fpath.length()) + { + derror("invalid source_dir(%s) does not prefix file(%s)", + request.source_dir.c_str(), fpath.c_str()); + err = ERR_INVALID_PARAMETERS; + break; + } + resp.size_list.push_back((uint64_t)sz); resp.file_list.push_back(fpath.substr(request.source_dir.length(), fpath.length() - 1)); } @@ -237,6 +251,18 @@ namespace dsn { // Done uint64_t size = st.st_size; + // path_combine normalizes file_path, which may be shorter than the + // raw (untrusted) request.source_dir; using source_dir.length() as the + // substr position would throw std::out_of_range and abort the server on + // a non-normalized source_dir from the peer. Reject it instead. + if (request.source_dir.length() > file_path.length()) + { + derror("invalid source_dir(%s) does not prefix file(%s)", + request.source_dir.c_str(), file_path.c_str()); + err = ERR_INVALID_PARAMETERS; + break; + } + resp.size_list.push_back(size); resp.file_list.push_back(file_path.substr(request.source_dir.length())); } From 7f885c1dd6fcbb96796e50623a598b50d8330084 Mon Sep 17 00:00:00 2001 From: HX Lin <> Date: Tue, 30 Jun 2026 20:51:55 +0800 Subject: [PATCH 17/17] Sync dist service submodule to master (error-handling hardening merged) The error-handling-follow-up changes in the rDSN.dist.service submodule were squash-merged into its master branch (#30). Advance the submodule pointer from the now-deleted branch commit to the merged master commit. --- src/plugins_ext/rDSN.dist.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins_ext/rDSN.dist.service b/src/plugins_ext/rDSN.dist.service index 264cba18a..95694f062 160000 --- a/src/plugins_ext/rDSN.dist.service +++ b/src/plugins_ext/rDSN.dist.service @@ -1 +1 @@ -Subproject commit 264cba18a4cae2cf42d51ce1d7072816d9210612 +Subproject commit 95694f062a1ea03aba5368780265ca8ace53806a