From 4e624bbea339fbc489c166dd4e4f330a0964ad26 Mon Sep 17 00:00:00 2001 From: meiyi Date: Tue, 15 Sep 2026 17:17:39 +0800 Subject: [PATCH 1/6] [refactor](cloud) Share packed slice metadata collection ### What problem does this PR solve? Problem Summary: Packed slice collection depends only on a closed file writer, its logical path, and the destination rowset metadata, but was a private method of CloudRowsetWriter. Move it to RowsetMeta::collect_packed_slice_location and reuse it for existing Cloud segment and index collection without changing collection timing. Preserve skipping unfinished and non-packed files. Require a nonempty packed object path after a successful packed-location lookup, replacing the previous silent skip with an invariant assertion. Move the associated RowsetMetaPackedFileTest coverage with this refactor: completed data/index files, logical paths distinct from writer URIs, repeated collection, serialization, non-packed files, and missing mappings. Memtable forwarding call sites and MOW collection timing changes remain in subsequent commits. During rebase, retain the upstream writer-owned packed slice lookup so global index cleanup cannot invalidate metadata collection. Adapt the unit coverage to real PackedFileWriter handles, including incomplete writers, direct files, serialization, repeated collection, and missing-handle invariant failures. ### Release note None ### Check List (For Author) - Test: Unit tests adapted but not run, per request. No build or service execution. Conflict-resolution files passed clang-format 16, format checks, build hygiene, and Git whitespace checks. - Behavior changed: Yes; an empty packed object path after successful lookup is an invariant failure instead of a silent skip - Does this need documentation: No --- be/src/cloud/cloud_rowset_writer.cpp | 41 +-------- be/src/cloud/cloud_rowset_writer.h | 2 - be/src/storage/rowset/rowset_meta.cpp | 22 +++++ be/src/storage/rowset/rowset_meta.h | 5 ++ be/test/storage/rowset/rowset_meta_test.cpp | 96 +++++++++++++++++++++ 5 files changed, 127 insertions(+), 39 deletions(-) diff --git a/be/src/cloud/cloud_rowset_writer.cpp b/be/src/cloud/cloud_rowset_writer.cpp index 9753ca3dd99cd9..330527329d5bd6 100644 --- a/be/src/cloud/cloud_rowset_writer.cpp +++ b/be/src/cloud/cloud_rowset_writer.cpp @@ -20,8 +20,7 @@ #include "common/logging.h" #include "common/status.h" #include "io/cache/block_file_cache_factory.h" -#include "io/fs/packed_file_manager.h" -#include "io/fs/packed_file_writer.h" +#include "io/fs/file_writer.h" #include "storage/rowset/rowset_factory.h" namespace doris { @@ -187,8 +186,7 @@ Status CloudRowsetWriter::_collect_all_packed_slice_locations(RowsetMeta* rowset const auto& file_writers = _seg_files.get_file_writers(); for (const auto& [seg_id, writer_ptr] : file_writers) { auto segment_path = _context.segment_path(seg_id); - RETURN_IF_ERROR( - _collect_packed_slice_location(writer_ptr.get(), segment_path, rowset_meta)); + RETURN_IF_ERROR(rowset_meta->collect_packed_slice_location(*writer_ptr, segment_path)); } // Collect inverted index file packed indices @@ -200,43 +198,12 @@ Status CloudRowsetWriter::_collect_all_packed_slice_locations(RowsetMeta* rowset InvertedIndexDescriptor::get_index_file_path_prefix(segment_path); std::string index_path = InvertedIndexDescriptor::get_index_file_path_v2(std::string(index_prefix_view)); - RETURN_IF_ERROR(_collect_packed_slice_location(idx_writer_ptr->get_file_writer(), - index_path, rowset_meta)); + RETURN_IF_ERROR(rowset_meta->collect_packed_slice_location( + *idx_writer_ptr->get_file_writer(), index_path)); } } return Status::OK(); } -Status CloudRowsetWriter::_collect_packed_slice_location(io::FileWriter* file_writer, - const std::string& file_path, - RowsetMeta* rowset_meta) { - VLOG_NOTICE << "collect packed slice location for file: " << file_path; - // Check if file writer is closed - if (file_writer->state() != io::FileWriter::State::CLOSED) { - // Writer is still open; index will be collected after it is closed. - return Status::OK(); - } - - // Check if file is actually in packed file (not direct write for large files) - if (!file_writer->is_in_packed_file()) { - return Status::OK(); - } - - // Ask the writer, which holds a reference to its own slice location. Looking it up by - // path in PackedFileManager would race with the retention based cleanup of the index. - io::PackedSliceLocation index; - RETURN_IF_ERROR( - static_cast(file_writer)->get_packed_slice_location(&index)); - if (index.packed_file_path.empty()) { - return Status::OK(); // File not in packed file, skip - } - - rowset_meta->add_packed_slice_location(file_path, index.packed_file_path, index.offset, - index.size, index.packed_file_size); - LOG(INFO) << "collect packed file index: " << file_path << " -> " << index.packed_file_path - << ", offset: " << index.offset << ", size: " << index.size; - return Status::OK(); -} - } // namespace doris diff --git a/be/src/cloud/cloud_rowset_writer.h b/be/src/cloud/cloud_rowset_writer.h index 4841fbf6cf7132..e351405f738d06 100644 --- a/be/src/cloud/cloud_rowset_writer.h +++ b/be/src/cloud/cloud_rowset_writer.h @@ -38,8 +38,6 @@ class CloudRowsetWriter : public BaseBetaRowsetWriter { Status _collect_all_packed_slice_locations(RowsetMeta* rowset_meta); - Status _collect_packed_slice_location(io::FileWriter* file_writer, const std::string& file_path, - RowsetMeta* rowset_meta); CloudStorageEngine& _engine; }; diff --git a/be/src/storage/rowset/rowset_meta.cpp b/be/src/storage/rowset/rowset_meta.cpp index 05ca4be46b7b10..123813c30c11d3 100644 --- a/be/src/storage/rowset/rowset_meta.cpp +++ b/be/src/storage/rowset/rowset_meta.cpp @@ -37,6 +37,7 @@ #include "io/fs/local_file_system.h" #include "io/fs/packed_file_manager.h" #include "io/fs/packed_file_system.h" +#include "io/fs/packed_file_writer.h" #include "json2pb/json_to_pb.h" #include "json2pb/pb_to_json.h" #include "runtime/exec_env.h" @@ -56,6 +57,27 @@ RowsetMeta::~RowsetMeta() { } } +Status RowsetMeta::collect_packed_slice_location(const io::FileWriter& file_writer, + const std::string& file_path) { + if (file_writer.state() != io::FileWriter::State::CLOSED) { + return Status::OK(); + } + if (!file_writer.is_in_packed_file()) { + return Status::OK(); + } + + // Read the writer-owned location so collection cannot race with global index cleanup. + io::PackedSliceLocation location; + RETURN_IF_ERROR(static_cast(file_writer) + .get_packed_slice_location(&location)); + DORIS_CHECK(!location.packed_file_path.empty()); + add_packed_slice_location(file_path, location.packed_file_path, location.offset, location.size, + location.packed_file_size); + LOG(INFO) << "collect packed file index: " << file_path << " -> " << location.packed_file_path + << ", offset: " << location.offset << ", size: " << location.size; + return Status::OK(); +} + bool RowsetMeta::init(std::string_view pb_rowset_meta) { bool ret = _deserialize_from_pb(pb_rowset_meta); if (!ret) { diff --git a/be/src/storage/rowset/rowset_meta.h b/be/src/storage/rowset/rowset_meta.h index 9255cdb9a09d08..d140e4d891e2f6 100644 --- a/be/src/storage/rowset/rowset_meta.h +++ b/be/src/storage/rowset/rowset_meta.h @@ -535,6 +535,11 @@ class RowsetMeta : public MetadataAdder { RowsetMeta(const RowsetMeta&) = delete; RowsetMeta operator=(const RowsetMeta&) = delete; + // Collect only completed files; interim rowsets may still have open writers. + // file_path is the logical path used at creation, not the S3 writer's absolute path. + Status collect_packed_slice_location(const io::FileWriter& file_writer, + const std::string& file_path); + void add_packed_slice_location(const std::string& segment_path, const std::string& packed_file_path, int64_t offset, int64_t size, int64_t packed_file_size) { diff --git a/be/test/storage/rowset/rowset_meta_test.cpp b/be/test/storage/rowset/rowset_meta_test.cpp index 633c4f4999abed..b8a415e574a0bc 100644 --- a/be/test/storage/rowset/rowset_meta_test.cpp +++ b/be/test/storage/rowset/rowset_meta_test.cpp @@ -25,15 +25,19 @@ #include #include +#include #include #include +#include "common/exception.h" #include "common/status.h" #include "cpp/sync_point.h" #include "gtest/gtest_pred_impl.h" #include "io/fs/file_reader.h" #include "io/fs/file_system.h" +#include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" +#include "io/fs/packed_file_writer.h" #include "storage/olap_common.h" #include "storage/olap_meta.h" #include "storage/tablet/tablet_schema.h" @@ -47,6 +51,98 @@ using std::string; namespace doris { +namespace { + +class PackedLocationWriter : public io::FileWriter { +public: + explicit PackedLocationWriter(std::string path) : _path(std::move(path)) {} + + Status close(bool non_block = false) override { + _state = non_block ? State::ASYNC_CLOSING : State::CLOSED; + return Status::OK(); + } + Status appendv(const Slice*, size_t) override { return Status::NotSupported("test writer"); } + const io::Path& path() const override { return _path; } + size_t bytes_appended() const override { return 10; } + State state() const override { return _state; } + +private: + io::Path _path; + State _state = State::OPENED; +}; + +} // namespace + +TEST(RowsetMetaPackedFileTest, CollectCompletedSegmentAndIndex) { + RowsetMeta meta; + RowsetId rowset_id; + rowset_id.init(1); + meta.set_rowset_id(rowset_id); + // Use destination paths with a nonzero segment ID, including the V2 index file. + for (const auto* suffix : {".dat", ".idx"}) { + std::string path = std::string("data/123/packed_location_test_7") + suffix; + io::PackedFileWriter writer( + std::make_unique("s3://bucket/prefix/" + path), path); + io::PackedSliceLocation location; + location.packed_file_path = "data/packed_file/test.bin"; + location.offset = 20; + location.size = 10; + location.packed_file_size = 100; + // The writer's handle remains sufficient after the manager's index is gone. + writer._packed_slice_handle = std::make_shared(location); + writer._packed_slice_handle->set_upload_result(io::PackedSliceUploadState::UPLOADED, + location.packed_file_size); + + ASSERT_TRUE(meta.collect_packed_slice_location(writer, path).ok()); + EXPECT_EQ(meta.get_rowset_pb().packed_slice_locations().count(path), 0); + writer._state = io::FileWriter::State::ASYNC_CLOSING; + ASSERT_TRUE(meta.collect_packed_slice_location(writer, path).ok()); + EXPECT_EQ(meta.get_rowset_pb().packed_slice_locations().count(path), 0); + writer._state = io::FileWriter::State::CLOSED; + ASSERT_TRUE(meta.collect_packed_slice_location(writer, path).ok()); + // Repeated collection must not duplicate the mapping. + ASSERT_TRUE(meta.collect_packed_slice_location(writer, path).ok()); + auto pb = meta.get_rowset_pb(); + const auto& stored = pb.packed_slice_locations().at(path); + EXPECT_EQ(stored.packed_file_path(), location.packed_file_path); + EXPECT_EQ(stored.offset(), location.offset); + EXPECT_EQ(stored.size(), location.size); + EXPECT_EQ(stored.packed_file_size(), location.packed_file_size); + } + RowsetMeta restored; + std::string serialized; + ASSERT_TRUE(meta.serialize(&serialized)); + ASSERT_TRUE(restored.init(serialized)); + EXPECT_EQ(restored.get_rowset_pb().packed_slice_locations_size(), 2); +} + +TEST(RowsetMetaPackedFileTest, DirectFileNeedsNoPackedLocation) { + RowsetMeta meta; + PackedLocationWriter writer("direct_file.dat"); + ASSERT_TRUE(writer.close().ok()); + EXPECT_TRUE(meta.collect_packed_slice_location(writer, "direct_file.dat").ok()); + EXPECT_EQ(meta.get_rowset_pb().packed_slice_locations_size(), 0); +} + +TEST(RowsetMetaPackedFileTest, MissingPackedLocationFails) { + RowsetMeta meta; + io::PackedFileWriter writer( + std::make_unique("missing_packed_location.dat"), + "missing_packed_location.dat"); + writer._state = io::FileWriter::State::CLOSED; +#ifndef NDEBUG + GTEST_FLAG_SET(death_test_style, "threadsafe"); + EXPECT_DEATH(static_cast( + meta.collect_packed_slice_location(writer, "missing_packed_location.dat")), + "location.packed_file_path.empty"); +#else + EXPECT_THROW(static_cast( + meta.collect_packed_slice_location(writer, "missing_packed_location.dat")), + Exception); +#endif + EXPECT_EQ(meta.get_rowset_pb().packed_slice_locations_size(), 0); +} + const std::string rowset_meta_path = "./be/test/storage/test_data/rowset.json"; class RowsetMetaTest : public testing::Test { From fa0de8781640995f7ef10697c521195be627ca28 Mon Sep 17 00:00:00 2001 From: meiyi Date: Tue, 15 Sep 2026 17:39:18 +0800 Subject: [PATCH 2/6] [fix](cloud) Avoid concurrent and duplicate packed metadata collection ### What problem does this PR solve? Problem Summary: Cloud MOW loads build temporary rowsets in asynchronous delete bitmap tasks while other flushes can still add or close file writers. Building those temporary rowsets previously traversed all segment and index writer collections without their locks to collect packed mappings. Collect only the current segment's mapping after its writer closes, and reserve full collection for final rowset construction after flushes finish. Pass the segment ID through _build_tmp and update the existing unit-test wrapper and call accordingly. CloudRowsetWriter::build also collected the same mappings a second time directly after _build_rowset_meta had already collected them. Remove that redundant call. These issues affect existing Cloud load paths independently of memtable forwarding. Forwarded-file snapshots and direct-upload logic remain in the following feature. ### Release note Avoid concurrent traversal of file writer collections during Cloud MOW packed file loads and eliminate duplicate packed mapping collection at finalization. ### Check List (For Author) - Test: Static call-site review and git diff --check; existing TmpRowsetUsesCompletedSegmentIds test adapted to the signature change. No builds or runtime tests at user request; concurrent execution was not reproduced during this split. - Behavior changed: Yes; temporary rowsets collect only the current segment's packed mapping and final rowsets collect all mappings once - Does this need documentation: No --- be/src/cloud/cloud_rowset_writer.cpp | 10 ++++++---- be/src/storage/rowset/beta_rowset_writer.cpp | 13 +++++++++++-- be/src/storage/rowset/beta_rowset_writer.h | 2 +- be/test/storage/rowset/beta_rowset_test.cpp | 6 ++++-- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/be/src/cloud/cloud_rowset_writer.cpp b/be/src/cloud/cloud_rowset_writer.cpp index 330527329d5bd6..c633bc25e30b13 100644 --- a/be/src/cloud/cloud_rowset_writer.cpp +++ b/be/src/cloud/cloud_rowset_writer.cpp @@ -102,8 +102,12 @@ Status CloudRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool check RETURN_IF_ERROR(BaseBetaRowsetWriter::_build_rowset_meta(rowset_meta, check_segment_num, completed_segment_ids)); - // Collect packed file segment index information for interim rowsets as well. - return _collect_all_packed_slice_locations(rowset_meta); + // Temporary bitmap rowsets collect their own segment after its file is closed. + // The complete collections are safe to traverse only after all flushes finish. + if (completed_segment_ids == nullptr) { + return _collect_all_packed_slice_locations(rowset_meta); + } + return Status::OK(); } Status CloudRowsetWriter::build(RowsetSharedPtr& rowset) { @@ -115,8 +119,6 @@ Status CloudRowsetWriter::build(RowsetSharedPtr& rowset) { // TODO(plat1ko): check_segment_footer RETURN_IF_ERROR(_build_rowset_meta(_rowset_meta.get())); - // At this point all writers have been closed, so collecting packed file indices is safe. - RETURN_IF_ERROR(_collect_all_packed_slice_locations(_rowset_meta.get())); // If the current load is a partial update, new segments may be appended to the tmp rowset after the tmp rowset // has been committed if conflicts occur due to concurrent partial updates. However, when the recycler do recycling, // it will generate the paths for the segments to be recycled on the object storage based on the number of segments diff --git a/be/src/storage/rowset/beta_rowset_writer.cpp b/be/src/storage/rowset/beta_rowset_writer.cpp index 265448a831472b..72b3b5dcca0095 100644 --- a/be/src/storage/rowset/beta_rowset_writer.cpp +++ b/be/src/storage/rowset/beta_rowset_writer.cpp @@ -458,7 +458,7 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { OlapStopWatch watch; // Step 2: Build tmp rowset (needs file_writer to be closed) RowsetSharedPtr rowset_ptr; - st = _build_tmp(rowset_ptr); + st = _build_tmp(rowset_ptr, segment_id); if (!st.ok()) { return st; } @@ -1156,7 +1156,7 @@ Status BaseBetaRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool ch return Status::OK(); } -Status BaseBetaRowsetWriter::_build_tmp(RowsetSharedPtr& rowset_ptr) { +Status BaseBetaRowsetWriter::_build_tmp(RowsetSharedPtr& rowset_ptr, int32_t segment_id) { Status status; std::shared_ptr tmp_rs_meta = std::make_shared(); tmp_rs_meta->init(_rowset_meta.get()); @@ -1169,6 +1169,15 @@ Status BaseBetaRowsetWriter::_build_tmp(RowsetSharedPtr& rowset_ptr) { } tmp_rs_meta->set_segment_ids(completed_segment_ids); + if (_context.packed_file_active) { + // Bitmap calculation reads only this segment's primary-key index. Other flushes + // may still be creating/closing files, so do not traverse their writer collections. + if (auto* writer = _seg_files.get(segment_id); writer != nullptr) { + RETURN_IF_ERROR(tmp_rs_meta->collect_packed_slice_location( + *writer, _context.segment_path(segment_id))); + } + } + status = RowsetFactory::create_rowset(_context.tablet_schema, _context.tablet_path, tmp_rs_meta, &rowset_ptr); DBUG_EXECUTE_IF("BaseBetaRowsetWriter::_build_tmp.create_rowset_failed", diff --git a/be/src/storage/rowset/beta_rowset_writer.h b/be/src/storage/rowset/beta_rowset_writer.h index 49811ac7714512..918edb057f0bdc 100644 --- a/be/src/storage/rowset/beta_rowset_writer.h +++ b/be/src/storage/rowset/beta_rowset_writer.h @@ -224,7 +224,7 @@ class BaseBetaRowsetWriter : public RowsetWriter { virtual Status _check_segment_number_limit(size_t segnum); virtual int64_t _num_seg() const; // build a tmp rowset for load segment to calc delete_bitmap for this segment - Status _build_tmp(RowsetSharedPtr& rowset_ptr); + Status _build_tmp(RowsetSharedPtr& rowset_ptr, int32_t segment_id); uint64_t get_rowset_num_rows() { std::lock_guard l(_segid_statistics_map_mutex); diff --git a/be/test/storage/rowset/beta_rowset_test.cpp b/be/test/storage/rowset/beta_rowset_test.cpp index c7337ff4f9bfc6..c6c1d3e4343722 100644 --- a/be/test/storage/rowset/beta_rowset_test.cpp +++ b/be/test/storage/rowset/beta_rowset_test.cpp @@ -239,7 +239,9 @@ class BetaRowsetWriterForTest : public BetaRowsetWriter { public: explicit BetaRowsetWriterForTest(StorageEngine& engine) : BetaRowsetWriter(engine) {} - Status build_tmp(RowsetSharedPtr& rowset) { return _build_tmp(rowset); } + Status build_tmp(RowsetSharedPtr& rowset, int32_t segment_id) { + return _build_tmp(rowset, segment_id); + } }; class S3ClientMock : public Aws::S3::S3Client { @@ -533,7 +535,7 @@ TEST_F(BetaRowsetTest, TmpRowsetUsesCompletedSegmentIds) { ASSERT_TRUE(writer.add_segment(2, segment_statistics).ok()); RowsetSharedPtr tmp_rowset; - ASSERT_TRUE(writer.build_tmp(tmp_rowset).ok()); + ASSERT_TRUE(writer.build_tmp(tmp_rowset, 6).ok()); ASSERT_NE(tmp_rowset, nullptr); EXPECT_EQ(tmp_rowset->num_segments(), 2); EXPECT_EQ(tmp_rowset->rowset_meta()->position_of(2), 0); From cc5cea49d1b652227b3af22c88e3bb7343ce0df8 Mon Sep 17 00:00:00 2001 From: meiyi Date: Mon, 21 Sep 2026 15:44:12 +0800 Subject: [PATCH 3/6] [feature](cloud) Support memtable sink loads through file forwarding ### What problem does this PR solve? Problem Summary: Cloud loads previously excluded the memtable-on-sink path. Allow eligible full-row DUPLICATE, AGGREGATE, UNIQUE MOR, and UNIQUE MOW loads to build memtables and segments on sink BEs, then forward segment and index files to the target BE for object storage upload and rowset commit. Retain the restrictions for partial updates, V1 indexes, and row binlog. Pass transaction expiration, storage vault, and cache policy through stream-open requests. Select CloudRowsetBuilder in LoadStreamWriter, collect forwarded packed-file mappings and file sizes, and snapshot temporary rowset metadata before asynchronous MOW bitmap calculation on the target BE. Honor skip_writing_empty_rowset_metadata for untouched tablets by setting the policy before initialization and skipping prepare/commit RPCs while preserving empty markers and statistics. Close cloud tablet writers with concurrency 10 after pre-close, collecting all tablet results even when some fail. Preserve local-mode close behavior and existing locking. Add coverage for DUP/AGG/MOR/MOW, packed files, Stream Load, Broker Load, async Group Commit internal writer profiles, duplicate-key checks, cold reads, failed load invisibility and retry, and partial-update loading results. Add BE coverage for bitmap snapshot failures, empty-rowset lifecycle, and bounded concurrent close with mixed results. Align regression output blocks with the file-forwarding suite. ### Release note Cloud loads can enable memtable-on-sink for eligible full-row DUPLICATE, AGGREGATE, and UNIQUE MOR/MOW tables. Sink BEs construct files and target BEs upload them and calculate MOW bitmaps. Empty tablets honor the existing metadata-skip configuration, and cloud tablet closes run with bounded concurrency. ### Check List (For Author) - Test: Static call-site review, clang-format 16 checks, build hygiene checks, and source git diff --check passed. BE unit tests and regression cases added or updated; no compilation, source generation, or runtime tests performed during these changes, as requested. Unused regression output blocks removed without changing retained results; existing output-file trailing separators preserved. - Behavior changed: Yes; eligible Cloud loads use file forwarding, preserve empty metadata skipping, and close tablet writers concurrently. - Does this need documentation: Yes; design documents excluded from this commit at user request. --- be/src/cloud/cloud_rowset_builder.cpp | 11 + be/src/cloud/cloud_rowset_builder.h | 2 + .../pipeline/pipeline_fragment_context.cpp | 2 +- be/src/exec/sink/load_stream_stub.cpp | 15 +- be/src/exec/sink/load_stream_stub.h | 6 +- be/src/exec/sink/writer/vtablet_writer_v2.cpp | 15 +- be/src/exec/sink/writer/vtablet_writer_v2.h | 2 + be/src/load/channel/load_stream.cpp | 64 ++- be/src/load/channel/load_stream.h | 17 +- be/src/load/channel/load_stream_writer.cpp | 56 ++- be/src/load/channel/load_stream_writer.h | 3 +- be/src/storage/rowset/beta_rowset_writer.cpp | 16 +- be/test/cloud/cloud_load_stream_test.cpp | 291 +++++++++++++ be/test/storage/rowset/beta_rowset_test.cpp | 35 ++ .../doris/load/loadv2/BrokerLoadJob.java | 2 +- gensrc/proto/internal_service.proto | 3 + .../test_cloud_memtable_mow_forward.out | 109 +++++ .../cloud_p0/test_cloud_memtable_on_sink.out | 148 +++++++ .../test_cloud_memtable_mow_forward.groovy | 255 +++++++++++ .../test_cloud_memtable_on_sink.groovy | 405 ++++++++++++++++++ 20 files changed, 1416 insertions(+), 41 deletions(-) create mode 100644 be/test/cloud/cloud_load_stream_test.cpp create mode 100644 regression-test/data/cloud_p0/test_cloud_memtable_mow_forward.out create mode 100644 regression-test/data/cloud_p0/test_cloud_memtable_on_sink.out create mode 100644 regression-test/suites/cloud_p0/test_cloud_memtable_mow_forward.groovy create mode 100644 regression-test/suites/cloud_p0/test_cloud_memtable_on_sink.groovy diff --git a/be/src/cloud/cloud_rowset_builder.cpp b/be/src/cloud/cloud_rowset_builder.cpp index d9065c340ece9e..7d363aea2d1aff 100644 --- a/be/src/cloud/cloud_rowset_builder.cpp +++ b/be/src/cloud/cloud_rowset_builder.cpp @@ -249,6 +249,17 @@ Status CloudRowsetBuilder::commit_rowset(const std::string& job_id, int64_t tabl return _engine.meta_mgr().commit_rowset(*rowset_meta(), job_id, table_id); } +Status CloudRowsetBuilder::commit_txn() { + DCHECK(is_data_builder()); + if (!_skip_writing_rowset_metadata) { + RETURN_IF_ERROR(commit_rowset("", _tablet->table_id())); + } + RETURN_IF_ERROR(set_txn_related_info()); + update_tablet_stats(); + _is_committed = true; + return Status::OK(); +} + Status CloudRowsetBuilder::set_txn_related_info() { if (_tablet->enable_unique_key_merge_on_write() || _tablet->is_row_binlog_tablet()) { // For empty rowsets when skip_writing_empty_rowset_metadata=true, diff --git a/be/src/cloud/cloud_rowset_builder.h b/be/src/cloud/cloud_rowset_builder.h index c65e192444865d..d7db14f43f2595 100644 --- a/be/src/cloud/cloud_rowset_builder.h +++ b/be/src/cloud/cloud_rowset_builder.h @@ -33,6 +33,8 @@ class CloudRowsetBuilder : public BaseRowsetBuilder { Status init() override; + Status commit_txn() override; + virtual void update_tablet_stats(); const RowsetMetaSharedPtr& rowset_meta(); diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index 5fa5c4c2932294..cc1d359d02df24 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -1305,7 +1305,7 @@ Status PipelineFragmentContext::_create_data_sink(ObjectPool* pool, const TDataS int child_node_id = pipeline->operators().back()->node_id(); if (state->query_options().enable_memtable_on_sink_node && !_has_inverted_index_v1_or_partial_update(thrift_sink.olap_table_sink) && - !_has_row_binlog(thrift_sink.olap_table_sink) && !config::is_cloud_mode()) { + !_has_row_binlog(thrift_sink.olap_table_sink)) { _sink = std::make_shared( pool, next_sink_operator_id(), child_node_id + 1, row_desc, output_exprs); } else { diff --git a/be/src/exec/sink/load_stream_stub.cpp b/be/src/exec/sink/load_stream_stub.cpp index e46eec24333868..967e35fed49181 100644 --- a/be/src/exec/sink/load_stream_stub.cpp +++ b/be/src/exec/sink/load_stream_stub.cpp @@ -170,7 +170,8 @@ Status LoadStreamStub::open(BrpcClientCache* client_cache, const NodeInfo& node_info, int64_t txn_id, const OlapTableSchemaParam& schema, const std::vector& tablets_for_schema, int total_streams, - int64_t idle_timeout_ms, bool enable_profile) { + int64_t idle_timeout_ms, bool enable_profile, int64_t txn_expiration, + const std::string& storage_vault_id, bool write_file_cache) { std::unique_lock lock(_open_mutex); if (_is_init.load()) { return _status; @@ -194,6 +195,9 @@ Status LoadStreamStub::open(BrpcClientCache* client_cache, request.set_src_id(_src_id); request.set_txn_id(txn_id); request.set_enable_profile(enable_profile); + request.set_txn_expiration(txn_expiration); + request.set_storage_vault_id(storage_vault_id); + request.set_write_file_cache(write_file_cache); if (_is_incremental) { request.set_total_streams(0); } else if (total_streams > 0) { @@ -591,7 +595,8 @@ Status LoadStreamStubs::open(BrpcClientCache* client_cache const NodeInfo& node_info, int64_t txn_id, const OlapTableSchemaParam& schema, const std::vector& tablets_for_schema, int total_streams, - int64_t idle_timeout_ms, bool enable_profile) { + int64_t idle_timeout_ms, bool enable_profile, int64_t txn_expiration, + const std::string& storage_vault_id, bool write_file_cache) { bool get_schema = true; auto status = Status::OK(); bool first_stream = true; @@ -599,10 +604,12 @@ Status LoadStreamStubs::open(BrpcClientCache* client_cache Status st; if (get_schema) { st = stream->open(client_cache, node_info, txn_id, schema, tablets_for_schema, - total_streams, idle_timeout_ms, enable_profile); + total_streams, idle_timeout_ms, enable_profile, txn_expiration, + storage_vault_id, write_file_cache); } else { st = stream->open(client_cache, node_info, txn_id, schema, {}, total_streams, - idle_timeout_ms, enable_profile); + idle_timeout_ms, enable_profile, txn_expiration, storage_vault_id, + write_file_cache); } // Simulate one stream open failure within LoadStreamStubs. // This causes the successfully opened streams to be cancelled, diff --git a/be/src/exec/sink/load_stream_stub.h b/be/src/exec/sink/load_stream_stub.h index 8951c483220c01..4fb16b2db71708 100644 --- a/be/src/exec/sink/load_stream_stub.h +++ b/be/src/exec/sink/load_stream_stub.h @@ -144,7 +144,8 @@ class LoadStreamStub : public std::enable_shared_from_this { Status open(BrpcClientCache* client_cache, const NodeInfo& node_info, int64_t txn_id, const OlapTableSchemaParam& schema, const std::vector& tablets_for_schema, int total_streams, - int64_t idle_timeout_ms, bool enable_profile); + int64_t idle_timeout_ms, bool enable_profile, int64_t txn_expiration, + const std::string& storage_vault_id, bool write_file_cache); // for mock this class in UT #ifdef BE_TEST @@ -331,7 +332,8 @@ class LoadStreamStubs { Status open(BrpcClientCache* client_cache, const NodeInfo& node_info, int64_t txn_id, const OlapTableSchemaParam& schema, const std::vector& tablets_for_schema, int total_streams, - int64_t idle_timeout_ms, bool enable_profile); + int64_t idle_timeout_ms, bool enable_profile, int64_t txn_expiration, + const std::string& storage_vault_id, bool write_file_cache); bool is_incremental() const { return _is_incremental; } diff --git a/be/src/exec/sink/writer/vtablet_writer_v2.cpp b/be/src/exec/sink/writer/vtablet_writer_v2.cpp index 8a5fe58500a1d9..fa97d272af148c 100644 --- a/be/src/exec/sink/writer/vtablet_writer_v2.cpp +++ b/be/src/exec/sink/writer/vtablet_writer_v2.cpp @@ -149,9 +149,18 @@ Status VTabletWriterV2::_init(RuntimeState* state, RuntimeProfile* profile) { _load_id.set_lo(table_sink.load_id.lo); signal::set_signal_task_id(_load_id); _txn_id = table_sink.txn_id; + if (config::is_cloud_mode()) { + if (!table_sink.__isset.txn_timeout_s || table_sink.txn_timeout_s <= 0) { + return Status::InternalError("The txn_timeout_s of TDataSink is invalid"); + } + _txn_expiration = UnixSeconds() + table_sink.txn_timeout_s; + } _num_replicas = table_sink.num_replicas; _tuple_desc_id = table_sink.tuple_id; _write_file_cache = table_sink.write_file_cache; + if (table_sink.__isset.storage_vault_id) { + _storage_vault_id = table_sink.storage_vault_id; + } _schema.reset(new OlapTableSchemaParam()); RETURN_IF_ERROR(_schema->init(table_sink.schema)); _schema->set_timestamp_ms(state->timestamp_ms()); @@ -318,7 +327,8 @@ Status VTabletWriterV2::_open_streams_to_backend(int64_t dst_id, LoadStreamStubs { tablets_for_schema.clear(); }); auto st = streams.open(_state->exec_env()->brpc_streaming_client_cache(), *node_info, _txn_id, *_schema, tablets_for_schema, _total_streams, idle_timeout_ms, - _state->enable_profile()); + _state->enable_profile(), _txn_expiration, _storage_vault_id, + _write_file_cache); if (!st.ok()) { LOG(WARNING) << "failed to open stream to backend " << dst_id << ", load_id=" << print_id(_load_id) << ", err=" << st; @@ -791,6 +801,9 @@ Status VTabletWriterV2::close(Status exec_status) { _row_distribution.output_profile_info(_operator_profile); } + // Keep the fragment alive so tests can fetch the completed writer profile over Thrift. + DBUG_EXECUTE_IF("VTabletWriterV2.close.profile_ready", DBUG_BLOCK); + LOG(INFO) << "finished to close olap table sink. load_id=" << print_id(_load_id) << ", txn_id=" << _txn_id; } else { diff --git a/be/src/exec/sink/writer/vtablet_writer_v2.h b/be/src/exec/sink/writer/vtablet_writer_v2.h index a3d86a613bbde1..7c6269e9e8726f 100644 --- a/be/src/exec/sink/writer/vtablet_writer_v2.h +++ b/be/src/exec/sink/writer/vtablet_writer_v2.h @@ -179,6 +179,7 @@ class VTabletWriterV2 final : public AsyncResultWriter { // unique load id PUniqueId _load_id; int64_t _txn_id = -1; + int64_t _txn_expiration = 0; int _num_replicas = -1; int _tuple_desc_id = -1; @@ -197,6 +198,7 @@ class VTabletWriterV2 final : public AsyncResultWriter { int _num_local_sink = -1; bool _is_high_priority = false; bool _write_file_cache = false; + std::string _storage_vault_id; // TODO(zc): think about cache this data std::shared_ptr _schema; diff --git a/be/src/load/channel/load_stream.cpp b/be/src/load/channel/load_stream.cpp index 64c8beb0a2a073..d0ac39c7782d69 100644 --- a/be/src/load/channel/load_stream.cpp +++ b/be/src/load/channel/load_stream.cpp @@ -26,6 +26,7 @@ #include #include "bvar/bvar.h" +#include "cloud/cloud_meta_mgr.h" #include "cloud/config.h" #include "common/signal_handler.h" #include "load/channel/load_channel.h" @@ -57,11 +58,16 @@ bvar::LatencyRecorder g_load_stream_flush_wait_ms("load_stream_flush_wait_ms"); bvar::Adder g_load_stream_flush_running_threads("load_stream_flush_wait_threads"); TabletStream::TabletStream(const PUniqueId& load_id, int64_t id, int64_t txn_id, - LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile) + LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile, + int64_t txn_expiration, std::string storage_vault_id, + bool write_file_cache) : _id(id), _next_segid(0), _load_id(load_id), _txn_id(txn_id), + _txn_expiration(txn_expiration), + _storage_vault_id(std::move(storage_vault_id)), + _write_file_cache(write_file_cache), _load_stream_mgr(load_stream_mgr) { load_stream_mgr->create_token(_flush_token); _profile = profile->create_child(fmt::format("TabletStream {}", id), true, true); @@ -77,16 +83,17 @@ inline std::ostream& operator<<(std::ostream& ostr, const TabletStream& tablet_s } Status TabletStream::init(std::shared_ptr schema, int64_t index_id, - int64_t partition_id) { + int64_t partition_id, bool is_empty) { WriteRequest req { .tablet_id = _id, .txn_id = _txn_id, + .txn_expiration = _txn_expiration, .index_id = index_id, .partition_id = partition_id, .load_id = _load_id, .table_schema_param = schema, - // TODO(plat1ko): write_file_cache - .storage_vault_id {}, + .write_file_cache = _write_file_cache, + .storage_vault_id = _storage_vault_id, }; _load_stream_writer = std::make_shared(&req, _profile); @@ -94,7 +101,7 @@ Status TabletStream::init(std::shared_ptr schema, int64_t _status.update(Status::Uninitialized("fault injection")); return _status.status(); }); - _status.update(_load_stream_writer->init()); + _status.update(_load_stream_writer->init(is_empty)); if (!_status.ok()) { LOG(INFO) << "failed to init rowset builder due to " << *this; } @@ -346,10 +353,15 @@ Status TabletStream::close() { IndexStream::IndexStream(const PUniqueId& load_id, int64_t id, int64_t txn_id, std::shared_ptr schema, - LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile) + LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile, + int64_t txn_expiration, std::string storage_vault_id, + bool write_file_cache) : _id(id), _load_id(load_id), _txn_id(txn_id), + _txn_expiration(txn_expiration), + _storage_vault_id(std::move(storage_vault_id)), + _write_file_cache(write_file_cache), _schema(schema), _load_stream_mgr(load_stream_mgr) { _profile = profile->create_child(fmt::format("IndexStream {}", id), true, true); @@ -375,7 +387,7 @@ Status IndexStream::append_data(const PStreamHeader& header, butil::IOBuf* data) std::lock_guard lock_guard(_lock); auto it = _tablet_streams_map.find(tablet_id); if (it == _tablet_streams_map.end()) { - _init_tablet_stream(tablet_stream, tablet_id, header.partition_id()); + _init_tablet_stream(tablet_stream, tablet_id, header.partition_id(), false); } else { tablet_stream = it->second; } @@ -385,11 +397,12 @@ Status IndexStream::append_data(const PStreamHeader& header, butil::IOBuf* data) } void IndexStream::_init_tablet_stream(TabletStreamSharedPtr& tablet_stream, int64_t tablet_id, - int64_t partition_id) { - tablet_stream = std::make_shared(_load_id, tablet_id, _txn_id, _load_stream_mgr, - _profile); + int64_t partition_id, bool is_empty) { + tablet_stream = + std::make_shared(_load_id, tablet_id, _txn_id, _load_stream_mgr, _profile, + _txn_expiration, _storage_vault_id, _write_file_cache); _tablet_streams_map[tablet_id] = tablet_stream; - auto st = tablet_stream->init(_schema, _id, partition_id); + auto st = tablet_stream->init(_schema, _id, partition_id, is_empty); if (!st.ok()) { LOG(WARNING) << "tablet stream init failed " << *tablet_stream; } @@ -414,7 +427,8 @@ void IndexStream::close(const std::vector& tablets_to_commit, TabletStreamSharedPtr tablet_stream; auto it = _tablet_streams_map.find(tablet.tablet_id()); if (it == _tablet_streams_map.end()) { - _init_tablet_stream(tablet_stream, tablet.tablet_id(), tablet.partition_id()); + // A tablet first seen at close received no files from any sender. + _init_tablet_stream(tablet_stream, tablet.tablet_id(), tablet.partition_id(), true); } else { tablet_stream = it->second; } @@ -430,8 +444,28 @@ void IndexStream::close(const std::vector& tablets_to_commit, tablet_stream->pre_close(); } + const bool is_cloud = config::is_cloud_mode(); + std::vector close_statuses; + if (is_cloud) { + close_statuses.resize(_tablet_streams_map.size()); + std::vector> tasks; + tasks.reserve(_tablet_streams_map.size()); + size_t i = 0; + for (auto& [_, tablet_stream] : _tablet_streams_map) { + tasks.emplace_back([tablet_stream, &close_statuses, i] { + close_statuses[i] = tablet_stream->close(); + // A tablet failure must not stop the remaining tablets from closing. + return Status::OK(); + }); + ++i; + } + auto st = cloud::bthread_fork_join(tasks, 10); + DORIS_CHECK(st.ok()) << st; + } + + size_t i = 0; for (auto& [_, tablet_stream] : _tablet_streams_map) { - auto st = tablet_stream->close(); + auto st = is_cloud ? std::move(close_statuses[i++]) : tablet_stream->close(); if (st.ok()) { success_tablet_ids->push_back(tablet_stream->id()); } else { @@ -489,7 +523,9 @@ Status LoadStream::init(const POpenLoadStreamRequest* request) { RETURN_IF_ERROR(_schema->init(request->schema())); for (auto& index : request->schema().indexes()) { _index_streams_map[index.id()] = std::make_shared( - _load_id, index.id(), _txn_id, _schema, _load_stream_mgr, _profile.get()); + _load_id, index.id(), _txn_id, _schema, _load_stream_mgr, _profile.get(), + request->txn_expiration(), request->storage_vault_id(), + request->write_file_cache()); } LOG(INFO) << "succeed to init load stream " << *this; return Status::OK(); diff --git a/be/src/load/channel/load_stream.h b/be/src/load/channel/load_stream.h index 4c8865e160306d..22f5845bd1a0fb 100644 --- a/be/src/load/channel/load_stream.h +++ b/be/src/load/channel/load_stream.h @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -45,10 +46,11 @@ using FailedTablets = std::vector>; class TabletStream { public: TabletStream(const PUniqueId& load_id, int64_t id, int64_t txn_id, - LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile); + LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile, int64_t txn_expiration, + std::string storage_vault_id, bool write_file_cache); Status init(std::shared_ptr schema, int64_t index_id, - int64_t partition_id); + int64_t partition_id, bool is_empty = false); Status append_data(const PStreamHeader& header, butil::IOBuf* data); Status add_segment(const PStreamHeader& header, butil::IOBuf* data); @@ -78,6 +80,9 @@ class TabletStream { AtomicStatus _status; PUniqueId _load_id; int64_t _txn_id; + int64_t _txn_expiration; + std::string _storage_vault_id; + bool _write_file_cache; RuntimeProfile* _profile = nullptr; RuntimeProfile::Counter* _append_data_timer = nullptr; RuntimeProfile::Counter* _add_segment_timer = nullptr; @@ -91,7 +96,8 @@ class IndexStream { public: IndexStream(const PUniqueId& load_id, int64_t id, int64_t txn_id, std::shared_ptr schema, LoadStreamMgr* load_stream_mgr, - RuntimeProfile* profile); + RuntimeProfile* profile, int64_t txn_expiration, std::string storage_vault_id, + bool write_file_cache); ~IndexStream(); Status append_data(const PStreamHeader& header, butil::IOBuf* data); @@ -103,7 +109,7 @@ class IndexStream { private: void _init_tablet_stream(TabletStreamSharedPtr& tablet_stream, int64_t tablet_id, - int64_t partition_id); + int64_t partition_id, bool is_empty); private: int64_t _id; @@ -111,6 +117,9 @@ class IndexStream { bthread::Mutex _lock; PUniqueId _load_id; int64_t _txn_id; + int64_t _txn_expiration; + std::string _storage_vault_id; + bool _write_file_cache; std::shared_ptr _schema; std::unordered_map _tablet_partitions; RuntimeProfile* _profile = nullptr; diff --git a/be/src/load/channel/load_stream_writer.cpp b/be/src/load/channel/load_stream_writer.cpp index df67245ef375d7..b507c019fb3b5d 100644 --- a/be/src/load/channel/load_stream_writer.cpp +++ b/be/src/load/channel/load_stream_writer.cpp @@ -29,6 +29,7 @@ #include #include "bvar/bvar.h" +#include "cloud/cloud_rowset_builder.h" #include "cloud/config.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" @@ -75,9 +76,13 @@ bvar::Adder g_load_stream_file_writer_cnt("load_stream_file_writer_coun LoadStreamWriter::LoadStreamWriter(WriteRequest* context, RuntimeProfile* profile) : _req(*context), _rowset_writer(nullptr) { g_load_stream_writer_cnt << 1; - // TODO(plat1ko): CloudStorageEngine - _rowset_builder = std::make_unique( - ExecEnv::GetInstance()->storage_engine().to_local(), *context, profile); + auto& engine = ExecEnv::GetInstance()->storage_engine(); + if (config::is_cloud_mode()) { + _rowset_builder = + std::make_unique(engine.to_cloud(), *context, profile); + } else { + _rowset_builder = std::make_unique(engine.to_local(), *context, profile); + } _resource_ctx = thread_context()->resource_ctx(); // from load stream } @@ -87,9 +92,15 @@ LoadStreamWriter::~LoadStreamWriter() { g_load_stream_writer_cnt << -1; } -Status LoadStreamWriter::init() { +Status LoadStreamWriter::init(bool is_empty) { DBUG_EXECUTE_IF("LoadStreamWriter.init.failure", { return Status::InternalError("fault injection"); }); + if (config::is_cloud_mode()) { + // Set this before initialization so empty tablets also skip PREPARE_ROWSET. + static_cast(_rowset_builder.get()) + ->set_skip_writing_rowset_metadata(is_empty && + config::skip_writing_empty_rowset_metadata); + } RETURN_IF_ERROR(_rowset_builder->init()); _rowset_writer = _rowset_builder->rowset_writer(); _is_init = true; @@ -178,6 +189,17 @@ Status LoadStreamWriter::close_writer(uint32_t segid, FileType file_type) { return Status::Corruption("file {} closed with 0 bytes, file type is {}", file_writer->path().native(), file_type); } + if (config::is_cloud_mode()) { + // Streamed writers are owned here, outside the RowsetWriter's file collections. + // Use the logical path used at creation, not file_writer->path(), which may be an S3 URI. + auto file_path = _rowset_writer->context().segment_path(segid); + if (file_type == FileType::INVERTED_INDEX_FILE) { + auto prefix = InvertedIndexDescriptor::get_index_file_path_prefix(file_path); + file_path = InvertedIndexDescriptor::get_index_file_path_v2(std::string(prefix)); + } + return _rowset_writer->rowset_meta()->collect_packed_slice_location(*file_writer, + file_path); + } return Status::OK(); } @@ -252,7 +274,7 @@ Status LoadStreamWriter::_pre_close() { // in same partition has data loaded. // so we have to also init this LoadStreamWriter, so that it can create an empty rowset // for this tablet when being closed. - RETURN_IF_ERROR(init()); + RETURN_IF_ERROR(init(true)); } DCHECK(_is_init) @@ -306,6 +328,26 @@ Status LoadStreamWriter::_pre_close() { } RETURN_IF_ERROR(_rowset_builder->build_rowset()); + if (config::is_cloud_mode()) { + // Forwarded files are owned here, outside CloudRowsetWriter's collections. + auto meta = _rowset_writer->rowset_meta(); + DORIS_CHECK(meta->segments_file_size().empty()); + DORIS_CHECK(meta->inverted_index_file_info().empty()); + std::vector sizes; + sizes.reserve(_segment_file_writers.size()); + for (const auto& writer : _segment_file_writers) { + sizes.push_back(writer->bytes_appended()); + } + meta->add_segments_file_size(sizes); + std::vector indexes(_inverted_file_writers.size()); + std::vector index_ptrs; + index_ptrs.reserve(indexes.size()); + for (size_t pos = 0; pos < indexes.size(); ++pos) { + indexes[pos].set_index_size(_inverted_file_writers[pos]->bytes_appended()); + index_ptrs.push_back(&indexes[pos]); + } + meta->add_inverted_index_files_info(index_ptrs); + } RETURN_IF_ERROR(_rowset_builder->submit_calc_delete_bitmap_task()); _pre_closed = true; return Status::OK(); @@ -317,9 +359,7 @@ Status LoadStreamWriter::close() { RETURN_IF_ERROR(_pre_close()); } RETURN_IF_ERROR(_rowset_builder->wait_calc_delete_bitmap()); - // FIXME(plat1ko): No `commit_txn` operation in cloud mode, need better abstractions - RETURN_IF_ERROR(static_cast(_rowset_builder.get())->commit_txn()); - + RETURN_IF_ERROR(_rowset_builder->commit_txn()); return Status::OK(); } diff --git a/be/src/load/channel/load_stream_writer.h b/be/src/load/channel/load_stream_writer.h index e82497ec02bfa1..8c4f4d79109e88 100644 --- a/be/src/load/channel/load_stream_writer.h +++ b/be/src/load/channel/load_stream_writer.h @@ -51,7 +51,8 @@ class LoadStreamWriter { ~LoadStreamWriter(); - Status init(); + // is_empty is true for tablets first seen when closing the load. + Status init(bool is_empty = false); Status append_data(uint32_t segid, uint64_t offset, butil::IOBuf buf, FileType file_type = FileType::SEGMENT_FILE); diff --git a/be/src/storage/rowset/beta_rowset_writer.cpp b/be/src/storage/rowset/beta_rowset_writer.cpp index 72b3b5dcca0095..8c0101594c9132 100644 --- a/be/src/storage/rowset/beta_rowset_writer.cpp +++ b/be/src/storage/rowset/beta_rowset_writer.cpp @@ -426,13 +426,20 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { _context.tablet->get_rowset_by_ids(_context.mow_context->rowset_ids.get()); } + RowsetSharedPtr streamed_rowset; + if (_seg_files.get(segment_id) == nullptr) { + // Streamed files are closed by LoadStreamWriter. Snapshot their packed mappings on + // its serial receive thread before later file closes mutate the shared rowset meta. + RETURN_IF_ERROR(_build_tmp(streamed_rowset, segment_id)); + } + // Submit the entire delete bitmap calculation process to thread pool for async execution // This avoids blocking memtable flush thread while waiting for file upload to complete // The process includes: file_writer->close(), _build_tmp, load_segments, and calc_delete_bitmap const auto submit_time_us = MonotonicMicros(); return _calc_delete_bitmap_token->submit_func([this, segment_id, specified_rowsets = std::move(specified_rowsets), - submit_time_us]() -> Status { + submit_time_us, streamed_rowset]() -> Status { const auto queue_time_us = MonotonicMicros() - submit_time_us; Status st = Status::OK(); // Step 1: Close file_writer (must be done before load_segments) @@ -457,10 +464,9 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { OlapStopWatch watch; // Step 2: Build tmp rowset (needs file_writer to be closed) - RowsetSharedPtr rowset_ptr; - st = _build_tmp(rowset_ptr, segment_id); - if (!st.ok()) { - return st; + RowsetSharedPtr rowset_ptr = streamed_rowset; + if (rowset_ptr == nullptr) { + RETURN_IF_ERROR(_build_tmp(rowset_ptr, segment_id)); } // Step 3: Load segments (needs file_writer to be closed and rowset to be built) diff --git a/be/test/cloud/cloud_load_stream_test.cpp b/be/test/cloud/cloud_load_stream_test.cpp new file mode 100644 index 00000000000000..884249f6f7d6cc --- /dev/null +++ b/be/test/cloud/cloud_load_stream_test.cpp @@ -0,0 +1,291 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cloud/cloud_committed_rs_mgr.h" +#include "cloud/cloud_rowset_builder.h" +#include "cloud/cloud_rowset_writer.h" +#include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet.h" +#include "cloud/cloud_txn_delete_bitmap_cache.h" +#include "cloud/config.h" +#include "load/channel/load_stream_mgr.h" +#include "load/channel/load_stream_writer.h" +#include "runtime/exec_env.h" +#include "storage/rowset/rowset.h" +#include "storage/rowset/rowset_factory.h" +#include "storage/rowset/rowset_meta.h" +#include "storage/tablet/tablet_meta.h" +#include "util/time.h" +#include "util/work_thread_pool.hpp" + +namespace doris { +namespace { + +class EmptyLoadRowsetBuilder : public CloudRowsetBuilder { +public: + using CloudRowsetBuilder::CloudRowsetBuilder; + + Status init() override { + skip_metadata_at_init = _skip_writing_rowset_metadata; + return Status::InternalError("stop before preparing rowset metadata"); + } + + Status commit_rowset(const std::string&, int64_t) override { + ++commit_calls; + return Status::InternalError("injected commit failure"); + } + + bool skip_metadata_at_init = false; + int commit_calls = 0; +}; + +class ClosingLoadRowsetBuilder : public BaseRowsetBuilder { +public: + ClosingLoadRowsetBuilder(CloudStorageEngine& engine, const WriteRequest& req, + std::atomic& prepared, std::function commit) + : BaseRowsetBuilder(req, nullptr), _prepared(prepared), _commit(std::move(commit)) { + _rowset_writer = std::make_shared(engine); + _rowset_writer->_rowset_meta = std::make_shared(); + } + + Status init() override { return Status::OK(); } + Status build_rowset() override { + ++_prepared; + return Status::OK(); + } + Status submit_calc_delete_bitmap_task() override { return Status::OK(); } + Status wait_calc_delete_bitmap() override { return Status::OK(); } + Status commit_txn() override { return _commit(); } + +private: + std::atomic& _prepared; + std::function _commit; +}; + +} // namespace + +class CloudLoadStreamTest : public testing::Test { +protected: + void SetUp() override { + _old_cloud_unique_id = config::cloud_unique_id; + _old_skip_empty = config::skip_writing_empty_rowset_metadata; + _old_make_visible = config::enable_cloud_make_rs_visible_on_be; + config::cloud_unique_id = "cloud_load_stream_test"; + config::enable_cloud_make_rs_visible_on_be = true; + _old_engine = std::move(ExecEnv::GetInstance()->_storage_engine); + auto engine = std::make_unique(EngineOptions {}); + _engine = engine.get(); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + _engine->_committed_rs_mgr = std::make_unique(); + _engine->_txn_delete_bitmap_cache = std::make_unique(1024); + ASSERT_TRUE(_engine->_txn_delete_bitmap_cache->init().ok()); + } + + void TearDown() override { + ExecEnv::GetInstance()->set_storage_engine(std::move(_old_engine)); + config::cloud_unique_id = _old_cloud_unique_id; + config::skip_writing_empty_rowset_metadata = _old_skip_empty; + config::enable_cloud_make_rs_visible_on_be = _old_make_visible; + } + + CloudTabletSPtr create_tablet(KeysType keys_type, bool mow) { + TabletMetaPB meta; + meta.set_tablet_id(10001); + meta.set_table_id(10002); + meta.set_enable_unique_key_merge_on_write(mow); + meta.mutable_schema()->set_keys_type(keys_type); + auto tablet_meta = std::make_shared(); + tablet_meta->init_from_pb(meta); + auto tablet = std::make_shared(*_engine, tablet_meta); + std::unique_lock lock(tablet->get_header_lock()); + tablet->reset_approximate_stats(0, 0, 0, 0); + return tablet; + } + + CloudStorageEngine* _engine = nullptr; + std::unique_ptr _old_engine; + std::string _old_cloud_unique_id; + bool _old_skip_empty = false; + bool _old_make_visible = false; +}; + +TEST_F(CloudLoadStreamTest, SetEmptyPolicyBeforePreparingMetadata) { + for (bool skip_empty : {false, true}) { + config::skip_writing_empty_rowset_metadata = skip_empty; + for (bool is_empty : {false, true}) { + WriteRequest req; + LoadStreamWriter writer(&req, nullptr); + auto builder = std::make_unique(*_engine, req, nullptr); + auto* observed_builder = builder.get(); + writer._rowset_builder = std::move(builder); + auto st = writer.init(is_empty); + ASSERT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("stop before preparing"), std::string::npos); + EXPECT_EQ(observed_builder->skip_metadata_at_init, skip_empty && is_empty); + } + } +} + +TEST_F(CloudLoadStreamTest, EmptyCommitSkipsRpcAndRegistersMarkers) { + for (auto keys_type : {DUP_KEYS, AGG_KEYS, UNIQUE_KEYS}) { + for (bool mow : {false, true}) { + if (mow && keys_type != UNIQUE_KEYS) { + continue; + } + WriteRequest req; + req.tablet_id = 10001; + req.txn_id = 100 + keys_type * 2 + mow; + req.txn_expiration = UnixSeconds() + 3600; + EmptyLoadRowsetBuilder builder(*_engine, req, nullptr); + auto tablet = create_tablet(keys_type, mow); + builder._tablet = tablet; + builder._tablet_schema = tablet->tablet_schema(); + auto meta = std::make_shared(); + meta->set_rowset_type(BETA_ROWSET); + meta->set_rowset_state(PREPARED); + meta->set_num_segments(0); + ASSERT_TRUE( + RowsetFactory::create_rowset(builder._tablet_schema, "", meta, &builder._rowset) + .ok()); + builder.set_skip_writing_rowset_metadata(true); + + auto st = builder.commit_txn(); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_EQ(builder.commit_calls, 0); + EXPECT_TRUE(builder._is_committed); + EXPECT_EQ(tablet->fetch_add_approximate_num_rowsets(0), 1); + if (mow) { + EXPECT_TRUE(_engine->txn_delete_bitmap_cache().is_empty_rowset(req.txn_id, + req.tablet_id)); + } else { + auto marker = + _engine->committed_rs_mgr().get_committed_rowset(req.txn_id, req.tablet_id); + ASSERT_TRUE(marker.has_value()); + EXPECT_EQ(marker->first, nullptr); + EXPECT_GE(marker->second, req.txn_expiration); + } + } + } +} + +TEST_F(CloudLoadStreamTest, MetadataCommitFailureDoesNotRegisterMarker) { + WriteRequest req; + req.tablet_id = 10001; + req.txn_id = 200; + EmptyLoadRowsetBuilder builder(*_engine, req, nullptr); + auto tablet = create_tablet(DUP_KEYS, false); + builder._tablet = tablet; + builder.set_skip_writing_rowset_metadata(false); + + auto st = builder.commit_txn(); + ASSERT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("injected commit failure"), std::string::npos); + EXPECT_EQ(builder.commit_calls, 1); + EXPECT_FALSE(builder._is_committed); + EXPECT_EQ(tablet->fetch_add_approximate_num_rowsets(0), 0); + EXPECT_FALSE(_engine->committed_rs_mgr() + .get_committed_rowset(req.txn_id, req.tablet_id) + .has_value()); +} + +TEST_F(CloudLoadStreamTest, CloseTabletsConcurrentlyAndCollectAllResults) { + constexpr int tablet_count = 25; + FifoThreadPool pool(16, 64, "CloudLoadStreamCloseTest"); + LoadStreamMgr manager(1); + manager.set_heavy_work_pool(&pool); + RuntimeProfile profile("CloudLoadStreamCloseTest"); + PUniqueId load_id; + IndexStream index(load_id, 1, 1, nullptr, &manager, &profile, UnixSeconds() + 3600, "", false); + + std::atomic prepared {0}; + std::mutex mutex; + std::condition_variable cv; + int started = 0; + int active = 0; + int peak_active = 0; + bool timed_out = false; + for (int64_t tablet_id = 1; tablet_id <= tablet_count; ++tablet_id) { + auto tablet = std::make_shared(load_id, tablet_id, 1, &manager, &profile, + UnixSeconds() + 3600, "", false); + WriteRequest req; + req.tablet_id = tablet_id; + auto writer = std::make_shared(&req, nullptr); + writer->_rowset_builder = + std::make_unique(*_engine, req, prepared, [&, tablet_id] { + EXPECT_EQ(prepared.load(), tablet_count); + std::unique_lock lock(mutex); + ++started; + ++active; + peak_active = std::max(peak_active, active); + cv.notify_all(); + // Hold the first wave until ten tablets are closing concurrently. + // A serial implementation times out once instead of hanging the test. + if (!cv.wait_for(lock, std::chrono::seconds(5), + [&] { return started >= 10 || timed_out; })) { + timed_out = true; + cv.notify_all(); + } + --active; + return tablet_id % 2 == 0 + ? Status::OK() + : Status::InternalError("injected tablet close failure"); + }); + writer->_rowset_writer = writer->_rowset_builder->rowset_writer(); + writer->_is_init = true; + tablet->_load_stream_writer = std::move(writer); + index._tablet_streams_map.emplace(tablet_id, std::move(tablet)); + } + + std::vector success_tablets; + FailedTablets failed_tablets; + index.close({}, &success_tablets, &failed_tablets); + + EXPECT_FALSE(timed_out); + EXPECT_EQ(peak_active, 10); + EXPECT_EQ(started, tablet_count); + std::sort(success_tablets.begin(), success_tablets.end()); + std::vector expected_success; + for (int64_t id = 2; id <= tablet_count; id += 2) { + expected_success.push_back(id); + } + EXPECT_EQ(success_tablets, expected_success); + std::vector failed_ids; + for (const auto& [id, st] : failed_tablets) { + failed_ids.push_back(id); + EXPECT_NE(st.to_string().find("injected tablet close failure"), std::string::npos); + } + std::sort(failed_ids.begin(), failed_ids.end()); + std::vector expected_failed; + for (int64_t id = 1; id <= tablet_count; id += 2) { + expected_failed.push_back(id); + } + EXPECT_EQ(failed_ids, expected_failed); +} + +} // namespace doris diff --git a/be/test/storage/rowset/beta_rowset_test.cpp b/be/test/storage/rowset/beta_rowset_test.cpp index c6c1d3e4343722..7f6d39d0fa7497 100644 --- a/be/test/storage/rowset/beta_rowset_test.cpp +++ b/be/test/storage/rowset/beta_rowset_test.cpp @@ -60,6 +60,8 @@ #include "storage/segment/segment.h" #include "storage/storage_engine.h" #include "storage/storage_policy.h" +#include "storage/tablet/tablet.h" +#include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_schema.h" #include "storage/utils.h" #include "util/s3_util.h" @@ -244,6 +246,39 @@ class BetaRowsetWriterForTest : public BetaRowsetWriter { } }; +TEST_F(BetaRowsetTest, StreamedBitmapSnapshotFailurePreventsQueueing) { + class SnapshotFailureWriter : public BetaRowsetWriter { + public: + explicit SnapshotFailureWriter(StorageEngine& engine) : BetaRowsetWriter(engine) {} + + protected: + Status _build_rowset_meta(RowsetMeta*, bool, std::vector*) override { + return Status::InternalError("streamed metadata snapshot failed"); + } + }; + + EngineOptions options; + StorageEngine engine(options); + auto tablet_meta = std::make_shared(); + tablet_meta->set_enable_unique_key_merge_on_write(true); + SnapshotFailureWriter writer(engine); + auto tablet_schema = std::make_shared(); + create_tablet_schema(tablet_schema); + RowsetWriterContext writer_context; + create_rowset_writer_context(tablet_schema, &writer_context); + ASSERT_TRUE(writer.init(writer_context).ok()); + writer._context.tablet = std::make_shared(engine, tablet_meta, nullptr); + writer._context.mow_context = std::make_shared( + 0, 1, std::make_shared(), std::vector {}, + std::make_shared(0)); + // No owned file writer: LoadStreamWriter has already closed the streamed file. + // Snapshot errors must be returned before touching the asynchronous executor. + ASSERT_EQ(writer._calc_delete_bitmap_token, nullptr); + auto st = writer._generate_delete_bitmap(0); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("streamed metadata snapshot failed"), std::string::npos); +} + class S3ClientMock : public Aws::S3::S3Client { S3ClientMock() {} S3ClientMock(const Aws::Auth::AWSCredentials& credentials, diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index 3c43e26f971048..99cd1439fb64e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -379,7 +379,7 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme } } } - if (isPartialUpdate() || hasInvertedIndexV1 || Config.isCloudMode()) { + if (isPartialUpdate() || hasInvertedIndexV1) { isEnableMemtableOnSinkNode = false; } diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 55c64d302a03a5..9d9bd1a496a2f9 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -1020,6 +1020,9 @@ message POpenLoadStreamRequest { optional bool enable_profile = 6 [default = false]; optional int64 total_streams = 7; optional int64 idle_timeout_ms = 8; + optional int64 txn_expiration = 9; + optional string storage_vault_id = 10; + optional bool write_file_cache = 11; } message PTabletSchemaWithIndex { diff --git a/regression-test/data/cloud_p0/test_cloud_memtable_mow_forward.out b/regression-test/data/cloud_p0/test_cloud_memtable_mow_forward.out new file mode 100644 index 00000000000000..6f0e9ba4e9beb3 --- /dev/null +++ b/regression-test/data/cloud_p0/test_cloud_memtable_mow_forward.out @@ -0,0 +1,109 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !initial_false -- +0 119960 11997 +1 119970 11998 +2 119980 11999 +3 119990 12000 + +-- !index_false -- +1 119970 11998 +3 119990 12000 + +-- !layout_false -- +true false true true true true + +-- !deleted_false -- +0 119960 11997 +1 119970 11998 +3 119990 12000 + +-- !stream_status_false -- +Success + +-- !stream_false -- +0 150000 15000 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !group_status_false -- +Success true + +-- !group_false -- +0 150000 15000 +1 160000 16000 +2 140000 14000 +3 119990 12000 + +-- !compacted_false -- +0 150000 15000 +1 160000 16000 +2 140000 14000 +3 119990 12000 + +-- !initial_true -- +0 119960 11997 +1 119970 11998 +2 119980 11999 +3 119990 12000 + +-- !index_true -- +1 119970 11998 +3 119990 12000 + +-- !layout_true -- +true true true true true true + +-- !deleted_true -- +0 119960 11997 +1 119970 11998 +3 119990 12000 + +-- !stream_status_true -- +Success + +-- !stream_true -- +0 150000 15000 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !group_status_true -- +Success true + +-- !group_true -- +0 150000 15000 +1 160000 16000 +2 140000 14000 +3 119990 12000 + +-- !compacted_true -- +0 150000 15000 +1 160000 16000 +2 140000 14000 +3 119990 12000 + +-- !mow_fallback -- +1 20 + +-- !broker_rows -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 100 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + diff --git a/regression-test/data/cloud_p0/test_cloud_memtable_on_sink.out b/regression-test/data/cloud_p0/test_cloud_memtable_on_sink.out new file mode 100644 index 00000000000000..6948612c886f8f --- /dev/null +++ b/regression-test/data/cloud_p0/test_cloud_memtable_on_sink.out @@ -0,0 +1,148 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !s3_rows_false -- +20 1133 2266 + +-- !s3_index_false -- +31 62 +50 100 +50 100 +57 114 + +-- !s3_packed_meta_false -- +6 20 2 1 1 0 0,1,2,3,4,5 + +-- !stream_rows -- +22 1334 2668 + +-- !agg_false -- +0 17994000 0 11996 3000 1 10 20 +1 17997000 1 11997 3000 1 11 21 +2 18000000 2 11998 3000 1 12 22 +3 18003000 3 11999 3000 1 13 23 + +-- !mor_false -- +0 0 +1 10 +2 20 +3 30 + +-- !seq_false -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !seq_index_false -- +1 119970 11997 +3 119990 11999 + +-- !seq_lower_false -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !mor_new_version_false -- +0 99 +1 10 +2 20 +3 30 + +-- !agg_new_version_false -- +0 17994010 -1 13000 3000 1 99 20 +1 17997000 1 11997 3000 1 11 21 +2 18000000 2 11998 3000 1 12 22 +3 18003000 3 11999 3000 1 13 23 + +-- !seq_empty_false -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !agg_compacted_false -- +0 17994010 -1 13000 3000 1 99 20 +1 17997000 1 11997 3000 1 11 21 +2 18000000 2 11998 3000 1 12 22 +3 18003000 3 11999 3000 1 13 23 + +-- !mor_compacted_false -- +0 99 +1 10 +2 20 +3 30 + +-- !seq_compacted_false -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !agg_broker_false -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 200 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + +-- !mor_broker_false -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 100 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + +-- !agg_stream -- +50 203 + +-- !mor_stream -- +0 119960 11996 +1 130010 13001 +2 119980 11998 +3 119990 11999 + +-- !mor_delete -- +0 119960 11996 +1 130010 13001 +3 119990 11999 + +-- !mor_reinsert -- +0 119960 11996 +1 130010 13001 +2 150000 15000 +3 119990 11999 + +-- !mor_final_compacted -- +0 119960 11996 +1 130010 13001 +2 150000 15000 +3 119990 11999 + diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_mow_forward.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_mow_forward.groovy new file mode 100644 index 00000000000000..1b84ca7214a838 --- /dev/null +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_mow_forward.groovy @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.suite.client.BackendClientImpl +import org.apache.doris.thrift.TGetRealtimeExecStatusRequest +import org.apache.doris.thrift.TNetworkAddress +import org.apache.doris.thrift.TStatusCode +import org.apache.doris.thrift.TUniqueId + +suite("test_cloud_memtable_mow_forward", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(3) + options.enableDebugPoints() + options.beConfigs += ['small_file_threshold_bytes=1048576', + 'enable_merge_on_write_correctness_check=true'] + options.feConfigs += ['stream_load_default_memtable_on_sink_node=true'] + docker(options) { + def backends = sql_return_maparray("SHOW BACKENDS") + def awaitGroupProfile = { label, tabletId -> + def idParts = label.substring('group_commit_'.length()).split('_') + def queryId = new TUniqueId(Long.parseUnsignedLong(idParts[0], 16), + Long.parseUnsignedLong(idParts[1], 16)) + def request = new TGetRealtimeExecStatusRequest().setId(queryId).setReqType("profile") + awaitUntil(60) { + backends.any { be -> + def backend = new BackendClientImpl( + new TNetworkAddress(be.Host, be.BePort as int), be.HttpPort as int) + try { + backend.client.inputProtocol.transport.setTimeout(5000) + def response = backend.client.getRealtimeExecStatus(request) + if (response.status.statusCode == TStatusCode.NOT_FOUND) { + return false + } + if (response.status.statusCode != TStatusCode.OK) { + throw new IllegalStateException("Get BE profile failed: ${response.status}") + } + def nodes = response.reportExecStatusParams.queryProfile.fragmentIdToProfile + .values().flatten().collectMany { it.profile.nodes } + return nodes.any { it.name == "DeltaWriterV2 ${tabletId}" } + } finally { + backend.close() + } + } + } + } + def checkUniqueKeys = { + sql """ + SELECT assert_true(COUNT(*) = 0, 'cloud_mow_forward contains duplicate keys') FROM ( + SELECT k, COUNT(*) AS a FROM cloud_mow_forward GROUP BY k HAVING a > 1 + ) duplicates + """ + } + + sql "DROP TABLE IF EXISTS cloud_mow_forward_source" + sql """ + CREATE TABLE cloud_mow_forward_source (n BIGINT NOT NULL) + DUPLICATE KEY(n) DISTRIBUTED BY HASH(n) BUCKETS 12 + PROPERTIES ("replication_num"="1") + """ + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_mow_forward_source SELECT number FROM numbers('number'='12000')" + sql "SET parallel_pipeline_task_num=4" + sql "SET profile_level=2" + sql "SET enable_sql_cache=false" + sql "SET enable_file_cache=false" + [false, true].each { packed -> + setBeConfigTemporary(['enable_packed_file': packed.toString()]) { + sql "DROP TABLE IF EXISTS cloud_mow_forward" + sql """ + CREATE TABLE cloud_mow_forward ( + k BIGINT NOT NULL, v BIGINT, seq BIGINT NOT NULL, + INDEX idx_k(k) USING INVERTED + ) UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "function_column.sequence_col"="seq", "inverted_index_storage_format"="V2", + "disable_auto_compaction"="true", "group_commit_interval_ms"="200") + """ + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_mow_forward VALUES (0,-1,-1),(1,-1,-1),(2,-1,-1),(3,-1,-1)" + sql "INSERT INTO cloud_mow_forward VALUES (0,0,0),(1,0,0),(2,0,0),(3,0,0)" + sql "SET enable_memtable_on_sink_node=true" + sql "SET enable_profile=true" + sql """ + /* cloud_mow_forward_${packed} */ + INSERT INTO cloud_mow_forward SELECT n%4,n*10,n+1 FROM cloud_mow_forward_source + """ + new ProfileAction(context).getProfileBySql("cloud_mow_forward_${packed}", ["DeltaWriterV2"]) + sql "SET enable_profile=false" + quickTest("initial_${packed}", "SELECT * FROM cloud_mow_forward", true) + checkUniqueKeys() + quickTest("index_${packed}", "SELECT * FROM cloud_mow_forward WHERE k IN (1,3)", true) + def tablet = sql_return_maparray("SHOW TABLETS FROM cloud_mow_forward")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM cloud_mow_forward")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + code, body -> + assertEquals(200, code) + def meta = parseJson(body) + quickTest("layout_${packed}", """ + SELECT ${meta.num_segments as int} > 1, + ${(meta.packed_slice_locations ?: [:]).size() > 0}, + ${meta.segments_file_size.size() == (meta.num_segments as int)}, + ${meta.segments_file_size.every { (it as long) > 0 }}, + ${meta.inverted_index_file_info.size() == (meta.num_segments as int)}, + ${meta.inverted_index_file_info.every { (it.index_size as long) > 0 }} + """, true) + } + sql "INSERT INTO cloud_mow_forward VALUES (0,-1,1)" + sql "INSERT INTO cloud_mow_forward (k,v,seq,__DORIS_DELETE_SIGN__) VALUES (2,130000,13000,1)" + sql "INSERT INTO cloud_mow_forward VALUES (2,-1,1)" + quickTest("deleted_${packed}", "SELECT * FROM cloud_mow_forward", true) + sql "INSERT INTO cloud_mow_forward VALUES (2,140000,14000)" + streamLoad { + table "cloud_mow_forward" + set "column_separator", "," + set "memtable_on_sink_node", "true" + set "group_commit", "off_mode" + inputText "0,150000,15000\n3,-1,1\n" + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + quickTest("stream_status_${packed}", "SELECT '${response.Status}'", true) + } + } + quickTest("stream_${packed}", "SELECT * FROM cloud_mow_forward", true) + def originalEnableProfile = sql("SHOW GLOBAL VARIABLES LIKE 'enable_profile'")[0][1] + def originalProfileLevel = sql("SHOW GLOBAL VARIABLES LIKE 'profile_level'")[0][1] + def profileBlock = "VTabletWriterV2.close.profile_ready" + def groupLabel = "" + try { + sql "SET GLOBAL enable_profile=true" + sql "SET GLOBAL profile_level=2" + GetDebugPoint().enableDebugPointForAllBEs(profileBlock, [timeout: "120"]) + streamLoad { + table "cloud_mow_forward" + set "column_separator", "," + set "group_commit", "async_mode" + set "memtable_on_sink_node", "false" + unset "label" + inputText "1,160000,16000\n" + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + groupLabel = response.Label + quickTest("group_status_${packed}", "SELECT '${response.Status}', '${response.GroupCommit}'", true) + } + } + awaitGroupProfile(groupLabel, tablet.TabletId) + } finally { + GetDebugPoint().disableDebugPointForAllBEs(profileBlock) + sql "SET GLOBAL enable_profile=${originalEnableProfile}" + sql "SET GLOBAL profile_level=${originalProfileLevel}" + } + awaitUntil(60) { + (sql "SELECT v FROM cloud_mow_forward WHERE k=1")[0][0] == 160000 + } + quickTest("group_${packed}", "SELECT * FROM cloud_mow_forward", true) + checkUniqueKeys() + trigger_and_wait_compaction("cloud_mow_forward", "full") + quickTest("compacted_${packed}", "SELECT * FROM cloud_mow_forward", true) + checkUniqueKeys() + } + } + // MOW without sink upload forwards files and calculates bitmaps on the target BE. + sql "DROP TABLE IF EXISTS cloud_memtable_mow_fallback" + sql """ + CREATE TABLE cloud_memtable_mow_fallback (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true") + """ + sql "INSERT INTO cloud_memtable_mow_fallback VALUES (1,10)" + sql "INSERT INTO cloud_memtable_mow_fallback VALUES (1,20)" + order_qt_mow_fallback "SELECT * FROM cloud_memtable_mow_fallback" + + sql "DROP TABLE IF EXISTS cloud_mow_forward_broker" + sql """ + CREATE TABLE cloud_mow_forward_broker (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "disable_auto_compaction"="true") + """ + sql "INSERT INTO cloud_mow_forward_broker VALUES (50,-1)" + def brokerLabel = "mow_forward_" + UUID.randomUUID().toString().replace('-', '_') + sql "SET enable_profile=true" + try { + sql """ + LOAD LABEL ${brokerLabel} ( + DATA INFILE("s3://${getS3BucketName()}/regression/load/data/basic_data.csv") + INTO TABLE cloud_mow_forward_broker COLUMNS TERMINATED BY "|" FORMAT AS "CSV" + (k, c01, c02, c03, c04, c05, c06, c07, c08, c09, + c10, c11, c12, c13, c14, c15, c16, c17, c18) SET (v=k*2) + ) WITH S3 ( + "AWS_ACCESS_KEY"="${getS3AK()}", "AWS_SECRET_KEY"="${getS3SK()}", + "AWS_ENDPOINT"="${getS3Endpoint()}", "AWS_REGION"="${getS3Region()}", + "provider"="${getS3Provider()}" + ) PROPERTIES ("load_parallelism"="1") + """ + waitForBrokerLoadDone(brokerLabel) + def load = sql_return_maparray("SHOW LOAD WHERE LABEL = '${brokerLabel}'")[0] + new ProfileAction(context).getProfile(load.JobId.toString(), ["DeltaWriterV2"]) + } finally { + sql "SET enable_profile=false" + } + sql """ + SELECT assert_true(COUNT(*) = 0, 'cloud_mow_forward_broker contains duplicate keys') FROM ( + SELECT k, COUNT(*) AS a FROM cloud_mow_forward_broker GROUP BY k HAVING a > 1 + ) duplicates + """ + order_qt_broker_rows "SELECT k,v FROM cloud_mow_forward_broker" + + sql "DROP TABLE IF EXISTS cloud_memtable_partial_update" + sql """ + CREATE TABLE cloud_memtable_partial_update (k INT NOT NULL, v INT, untouched INT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true") + """ + sql "INSERT INTO cloud_memtable_partial_update VALUES (1,10,100)" + streamLoad { + table "cloud_memtable_partial_update" + set "column_separator", "," + set "columns", "k,v" + set "partial_columns", "true" + set "memtable_on_sink_node", "true" + set "group_commit", "off_mode" + inputText "1,20\n" + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + sql "SELECT assert_true('${response.Status}' = 'Success', 'partial-update Stream Load failed')" + } + } + sql """ + SELECT assert_true(COUNT(*) = 1 AND SUM(v) = 20 AND SUM(untouched) = 100, 'partial update result mismatch') + FROM cloud_memtable_partial_update WHERE k = 1 + """ + } +} diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_on_sink.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_on_sink.groovy new file mode 100644 index 00000000000000..f34dcc078b3745 --- /dev/null +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_on_sink.groovy @@ -0,0 +1,405 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction +import org.apache.doris.regression.suite.ClusterOptions + +suite("test_cloud_memtable_on_sink", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(3) + options.enableDebugPoints() + // Split the small S3 CSV across all three BEs with load_parallelism = 1. + options.feConfigs += ['min_bytes_per_broker_scanner = 100'] + // The segment-count checks require the configured four-row CSV batches. + options.beConfigs += ['enable_packed_file=true', 'small_file_threshold_bytes=1048576', + 'enable_adaptive_batch_size=false'] + + docker(options) { + def loadS3 = { table, label -> + sql """ + LOAD LABEL ${label} ( + DATA INFILE("s3://${getS3BucketName()}/regression/load/data/basic_data.csv") + INTO TABLE ${table} COLUMNS TERMINATED BY "|" FORMAT AS "CSV" + (k, c01, c02, c03, c04, c05, c06, c07, c08, c09, + c10, c11, c12, c13, c14, c15, c16, c17, c18) + SET (v = k * 2) + ) WITH S3 ( + "AWS_ACCESS_KEY"="${getS3AK()}", "AWS_SECRET_KEY"="${getS3SK()}", + "AWS_ENDPOINT"="${getS3Endpoint()}", "AWS_REGION"="${getS3Region()}", + "provider"="${getS3Provider()}" + ) PROPERTIES ("load_parallelism"="1") + """ + waitForBrokerLoadDone(label) + return sql_return_maparray("SHOW LOAD WHERE LABEL = '${label}'")[0] + } + + def testDuplicate = { + sql "SET enable_sql_cache = false" + sql "DROP TABLE IF EXISTS test_cloud_duplicate_memtable_on_sink_source" + sql "DROP TABLE IF EXISTS test_cloud_duplicate_memtable_on_sink" + + sql """ + CREATE TABLE test_cloud_duplicate_memtable_on_sink_source ( + k BIGINT NOT NULL, + v BIGINT NOT NULL + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 3 + PROPERTIES ("replication_num" = "1") + """ + sql """ + CREATE TABLE test_cloud_duplicate_memtable_on_sink ( + k BIGINT NOT NULL, + v BIGINT NOT NULL + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + + sql "SET enable_memtable_on_sink_node = false" + sql """ + INSERT INTO test_cloud_duplicate_memtable_on_sink_source + SELECT number, number * 2 FROM numbers("number" = "100000") + """ + + try { + sql "SET enable_memtable_on_sink_node = true" + sql "SET profile_level = 2" + sql "SET enable_profile = true" + sql """ + /* cloud_duplicate_memtable_on_sink_profile_false */ + INSERT INTO test_cloud_duplicate_memtable_on_sink + SELECT k, v FROM test_cloud_duplicate_memtable_on_sink_source + """ + def required = ["DeltaWriterV2"] + def profileString = new ProfileAction(context).getProfileBySql( + "cloud_duplicate_memtable_on_sink_profile_false", required) + logger.info("memtable-on-sink profile:\n{}", profileString) + } finally { + sql "SET enable_profile = false" + sql "SET enable_memtable_on_sink_node = false" + } + + sql """ + SELECT assert_true( + COUNT(*) = 100000 + AND SUM(k) = 4999950000 + AND SUM(v) = 9999900000, + 'cloud duplicate memtable-on-sink result mismatch') + FROM test_cloud_duplicate_memtable_on_sink + """ + + sql "DROP TABLE IF EXISTS test_cloud_duplicate_memtable_on_sink_s3" + sql """ + CREATE TABLE test_cloud_duplicate_memtable_on_sink_s3 ( + k BIGINT NOT NULL, + v BIGINT NOT NULL, + INDEX idx_v (v) USING INVERTED + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2", + "disable_auto_compaction" = "true" + ) + """ + + def label = "cloud_duplicate_memtable_on_sink_s3_" + UUID.randomUUID().toString().replace('-', '_') + try { + sql "SET enable_memtable_on_sink_node = true" + sql "SET enable_profile = true" + // The three scanners read 6, 6, and 8 rows. Flush after the first four rows on + // each BE, then flush the remaining rows at close to produce two segments each. + sql "SET broker_load_batch_size = 4" + GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush", [execute: 1]) + def load = loadS3("test_cloud_duplicate_memtable_on_sink_s3", label) + assertEquals("FINISHED", load.State, "S3 load did not finish: ${load}") + + def profileString = new ProfileAction(context).getProfile( + load.JobId.toString(), ["DeltaWriterV2", "NumScanners"]) + logger.info("S3 memtable-on-sink profile:\n{}", profileString) + + // Only inspect per-BE pipelines, excluding the merged profile's duplicate counters. + def pipelines = profileString.split(/(?m)(?=^[ \t]*(?:Pipeline \d+|FragmentLevelProfile:)\(host=)/) + .findAll { it.trim().startsWith("Pipeline ") } + def backends = sql_return_maparray("SHOW BACKENDS") + backends.each { backend -> + def pipeline = pipelines.find { + it.readLines()[0].contains("hostname:${backend.Host},") && it.contains("DeltaWriterV2") + } + assertNotNull(pipeline, "Missing S3 sink writer on BE ${backend.Host}") + assertTrue(pipeline.contains("FILE_SCAN_OPERATOR"), "Missing S3 scanner on BE ${backend.Host}") + assertTrue((pipeline =~ /(?m)^\s*- NumScanners: 1\s*$/).find(), + "Expected one S3 scanner on BE ${backend.Host}") + assertTrue((pipeline =~ /(?m)^\s*- SegmentNum: 2\s*$/).find(), + "Expected two flushed segments on BE ${backend.Host}") + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs("MemTable.need_flush") + sql "SET enable_profile = false" + sql "SET enable_memtable_on_sink_node = false" + } + + // Read both segment data and V2 indexes without the uploader's file cache. + sql "SET enable_file_cache = false" + quickTest("s3_rows_false", """ + SELECT COUNT(*), SUM(k), SUM(v) + FROM test_cloud_duplicate_memtable_on_sink_s3 + """, true) + quickTest("s3_index_false", """ + SELECT k, v FROM test_cloud_duplicate_memtable_on_sink_s3 WHERE v IN (62, 100, 114) + """, true) + def tablet = sql_return_maparray("SHOW TABLETS FROM test_cloud_duplicate_memtable_on_sink_s3")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM test_cloud_duplicate_memtable_on_sink_s3")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + responseCode, body -> + assertEquals(200, responseCode) + logger.info("S3 memtable-on-sink rowset meta: {}", body) + def rowsetMeta = parseJson(body) + def locations = rowsetMeta.packed_slice_locations + def segmentIds = rowsetMeta.segment_ids ?: (0..<(rowsetMeta.num_segments as int)).toList() + logger.info("S3 forwarded rowset layout: {}", [ + tablet_id: tablet.TabletId, version: partition.VisibleVersion, + rowset_id: rowsetMeta.rowset_id_v2, segment_ids: segmentIds, + num_segment_rows: rowsetMeta.num_segment_rows, + segments_file_size: rowsetMeta.segments_file_size, + packed_files: locations.keySet().sort() + ]) + // Only the destination rowset's first segment and its V2 index are packed. + // The other five segments retain independent files. + quickTest("s3_packed_meta_false", """ + SELECT ${rowsetMeta.num_segments as int}, ${rowsetMeta.num_rows as long}, + ${locations.size()}, + ${locations.keySet().count { it.endsWith('_0.dat') }}, + ${locations.keySet().count { it.endsWith('_0.idx') }}, + ${segmentIds.count { (it as long) >= 1000 }}, '${segmentIds.join(",")}' + """, true) + } + + // Stream Load also uses the file-forwarding path when memtable-on-sink is enabled. + // This hook is only reached by the memtable-on-sink receiving writer. + // A successful fallback would fail this check instead of silently passing. + def failClose = "LoadStreamWriter.close.cancelled" + try { + GetDebugPoint().enableDebugPointForAllBEs(failClose) + streamLoad { + table "test_cloud_duplicate_memtable_on_sink_s3" + set "column_separator", "," + set "memtable_on_sink_node", "true" + set "group_commit", "off_mode" + inputText "100,200\n101,202\n" + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + sql "SELECT assert_true('${response.Status}' = 'Fail', 'Stream Load should fail at receiver close')" + } + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs(failClose) + } + sql """ + SELECT assert_true(COUNT(*) = 20 AND SUM(k) = 1133 AND SUM(v) = 2266, 'failed Stream Load changed visible rows') + FROM test_cloud_duplicate_memtable_on_sink_s3 + """ + + streamLoad { + table "test_cloud_duplicate_memtable_on_sink_s3" + set "column_separator", "," + set "memtable_on_sink_node", "true" + set "group_commit", "off_mode" + inputStream new ByteArrayInputStream("100,200\n101,202\n".getBytes()) + time 30000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + assertEquals("Success", parseJson(result).Status) + } + } + order_qt_stream_rows """ + SELECT COUNT(*), SUM(k), SUM(v) FROM test_cloud_duplicate_memtable_on_sink_s3 + """ + } + + def testAggregateAndMor = { + sql "DROP TABLE IF EXISTS cloud_memtable_models_source" + sql """ + CREATE TABLE cloud_memtable_models_source (n BIGINT NOT NULL) + DUPLICATE KEY(n) DISTRIBUTED BY HASH(n) BUCKETS 12 + PROPERTIES ("replication_num"="1") + """ + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_memtable_models_source SELECT number FROM numbers('number'='12000')" + sql "SET enable_memtable_on_sink_node=true" + sql "SET parallel_pipeline_task_num=4" + sql "SET profile_level=2" + sql "SET enable_file_cache=false" + + def aggQuery = """ + SELECT k, s, lo, hi, bitmap_count(b), hll_cardinality(h), r, rn + FROM cloud_memtable_agg + """ + def morQuery = "SELECT k, v FROM cloud_memtable_mor" + def seqQuery = "SELECT k, v, seq FROM cloud_memtable_mor_seq" + def loadModelS3 = { table -> + def label = "agg_mor_" + UUID.randomUUID().toString().replace('-', '_') + sql "SET enable_profile=true" + def load = loadS3(table, label) + def required = ["DeltaWriterV2"] + new ProfileAction(context).getProfile(load.JobId.toString(), required) + sql "SET enable_profile=false" + } + + sql "DROP TABLE IF EXISTS cloud_memtable_agg" + sql "DROP TABLE IF EXISTS cloud_memtable_mor" + sql "DROP TABLE IF EXISTS cloud_memtable_mor_seq" + sql "DROP TABLE IF EXISTS cloud_memtable_agg_broker" + sql "DROP TABLE IF EXISTS cloud_memtable_mor_broker" + sql """ + CREATE TABLE cloud_memtable_agg ( + k BIGINT NOT NULL, s BIGINT SUM, lo BIGINT MIN, hi BIGINT MAX, + b BITMAP BITMAP_UNION, h HLL HLL_UNION, + r BIGINT REPLACE, rn BIGINT REPLACE_IF_NOT_NULL + ) AGGREGATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true") + """ + sql """ + CREATE TABLE cloud_memtable_mor (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="false", + "disable_auto_compaction"="true") + """ + sql """ + CREATE TABLE cloud_memtable_mor_seq ( + k BIGINT NOT NULL, v BIGINT, seq BIGINT NOT NULL, + INDEX idx_k (k) USING INVERTED + ) UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="false", + "function_column.sequence_col"="seq", "inverted_index_storage_format"="V2", + "disable_auto_compaction"="true") + """ + sql "SET enable_profile=true" + sql """ + /* cloud_memtable_agg_false */ + INSERT INTO cloud_memtable_agg + SELECT n % 4, n, n, n, to_bitmap(n), hll_hash(CAST(n % 4 AS STRING)), + n % 4 + 10, IF(n % 8 < 4, n % 4 + 20, NULL) + FROM cloud_memtable_models_source + """ + sql """ + /* cloud_memtable_mor_false */ + INSERT INTO cloud_memtable_mor SELECT n % 4, n % 4 * 10 FROM cloud_memtable_models_source + """ + sql """ + /* cloud_memtable_mor_seq_false */ + INSERT INTO cloud_memtable_mor_seq SELECT n % 4, n * 10, n FROM cloud_memtable_models_source + """ + ['agg', 'mor', 'mor_seq'].each { model -> + def required = ["DeltaWriterV2"] + new ProfileAction(context).getProfileBySql("cloud_memtable_${model}_false", required) + } + sql "SET enable_profile=false" + quickTest("agg_false", aggQuery, true) + quickTest("mor_false", morQuery, true) + quickTest("seq_false", seqQuery, true) + quickTest("seq_index_false", seqQuery + " WHERE k IN (1,3)", true) + + // A newer transaction with a lower Sequence must not replace the business-newer row. + sql "INSERT INTO cloud_memtable_mor_seq VALUES (0,-1,1)" + quickTest("seq_lower_false", seqQuery, true) + sql "INSERT INTO cloud_memtable_mor VALUES (0,99)" + quickTest("mor_new_version_false", morQuery, true) + sql """ + INSERT INTO cloud_memtable_agg + VALUES (0,10,-1,13000,bitmap_empty(),hll_empty(),99,NULL) + """ + quickTest("agg_new_version_false", aggQuery, true) + sql "INSERT INTO cloud_memtable_mor_seq SELECT n % 4,n*10,n FROM cloud_memtable_models_source WHERE n<0" + quickTest("seq_empty_false", seqQuery, true) + ['agg', 'mor', 'mor_seq'].each { model -> + trigger_and_wait_compaction("cloud_memtable_${model}", "full") + } + quickTest("agg_compacted_false", aggQuery, true) + quickTest("mor_compacted_false", morQuery, true) + quickTest("seq_compacted_false", seqQuery, true) + + sql """ + CREATE TABLE cloud_memtable_agg_broker (k BIGINT NOT NULL, v BIGINT SUM) + AGGREGATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true") + """ + sql """ + CREATE TABLE cloud_memtable_mor_broker (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="false", + "disable_auto_compaction"="true") + """ + loadModelS3("cloud_memtable_agg_broker") + loadModelS3("cloud_memtable_mor_broker") + quickTest("agg_broker_false", "SELECT k,v FROM cloud_memtable_agg_broker", true) + quickTest("mor_broker_false", "SELECT k,v FROM cloud_memtable_mor_broker", true) + + streamLoad { + table "cloud_memtable_agg_broker" + set "column_separator", "," + set "memtable_on_sink_node", "true" + inputStream new ByteArrayInputStream("50,1\n50,2\n".getBytes()) + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + assertEquals("Success", parseJson(result).Status) + } + } + order_qt_agg_stream "SELECT k,v FROM cloud_memtable_agg_broker WHERE k=50" + streamLoad { + table "cloud_memtable_mor_seq" + set "column_separator", "," + set "memtable_on_sink_node", "true" + inputStream new ByteArrayInputStream("0,1,1\n1,130010,13001\n".getBytes()) + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + assertEquals("Success", parseJson(result).Status) + } + } + order_qt_mor_stream seqQuery + sql "INSERT INTO cloud_memtable_mor_seq (k,v,seq,__DORIS_DELETE_SIGN__) VALUES (2,140000,14000,1)" + order_qt_mor_delete seqQuery + sql "INSERT INTO cloud_memtable_mor_seq VALUES (2,150000,15000)" + order_qt_mor_reinsert seqQuery + trigger_and_wait_compaction("cloud_memtable_mor_seq", "full") + order_qt_mor_final_compacted seqQuery + } + + def originalBatchSize = sql("SHOW VARIABLES LIKE 'broker_load_batch_size'")[0][1] + try { + testDuplicate() + } finally { + sql "SET broker_load_batch_size=${originalBatchSize}" + sql "SET enable_profile=false" + GetDebugPoint().disableDebugPointForAllBEs("MemTable.need_flush") + } + try { + testAggregateAndMor() + } finally { + sql "SET enable_profile=false" + } + } +} From 361e1ea32783f523de6c229c69b44c8630a9be3b Mon Sep 17 00:00:00 2001 From: meiyi Date: Thu, 3 Sep 2026 10:48:41 +0800 Subject: [PATCH 4/6] [refactor](cloud) Share partial rowset writer and cache controls ### What problem does this PR solve? Problem Summary: Distributed compaction and memtable sink uploads need common rowset writer controls independently of distributed compaction orchestration. Extract these existing controls into a standalone prerequisite. Add is_partial_output_writer to retain assigned physical segment IDs for partial compaction outputs, skip writer-local segment compaction, and keep complete per-segment key bounds without aggregation or truncation. Let RowsetMeta callers explicitly disable key-bound truncation. Add disable_file_cache to override index-only and adaptive cache admission in file writer options, with the existing cache-policy unit test. This commit only extracts existing changes; the final branch source tree is unchanged. The sink upload feature subsequently enables explicit segment IDs for partial load writers as well. ### Release note None ### Check List (For Author) - Test: No compilation or runtime tests run, as requested. Clang-format 16, build hygiene, and git diff whitespace checks passed. Static application to cloud-load-2 and its previously conflicting sink-upload writer changes succeeded in an isolated index. Retained CompactionFileCacheTest.DisableFileCacheOverridesAllWritePolicies. - Behavior changed: No; extracts existing internal controls without enabling them for default writers. - Does this need documentation: No; internal prerequisite extraction. --- be/src/storage/rowset/beta_rowset_writer.cpp | 26 ++++++++++++------- be/src/storage/rowset/rowset_meta.cpp | 7 ++--- be/src/storage/rowset/rowset_meta.h | 6 +++-- be/src/storage/rowset/rowset_writer_context.h | 12 +++++++++ .../rowset/vertical_beta_rowset_writer.cpp | 2 +- .../compaction/compaction_file_cache_test.cpp | 20 ++++++++++++++ 6 files changed, 58 insertions(+), 15 deletions(-) diff --git a/be/src/storage/rowset/beta_rowset_writer.cpp b/be/src/storage/rowset/beta_rowset_writer.cpp index 8c0101594c9132..02421fbe0fa8e4 100644 --- a/be/src/storage/rowset/beta_rowset_writer.cpp +++ b/be/src/storage/rowset/beta_rowset_writer.cpp @@ -967,9 +967,11 @@ Status BaseBetaRowsetWriter::_close_file_writers() { Status BetaRowsetWriter::_close_file_writers() { RETURN_IF_ERROR(BaseBetaRowsetWriter::_close_file_writers()); - // if _segment_start_id is not zero, that means it's a transient rowset writer for - // MoW partial update, don't need to do segment compaction. - if (_segment_start_id == 0) { + // Skip writer-local segment compaction for: + // 1. A distributed partial output writer, whose physical segment IDs are constrained to a + // coordinator-assigned slot that may start at zero. + // 2. A transient MoW partial-update writer, which appends segments from a nonzero segment ID. + if (!_context.is_partial_output_writer && _segment_start_id == 0) { if (_segcompaction_worker->cancel()) { std::lock_guard lk(_is_doing_segcompaction_lock); _is_doing_segcompaction = false; @@ -1062,8 +1064,8 @@ Status BaseBetaRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool ch std::vector segment_rows; std::vector segment_ids; std::optional segments_key_bounds_truncated; - const bool record_segment_ids = - _context.write_type == DataWriteType::TYPE_COMPACTION && _segment_start_id != 0; + const bool record_segment_ids = _context.write_type == DataWriteType::TYPE_COMPACTION && + (_context.is_partial_output_writer || _segment_start_id != 0); { std::lock_guard lock(_segid_statistics_map_mutex); if (record_segment_ids) { @@ -1154,8 +1156,11 @@ Status BaseBetaRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool ch rowset_meta->set_index_disk_size(total_index_size + _total_index_size); bool aggregate_key_bounds = config::enable_aggregate_non_mow_key_bounds && !_context.enable_unique_key_merge_on_write && - !_rowset_meta->is_row_binlog(); - rowset_meta->set_segments_key_bounds(segments_encoded_key_bounds, aggregate_key_bounds); + !_rowset_meta->is_row_binlog() && + !_context.is_partial_output_writer; + rowset_meta->set_segments_key_bounds( + segments_encoded_key_bounds, aggregate_key_bounds, + /*truncate_key_bounds=*/!_context.is_partial_output_writer); // TODO write zonemap to meta rowset_meta->set_empty((num_rows_written + _num_rows_written) == 0); rowset_meta->set_creation_time(time(nullptr)); @@ -1319,8 +1324,11 @@ Status BetaRowsetWriter::_check_segment_number_limit(size_t segnum) { Status BaseBetaRowsetWriter::add_segment(uint32_t segment_id, const SegmentStatistics& segstat) { uint32_t segid_offset = segment_id - _segment_start_id; bool key_bounds_truncated = false; - SegmentStatistics stored_segstat = - copy_segment_statistics_with_truncated_key_bounds(segstat, key_bounds_truncated); + SegmentStatistics stored_segstat = segstat; + if (!_context.is_partial_output_writer) { + stored_segstat = + copy_segment_statistics_with_truncated_key_bounds(segstat, key_bounds_truncated); + } { std::lock_guard lock(_segid_statistics_map_mutex); CHECK_EQ(_segid_statistics_map.find(segment_id) == _segid_statistics_map.end(), true); diff --git a/be/src/storage/rowset/rowset_meta.cpp b/be/src/storage/rowset/rowset_meta.cpp index 123813c30c11d3..4ea5aef2db94df 100644 --- a/be/src/storage/rowset/rowset_meta.cpp +++ b/be/src/storage/rowset/rowset_meta.cpp @@ -403,7 +403,7 @@ int64_t RowsetMeta::segment_file_size_by_pos(size_t pos) const { } void RowsetMeta::set_segments_key_bounds(const std::vector& segments_key_bounds, - bool aggregate_into_single) { + bool aggregate_into_single, bool truncate_key_bounds) { _rowset_meta_pb.clear_segments_key_bounds(); bool did_aggregate = aggregate_into_single && !segments_key_bounds.empty(); if (did_aggregate) { @@ -428,8 +428,9 @@ void RowsetMeta::set_segments_key_bounds(const std::vector& segment } set_segments_key_bounds_aggregated(did_aggregate); - int32_t truncation_threshold = config::segments_key_bounds_truncation_threshold; - if (config::random_segments_key_bounds_truncation) { + int32_t truncation_threshold = + truncate_key_bounds ? config::segments_key_bounds_truncation_threshold : 0; + if (truncate_key_bounds && config::random_segments_key_bounds_truncation) { std::mt19937 generator(std::random_device {}()); std::uniform_int_distribution distribution(-10, 40); truncation_threshold = distribution(generator); diff --git a/be/src/storage/rowset/rowset_meta.h b/be/src/storage/rowset/rowset_meta.h index d140e4d891e2f6..17594629a91695 100644 --- a/be/src/storage/rowset/rowset_meta.h +++ b/be/src/storage/rowset/rowset_meta.h @@ -467,9 +467,11 @@ class RowsetMeta : public MetadataAdder { } // If `aggregate_into_single` is true, collapse per-segment bounds into a single - // [rowset_min, rowset_max] entry and mark this rowset as aggregated. + // [rowset_min, rowset_max] entry and mark this rowset as aggregated. Partial distributed + // compaction outputs can disable truncation so the coordinator can validate adjacent bounds. void set_segments_key_bounds(const std::vector& segments_key_bounds, - bool aggregate_into_single = false); + bool aggregate_into_single = false, + bool truncate_key_bounds = true); void add_segment_key_bounds(KeyBoundsPB segments_key_bounds) { *_rowset_meta_pb.add_segments_key_bounds() = std::move(segments_key_bounds); diff --git a/be/src/storage/rowset/rowset_writer_context.h b/be/src/storage/rowset/rowset_writer_context.h index 7c9d4add5ce7bc..6a04d4a04e34fd 100644 --- a/be/src/storage/rowset/rowset_writer_context.h +++ b/be/src/storage/rowset/rowset_writer_context.h @@ -129,6 +129,8 @@ struct RowsetWriterContext { /// begin file cache opts bool write_file_cache = false; + // Disable all file-cache writes. + bool disable_file_cache = false; bool is_hot_data = false; // Absolute timestamp (seconds since epoch) after which the cache blocks written by // this rowset stop being TTL protected; 0 means no TTL. Always set it from @@ -147,6 +149,9 @@ struct RowsetWriterContext { bool is_transient_rowset_writer = false; + // Preserve assigned segment IDs and skip local segment compaction for a distributed shard. + bool is_partial_output_writer = false; + segment_v2::HistoricalRowRetrieverContext make_historical_row_retriever_context(); // Intent flag: caller can actively turn merge-file feature on/off for this rowset. @@ -306,6 +311,13 @@ struct RowsetWriterContext { .file_cache_expiration_time = file_cache_expiration_time, .approximate_bytes_to_write = approximate_bytes_to_write}; + if (disable_file_cache) { + opts.write_file_cache = false; + opts.allow_adaptive_file_cache_write = false; + opts.approximate_bytes_to_write = 0; + return opts; + } + if (config::enable_file_cache_write_index_file_only) { opts.allow_adaptive_file_cache_write = false; opts.approximate_bytes_to_write = 0; diff --git a/be/src/storage/rowset/vertical_beta_rowset_writer.cpp b/be/src/storage/rowset/vertical_beta_rowset_writer.cpp index 719b90b62be1cb..17a250c0fbbc08 100644 --- a/be/src/storage/rowset/vertical_beta_rowset_writer.cpp +++ b/be/src/storage/rowset/vertical_beta_rowset_writer.cpp @@ -205,7 +205,7 @@ Status VerticalBetaRowsetWriter::build(RowsetSharedPtr& rowset) { const int32_t next_segment_id = T::get_allocated_segment_id(); const int32_t segment_num = this->_num_segment.load(std::memory_order_relaxed); DORIS_CHECK_EQ(next_segment_id - this->_segment_start_id, segment_num); - if (this->_segment_start_id != 0) { + if (this->_context.is_partial_output_writer || this->_segment_start_id != 0) { std::vector segment_ids; segment_ids.reserve(segment_num); for (int32_t segment_id = this->_segment_start_id; segment_id < next_segment_id; diff --git a/be/test/storage/compaction/compaction_file_cache_test.cpp b/be/test/storage/compaction/compaction_file_cache_test.cpp index 765df31bf1c7b3..592c25390894f5 100644 --- a/be/test/storage/compaction/compaction_file_cache_test.cpp +++ b/be/test/storage/compaction/compaction_file_cache_test.cpp @@ -347,6 +347,26 @@ TEST_F(CompactionFileCacheTest, GlobalIndexFileOnlyTakesPrecedenceOverCompaction EXPECT_EQ(index_opts.approximate_bytes_to_write, 0); } +TEST_F(CompactionFileCacheTest, DisableFileCacheOverridesAllWritePolicies) { + config::enable_file_cache_write_index_file_only = true; + + RowsetWriterContext ctx; + ctx.write_file_cache = true; + ctx.disable_file_cache = true; + ctx.compaction_output_write_index_only = true; + ctx.approximate_bytes_to_write = 12345; + + auto segment_opts = ctx.get_file_writer_options(FileType::SEGMENT_FILE); + EXPECT_FALSE(segment_opts.write_file_cache); + EXPECT_FALSE(segment_opts.allow_adaptive_file_cache_write); + EXPECT_EQ(segment_opts.approximate_bytes_to_write, 0); + + auto index_opts = ctx.get_file_writer_options(FileType::INVERTED_INDEX_FILE); + EXPECT_FALSE(index_opts.write_file_cache); + EXPECT_FALSE(index_opts.allow_adaptive_file_cache_write); + EXPECT_EQ(index_opts.approximate_bytes_to_write, 0); +} + // ============================================================================ // Tests for should_enable_compaction_cache_index_only function // ============================================================================ From 68897854e6f06d0fbf6e89ed9f42b804591a3eb6 Mon Sep 17 00:00:00 2001 From: meiyi Date: Mon, 21 Sep 2026 14:06:57 +0800 Subject: [PATCH 5/6] [fix](be) Count completed segments when building temporary rowset metadata ### What problem does this PR solve? Problem Summary: Temporary rowsets collect completed segment IDs from a statistics snapshot, but metadata construction used the consecutive segment count from _num_seg(). Out-of-order completion can make these counts differ. When segment IDs are recorded, such as for partial output writers, the size check can fail before _build_tmp sets the completed segment IDs and corrects the count. Use the completed-ID snapshot size for temporary rowsets, while retaining the existing count for final rowset construction. Extend TmpRowsetUsesCompletedSegmentIds with is_partial_output_writer=true to cover the mismatch using completed segments 2 and 6. Keep the streamed bitmap metadata snapshot change in the following Cloud forwarding feature. ### Release note Fix a segment-count check failure when constructing temporary rowset metadata with explicit segment IDs after out-of-order segment completion. ### Check List (For Author) - Test: Existing TmpRowsetUsesCompletedSegmentIds test extension moved with the fix. No compilation or tests run, per user request. Static review, repository clang-format v16 checks of the extracted changes, and git diff --check passed. Final code tree verified identical before and after splitting. - Behavior changed: Yes; temporary rowset metadata uses the completed segment count instead of the consecutive segment count. - Does this need documentation: No --- be/src/storage/rowset/beta_rowset_writer.cpp | 4 +++- be/test/storage/rowset/beta_rowset_test.cpp | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/be/src/storage/rowset/beta_rowset_writer.cpp b/be/src/storage/rowset/beta_rowset_writer.cpp index 02421fbe0fa8e4..daec6547d644c1 100644 --- a/be/src/storage/rowset/beta_rowset_writer.cpp +++ b/be/src/storage/rowset/beta_rowset_writer.cpp @@ -1125,7 +1125,9 @@ Status BaseBetaRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool ch rowset_meta->set_segments_overlap(OVERLAPPING); } - auto segment_num = _num_seg(); + // A temporary rowset contains only completed segments, which may finish out of order. + auto segment_num = + completed_segment_ids != nullptr ? completed_segment_ids->size() : _num_seg(); if (check_segment_num && config::check_segment_when_build_rowset_meta) { auto segments_encoded_key_bounds_size = segments_encoded_key_bounds.size(); if (segments_encoded_key_bounds_size != segment_num) { diff --git a/be/test/storage/rowset/beta_rowset_test.cpp b/be/test/storage/rowset/beta_rowset_test.cpp index 7f6d39d0fa7497..8b5ad55be446c8 100644 --- a/be/test/storage/rowset/beta_rowset_test.cpp +++ b/be/test/storage/rowset/beta_rowset_test.cpp @@ -559,6 +559,8 @@ TEST_F(BetaRowsetTest, TmpRowsetUsesCompletedSegmentIds) { RowsetWriterContext writer_context; create_rowset_writer_context(tablet_schema, &writer_context); + writer_context.is_partial_output_writer = true; + EngineOptions options; StorageEngine engine(options); BetaRowsetWriterForTest writer(engine); From 8bbd7a4e451cf2496a77c3af3d6b6fc51e8ed5eb Mon Sep 17 00:00:00 2001 From: meiyi Date: Tue, 15 Sep 2026 17:17:39 +0800 Subject: [PATCH 6/6] [feature](cloud) Upload memtable sink segments directly to object storage ### What problem does this PR solve? Problem Summary: Cloud memtable-on-sink file forwarding routes segment and index bytes through the target BE. Allow sink BEs to upload these files directly to object storage while retaining file forwarding when disabled. Register sink writers with the target BE, allocate stable segment ID ranges, and return the storage, encryption, and cache context via GET_WRITE_CONTEXT. Writers report completed metadata through ADD_PARTIAL_ROWSET. Validate and assemble their metadata, preserving sparse segment IDs, packed mappings, key bounds, and merged VARIANT schemas. Distribute a target-owned snapshot for MOW bitmap calculation, then validate and merge sink results before commit-phase bitmap work. Preserve target-local caching and asynchronously warm uploaded rowsets on the target BE. Propagate enable_cloud_memtable_sink_upload through SQL and Broker Load. Use cloud_stream_load_default_memtable_sink_upload for HTTP Stream, Routine Load, Group Commit internal loads, and ordinary Stream Load requests without a cloud_memtable_sink_upload header. An explicit header overrides that configuration. Both the session option and FE configuration default to true; the upload option applies only when memtable-on-sink is selected. Leave the optional Thrift request field without a default so generated C++ requests preserve absence and allow FE to resolve the configured default. Extend unit and regression coverage for configuration inheritance, header overrides, metadata assembly, MOW, cache controls, independent writers, Group Commit, packed files, VARIANT, and DUP/AGG/MOR loads. Restore the source commit's generated outputs for the reintroduced upload test branches. Check C++ request presence for omitted and explicit upload options, and fail the MOW forwarding regression on Stream Load errors even during output generation. ### Release note Cloud memtable-on-sink loads can upload segment and index files directly from sink BEs and asynchronously warm the target BE cache. SQL uses session variable enable_cloud_memtable_sink_upload. Stream Load defaults are controlled by cloud_stream_load_default_memtable_sink_upload and can be overridden per ordinary Stream Load request using cloud_memtable_sink_upload. These upload options default to true; the Stream Load memtable-on-sink default remains off. ### Check List (For Author) - Test: No local compilation or runtime tests run, as requested. Repository clang-format 16, build hygiene, and source git diff --check passed for the changes. Inspected the user's remote test_cloud_memtable_on_sink run: the suite completed successfully and generated output matched HEAD. The new request-presence unit test has not been run. This squash preserves the complete code tree unchanged. - Behavior changed: Yes; eligible cloud loads can upload sink-built files directly, distribute MOW bitmap work, and warm the target cache. Ordinary Stream Load without an upload header inherits the FE configuration. - Does this need documentation: Yes; cloud_memtable.md and cloud_memtable_on_sink.md are maintained on the source development branch and excluded from this cherry-pick as requested. --- be/src/cloud/cloud_rowset_builder.cpp | 225 +++++++ be/src/cloud/cloud_rowset_builder.h | 21 + be/src/cloud/cloud_rowset_writer.cpp | 13 + be/src/cloud/cloud_rowset_writer.h | 2 + be/src/cloud/cloud_warm_up_manager.cpp | 29 +- be/src/cloud/cloud_warm_up_manager.h | 8 +- be/src/exec/sink/load_stream_map_pool.cpp | 9 + be/src/exec/sink/load_stream_map_pool.h | 4 +- be/src/exec/sink/load_stream_stub.cpp | 65 +- be/src/exec/sink/load_stream_stub.h | 11 + be/src/exec/sink/writer/vtablet_writer_v2.cpp | 2 + be/src/load/channel/load_stream.cpp | 54 +- be/src/load/channel/load_stream.h | 9 +- be/src/load/channel/load_stream_writer.cpp | 158 ++++- be/src/load/channel/load_stream_writer.h | 13 + .../load/delta_writer/delta_writer_context.h | 1 + be/src/load/delta_writer/delta_writer_v2.cpp | 139 +++- be/src/load/delta_writer/delta_writer_v2.h | 9 +- be/src/service/http/action/stream_load.cpp | 4 + be/src/service/http/http_common.h | 1 + be/src/storage/olap_common.h | 3 + be/src/storage/rowset/beta_rowset_writer.cpp | 24 +- be/src/storage/rowset/beta_rowset_writer.h | 16 + be/src/storage/rowset/beta_rowset_writer_v2.h | 2 - be/src/storage/rowset/rowset_writer_context.h | 1 + .../segment_index_file_cache_loader.cpp | 3 +- be/test/cloud/cloud_warm_up_manager_test.cpp | 30 +- .../channel/load_stream_stub_map_test.cpp | 13 + be/test/runtime/load_stream_test.cpp | 247 +++++++ be/test/service/http/stream_load_test.cpp | 11 + ...cloud_file_cache_write_index_only_test.cpp | 29 + .../java/org/apache/doris/common/Config.java | 5 + .../doris/load/loadv2/BrokerLoadJob.java | 2 + .../apache/doris/load/loadv2/BulkLoadJob.java | 2 + .../doris/load/loadv2/LoadLoadingTask.java | 6 + .../nereids/load/NereidsLoadTaskInfo.java | 5 + .../load/NereidsStreamLoadPlanner.java | 1 + .../nereids/load/NereidsStreamLoadTask.java | 12 + .../org/apache/doris/qe/SessionVariable.java | 6 + .../doris/service/FrontendServiceImpl.java | 1 + .../doris/load/loadv2/BrokerLoadJobTest.java | 12 +- .../doris/planner/StreamLoadPlannerTest.java | 30 + gensrc/proto/internal_service.proto | 32 + gensrc/thrift/FrontendService.thrift | 1 + gensrc/thrift/PaloInternalService.thrift | 1 + .../test_cloud_memtable_group_commit.out | 13 + .../cloud_p0/test_cloud_memtable_on_sink.out | 130 ++++ .../test_cloud_memtable_sink_upload_cache.out | 41 ++ .../test_cloud_memtable_sink_upload_mow.out | 147 +++++ ...st_cloud_memtable_sink_upload_unshared.out | 13 + .../test_cloud_memtable_group_commit.groovy | 115 ++++ .../test_cloud_memtable_mow_forward.groovy | 320 ++++----- .../test_cloud_memtable_on_sink.groovy | 608 +++++++++++------- ...st_cloud_memtable_sink_upload_cache.groovy | 134 ++++ ...test_cloud_memtable_sink_upload_mow.groovy | 282 ++++++++ ...cloud_memtable_sink_upload_unshared.groovy | 107 +++ 56 files changed, 2723 insertions(+), 459 deletions(-) create mode 100644 regression-test/data/cloud_p0/test_cloud_memtable_group_commit.out create mode 100644 regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_cache.out create mode 100644 regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_mow.out create mode 100644 regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_unshared.out create mode 100644 regression-test/suites/cloud_p0/test_cloud_memtable_group_commit.groovy create mode 100644 regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_cache.groovy create mode 100644 regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_mow.groovy create mode 100644 regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_unshared.groovy diff --git a/be/src/cloud/cloud_rowset_builder.cpp b/be/src/cloud/cloud_rowset_builder.cpp index 7d363aea2d1aff..7fc272032bb5e0 100644 --- a/be/src/cloud/cloud_rowset_builder.cpp +++ b/be/src/cloud/cloud_rowset_builder.cpp @@ -17,18 +17,24 @@ #include "cloud/cloud_rowset_builder.h" +#include +#include + #include #include "cloud/cloud_meta_mgr.h" +#include "cloud/cloud_rowset_writer.h" #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" #include "cloud/cloud_tablet_mgr.h" +#include "exec/common/variant_util.h" #include "io/fs/file_system.h" #include "storage/rowset/group_rowset_writer.h" #include "storage/rowset/rowset_factory.h" #include "storage/rowset/rowset_writer_context.h" #include "storage/storage_policy.h" #include "storage/tablet_info.h" +#include "util/defer_op.h" namespace doris { using namespace ErrorCode; @@ -249,6 +255,225 @@ Status CloudRowsetBuilder::commit_rowset(const std::string& job_id, int64_t tabl return _engine.meta_mgr().commit_rowset(*rowset_meta(), job_id, table_id); } +Status CloudRowsetBuilder::get_mow_snapshot_for_sink(PCloudLoadMowSnapshot* snapshot) { + DORIS_CHECK(_tablet->enable_unique_key_merge_on_write()); + if (_mow_snapshot_for_sink == nullptr) { + RETURN_IF_ERROR(cloud_tablet()->sync_rowsets()); + auto context = _rowset_writer->context().mow_context; + DORIS_CHECK(context != nullptr); + DeleteBitmap snapshot_bitmap(_tablet->tablet_id()); + { + std::unique_lock sync_lock(cloud_tablet()->get_sync_meta_lock()); + std::shared_lock lock(_tablet->get_header_lock()); + if (_tablet->tablet_state() != TABLET_RUNNING) { + return Status::NotSupported("sink MOW load requires a running tablet {}", + _tablet->tablet_id()); + } + _max_version_in_flush_phase = _tablet->max_version_unlocked(); + _rowset_ids->clear(); + RETURN_IF_ERROR(_tablet->get_all_rs_id_unlocked(_max_version_in_flush_phase, + _rowset_ids.get())); + context->max_version = _max_version_in_flush_phase; + context->rowset_ptrs = _tablet->get_rowset_by_ids(_rowset_ids.get()); + std::vector rowset_segments; + for (const auto& rowset : context->rowset_ptrs) { + std::vector ids; + for (auto segment : rowset->segments()) { + ids.push_back(cast_set(segment.id())); + } + rowset_segments.emplace_back(rowset->rowset_id(), std::move(ids)); + } + _tablet->tablet_meta()->delete_bitmap().subset_and_agg( + rowset_segments, 0, _max_version_in_flush_phase, &snapshot_bitmap); + } + _mow_snapshot_for_sink = std::make_unique(); + _mow_snapshot_for_sink->set_version(_max_version_in_flush_phase); + // Keep the rowset references in MowContext until the load has finished. + for (const auto& rowset : context->rowset_ptrs) { + auto* meta = _mow_snapshot_for_sink->add_rowsets(); + *meta = rowset->rowset_meta()->get_rowset_pb(); + meta->clear_tablet_schema(); + rowset->tablet_schema()->to_schema_pb(meta->mutable_tablet_schema()); + } + *_mow_snapshot_for_sink->mutable_delete_bitmap() = snapshot_bitmap.to_pb(); + } + *snapshot = *_mow_snapshot_for_sink; + DBUG_EXECUTE_IF("CloudRowsetBuilder.sink_mow.snapshot_ready", { + // Expose readiness while blocked, independent of asynchronous log flushing. + static bvar::Adder waiters("cloud_memtable_mow_snapshot_waiters"); + waiters << 1; + Defer release([] { waiters << -1; }); + DBUG_BLOCK; + }); + return Status::OK(); +} + +Status CloudRowsetBuilder::validate_sink_mow_result(const PCloudLoadMowResult& result, + int64_t snapshot_version) { + if (!result.has_snapshot_version() || result.snapshot_version() != snapshot_version || + !result.has_delete_bitmap()) { + return Status::InvalidArgument("missing or mismatched sink MOW snapshot result"); + } + const auto& bitmap = result.delete_bitmap(); + const auto count = bitmap.rowset_ids_size(); + if (bitmap.segment_ids_size() != count || bitmap.versions_size() != count || + bitmap.segment_delete_bitmaps_size() != count) { + return Status::InvalidArgument("misaligned sink MOW delete bitmap"); + } + for (int pos = 0; pos < count; ++pos) { + if (bitmap.versions(pos) != DeleteBitmap::TEMP_VERSION_COMMON) { + return Status::InvalidArgument("sink MOW bitmap must use the temporary version"); + } + const auto& bytes = bitmap.segment_delete_bitmaps(pos); + const auto size = + roaring::api::roaring_bitmap_portable_deserialize_size(bytes.data(), bytes.size()); + if (size == 0 || size != bytes.size()) { + return Status::Corruption("invalid serialized sink MOW bitmap"); + } + } + return Status::OK(); +} + +Status CloudRowsetBuilder::merge_sink_mow_bitmap(const PCloudLoadMowResult& result) { + DORIS_CHECK(_mow_snapshot_for_sink != nullptr); + RETURN_IF_ERROR(validate_sink_mow_result(result, _mow_snapshot_for_sink->version())); + _delete_bitmap->merge(DeleteBitmap::from_pb(result.delete_bitmap(), _tablet->tablet_id())); + return Status::OK(); +} + +Status CloudRowsetBuilder::validate_partial_rowset_meta(const RowsetMetaPB& base_meta, + const RowsetMetaPB& partial_meta, + int32_t segment_start_id, + int32_t segment_capacity) { + const auto count = partial_meta.num_segments(); + if (partial_meta.rowset_id_v2() != base_meta.rowset_id_v2() || + partial_meta.tablet_id() != base_meta.tablet_id() || + partial_meta.txn_id() != base_meta.txn_id() || + partial_meta.resource_id() != base_meta.resource_id() || + partial_meta.index_id() != base_meta.index_id() || + partial_meta.partition_id() != base_meta.partition_id() || + partial_meta.tablet_schema_hash() != base_meta.tablet_schema_hash() || + partial_meta.table_id() != base_meta.table_id() || + partial_meta.db_id() != base_meta.db_id() || + UniqueId(partial_meta.load_id()) != UniqueId(base_meta.load_id())) { + return Status::InvalidArgument("sink upload rowset identity mismatch for tablet {}", + base_meta.tablet_id()); + } + if (!partial_meta.has_tablet_schema() || count < 0 || count > segment_capacity || + partial_meta.segment_ids_size() != count || partial_meta.num_segment_rows_size() != count || + partial_meta.segments_key_bounds_size() != count || + partial_meta.segments_file_size_size() != count || + partial_meta.segments_key_bounds_aggregated() || + (partial_meta.inverted_index_file_info_size() != 0 && + partial_meta.inverted_index_file_info_size() != count)) { + return Status::InvalidArgument("misaligned sink upload metadata for tablet {}", + base_meta.tablet_id()); + } + const bool has_index = + std::ranges::any_of(partial_meta.tablet_schema().index(), [](const auto& index) { + return index.index_type() == IndexType::INVERTED || + index.index_type() == IndexType::ANN; + }); + if (has_index && partial_meta.inverted_index_file_info_size() != count) { + return Status::InvalidArgument("missing sink upload index metadata for tablet {}", + base_meta.tablet_id()); + } + int64_t partial_rows = 0; + for (int pos = 0; pos < count; ++pos) { + const int64_t id = partial_meta.segment_ids(pos); + if (id < segment_start_id || + id >= static_cast(segment_start_id) + segment_capacity || + (pos > 0 && id <= partial_meta.segment_ids(pos - 1)) || + partial_meta.num_segment_rows(pos) < 0 || partial_meta.segments_file_size(pos) <= 0) { + return Status::InvalidArgument("invalid sink upload segment {} for tablet {}", id, + base_meta.tablet_id()); + } + partial_rows += partial_meta.num_segment_rows(pos); + } + if (partial_rows != partial_meta.num_rows() || partial_meta.data_disk_size() < 0 || + partial_meta.index_disk_size() < 0 || + partial_meta.total_disk_size() != + partial_meta.data_disk_size() + partial_meta.index_disk_size()) { + return Status::InvalidArgument("invalid sink upload statistics for tablet {}", + base_meta.tablet_id()); + } + return Status::OK(); +} + +Status CloudRowsetBuilder::assemble_rowset_meta_from_partials( + const RowsetMetaPB& base_meta, const std::map& partial_rowset_metas, + int32_t max_segments_per_rowset, RowsetMetaPB* result) { + *result = base_meta; + int64_t rows = 0; + int64_t data_size = 0; + int64_t index_size = 0; + auto schema = std::make_shared(); + schema->init_from_pb(base_meta.tablet_schema()); + const bool has_variant = schema->num_variant_columns() > 0; + std::vector schemas; + for (const auto& [segment_start_id, partial_meta] : partial_rowset_metas) { + const auto count = partial_meta.num_segments(); + if (count > 0 && result->segment_ids_size() > 0 && + partial_meta.segment_ids(0) <= result->segment_ids(result->segment_ids_size() - 1)) { + return Status::InvalidArgument("overlapping sink upload segment ranges"); + } + result->mutable_segment_ids()->MergeFrom(partial_meta.segment_ids()); + result->mutable_num_segment_rows()->MergeFrom(partial_meta.num_segment_rows()); + result->mutable_segments_file_size()->MergeFrom(partial_meta.segments_file_size()); + result->mutable_segments_key_bounds()->MergeFrom(partial_meta.segments_key_bounds()); + result->mutable_inverted_index_file_info()->MergeFrom( + partial_meta.inverted_index_file_info()); + for (const auto& [path, location] : partial_meta.packed_slice_locations()) { + if (!result->mutable_packed_slice_locations()->emplace(path, location).second) { + return Status::InvalidArgument("duplicate packed slice {}", path); + } + } + rows += partial_meta.num_rows(); + data_size += partial_meta.data_disk_size(); + index_size += partial_meta.index_disk_size(); + result->set_segments_key_bounds_truncated(result->segments_key_bounds_truncated() || + partial_meta.segments_key_bounds_truncated()); + if (has_variant) { + auto partial_schema = std::make_shared(); + partial_schema->init_from_pb(partial_meta.tablet_schema()); + schemas.push_back(std::move(partial_schema)); + } + } + if (result->segment_ids_size() > max_segments_per_rowset) { + return Status::InvalidArgument("too many sink upload segments for tablet {}", + base_meta.tablet_id()); + } + if (has_variant && !schemas.empty()) { + TabletSchemaSPtr merged_schema; + schemas.push_back(schema); + RETURN_IF_ERROR(variant_util::get_least_common_schema(schemas, nullptr, merged_schema)); + result->clear_tablet_schema(); + merged_schema->to_schema_pb(result->mutable_tablet_schema()); + } + result->set_num_segments(result->segment_ids_size()); + result->set_num_rows(rows); + result->set_data_disk_size(data_size); + result->set_index_disk_size(index_size); + result->set_total_disk_size(data_size + index_size); + result->set_empty(rows == 0); + result->set_segments_overlap_pb( + schema->cluster_key_uids().empty() && + !is_segment_overlapping(result->segments_key_bounds()) + ? NONOVERLAPPING + : OVERLAPPING); + result->set_enable_segments_file_size(true); + result->set_enable_inverted_index_file_info(true); + result->set_creation_time(UnixSeconds()); + result->set_newest_write_timestamp(UnixSeconds()); + result->set_rowset_state(COMMITTED); + return Status::OK(); +} + +Status CloudRowsetBuilder::build_rowset_from_assembled_meta(const RowsetMetaPB& meta) { + return static_cast(_rowset_writer.get()) + ->build_from_assembled_meta(meta, _rowset); +} + Status CloudRowsetBuilder::commit_txn() { DCHECK(is_data_builder()); if (!_skip_writing_rowset_metadata) { diff --git a/be/src/cloud/cloud_rowset_builder.h b/be/src/cloud/cloud_rowset_builder.h index d7db14f43f2595..d77dcbfdb306fc 100644 --- a/be/src/cloud/cloud_rowset_builder.h +++ b/be/src/cloud/cloud_rowset_builder.h @@ -17,11 +17,15 @@ #pragma once +#include + #include "storage/rowset_builder.h" namespace doris { class CloudTablet; +class PCloudLoadMowSnapshot; +class PCloudLoadMowResult; class CloudStorageEngine; class CloudRowsetBuilder : public BaseRowsetBuilder { @@ -35,6 +39,22 @@ class CloudRowsetBuilder : public BaseRowsetBuilder { Status commit_txn() override; + static Status validate_partial_rowset_meta(const RowsetMetaPB& base_meta, + const RowsetMetaPB& partial_meta, + int32_t segment_start_id, int32_t segment_capacity); + // Inputs have already passed validate_partial_rowset_meta at the stream boundary. + static Status assemble_rowset_meta_from_partials( + const RowsetMetaPB& base_meta, + const std::map& partial_rowset_metas, + int32_t max_segments_per_rowset, RowsetMetaPB* result); + + Status build_rowset_from_assembled_meta(const RowsetMetaPB& meta); + Status get_mow_snapshot_for_sink(PCloudLoadMowSnapshot* snapshot); + Status merge_sink_mow_bitmap(const PCloudLoadMowResult& result); + + static Status validate_sink_mow_result(const PCloudLoadMowResult& result, + int64_t snapshot_version); + virtual void update_tablet_stats(); const RowsetMetaSharedPtr& rowset_meta(); @@ -56,6 +76,7 @@ class CloudRowsetBuilder : public BaseRowsetBuilder { Status check_tablet_version_count(); CloudStorageEngine& _engine; + std::unique_ptr _mow_snapshot_for_sink; // whether to skip writing rowset metadata to meta service. // This is used for empty rowset when config::skip_writing_empty_rowset_metadata is true. diff --git a/be/src/cloud/cloud_rowset_writer.cpp b/be/src/cloud/cloud_rowset_writer.cpp index c633bc25e30b13..313ccf3eb3528b 100644 --- a/be/src/cloud/cloud_rowset_writer.cpp +++ b/be/src/cloud/cloud_rowset_writer.cpp @@ -110,6 +110,19 @@ Status CloudRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool check return Status::OK(); } +Status CloudRowsetWriter::build_from_assembled_meta(const RowsetMetaPB& meta, + RowsetSharedPtr& rowset) { + DORIS_CHECK(_seg_files.get_file_writers().empty()); + if (!_rowset_meta->init_from_pb(meta)) { + return Status::InvalidArgument("invalid assembled rowset metadata for tablet {}", + _context.tablet_id); + } + RETURN_IF_ERROR(RowsetFactory::create_rowset(_rowset_meta->tablet_schema(), + _context.tablet_path, _rowset_meta, &rowset)); + _already_built = true; + return Status::OK(); +} + Status CloudRowsetWriter::build(RowsetSharedPtr& rowset) { if (_calc_delete_bitmap_token != nullptr) { RETURN_IF_ERROR(_calc_delete_bitmap_token->wait()); diff --git a/be/src/cloud/cloud_rowset_writer.h b/be/src/cloud/cloud_rowset_writer.h index e351405f738d06..1e8a4fa7a8a02a 100644 --- a/be/src/cloud/cloud_rowset_writer.h +++ b/be/src/cloud/cloud_rowset_writer.h @@ -32,6 +32,8 @@ class CloudRowsetWriter : public BaseBetaRowsetWriter { Status build(RowsetSharedPtr& rowset) override; + Status build_from_assembled_meta(const RowsetMetaPB& meta, RowsetSharedPtr& rowset); + private: Status _build_rowset_meta(RowsetMeta* rowset_meta, bool check_segment_num = false, std::vector* completed_segment_ids = nullptr) override; diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index bbfc8f5c1569a5..e1a0c43480eee8 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -684,11 +684,11 @@ std::vector CloudWarmUpManager::get_replica_info(int64_t tablet_ } void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, - int64_t sync_wait_timeout_ms) { + int64_t sync_wait_timeout_ms, bool warm_up_local) { if (sync_wait_timeout_ms <= 0) { auto rs_meta_pb = std::make_shared(rs_meta.get_rowset_pb()); - auto st = _thread_pool_token->submit_func([this, rs_meta_pb, table_id, - sync_wait_timeout_ms]() { + auto st = _thread_pool_token->submit_func([this, rs_meta_pb, table_id, sync_wait_timeout_ms, + warm_up_local]() { RowsetMeta async_rs_meta; bool init_succeed = async_rs_meta.init_from_pb(*rs_meta_pb); TEST_SYNC_POINT_CALLBACK("CloudWarmUpManager::warm_up_rowset.async_init_from_pb", @@ -697,7 +697,7 @@ void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, LOG(WARNING) << "Failed to init rowset meta when warming up rowset asynchronously"; return; } - _warm_up_rowset(async_rs_meta, table_id, sync_wait_timeout_ms); + _warm_up_rowset(async_rs_meta, table_id, sync_wait_timeout_ms, warm_up_local); }); if (!st.ok()) { LOG(WARNING) << "Failed to submit warm up rowset task: " << st; @@ -711,7 +711,7 @@ void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, bool finished = false; std::unique_lock lock(mu); auto st = _thread_pool_token->submit_func([&, this]() { - _warm_up_rowset(rs_meta, table_id, sync_wait_timeout_ms); + _warm_up_rowset(rs_meta, table_id, sync_wait_timeout_ms, warm_up_local); std::unique_lock l(mu); finished = true; cv.notify_one(); @@ -728,18 +728,28 @@ void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, } void CloudWarmUpManager::_warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, - int64_t sync_wait_timeout_ms) { + int64_t sync_wait_timeout_ms, bool warm_up_local) { TEST_SYNC_POINT_CALLBACK("CloudWarmUpManager::_warm_up_rowset.enter", &rs_meta, &sync_wait_timeout_ms); bool cache_hit = false; - auto replicas = get_replica_info(rs_meta.tablet_id(), table_id, false, cache_hit); + std::vector replicas; + if (warm_up_local) { + TReplicaInfo replica; + replica.__set_backend_id(ExecEnv::GetInstance()->cluster_info()->backend_id); + replica.__set_host(BackendOptions::get_localhost()); + replica.__set_brpc_port(config::brpc_port); + replicas.push_back(JobReplicaInfo {0, std::move(replica)}); + } else { + replicas = get_replica_info(rs_meta.tablet_id(), table_id, false, cache_hit); + } if (replicas.empty()) { VLOG_DEBUG << "There is no need to warmup tablet=" << rs_meta.tablet_id() << ", skipping rowset=" << rs_meta.rowset_id().to_string(); g_file_cache_event_driven_warm_up_skipped_rowset_num << 1; return; } - Status st = _do_warm_up_rowset(rs_meta, table_id, replicas, sync_wait_timeout_ms, !cache_hit); + Status st = _do_warm_up_rowset(rs_meta, table_id, replicas, sync_wait_timeout_ms, + !warm_up_local && !cache_hit); if (cache_hit && !st.ok() && st.is()) { replicas = get_replica_info(rs_meta.tablet_id(), table_id, true, cache_hit); st = _do_warm_up_rowset(rs_meta, table_id, replicas, sync_wait_timeout_ms, true); @@ -780,6 +790,9 @@ Status CloudWarmUpManager::_do_warm_up_rowset(RowsetMeta& rs_meta, int64_t table std::vector& replicas, int64_t sync_wait_timeout_ms, bool skip_existence_check) { + TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudWarmUpManager::_do_warm_up_rowset", Status::OK(), + &rs_meta, &replicas, &sync_wait_timeout_ms, + &skip_existence_check); auto tablet_id = rs_meta.tablet_id(); int64_t now_ts = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) diff --git a/be/src/cloud/cloud_warm_up_manager.h b/be/src/cloud/cloud_warm_up_manager.h index ba6e408da5ac81..b1f7ce1aaddf41 100644 --- a/be/src/cloud/cloud_warm_up_manager.h +++ b/be/src/cloud/cloud_warm_up_manager.h @@ -126,7 +126,10 @@ class CloudWarmUpManager { // @param rs_meta Metadata of the rowset to be warmed up. // @param sync_wait_timeout_ms Timeout in milliseconds to wait for the warm-up // to complete. Non-positive value means no waiting. - void warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, int64_t sync_wait_timeout_ms = -1); + // @param warm_up_local Only warms this BE's cache if true. Used when memtable-on-sink + // is enabled and the sink BE uploads files directly to object storage. + void warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, int64_t sync_wait_timeout_ms = -1, + bool warm_up_local = false); void recycle_cache(int64_t tablet_id, const std::vector& rowsets); @@ -183,7 +186,8 @@ class CloudWarmUpManager { std::vector get_replica_info(int64_t tablet_id, int64_t table_id, bool bypass_cache, bool& cache_hit); - void _warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, int64_t sync_wait_timeout_ms); + void _warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, int64_t sync_wait_timeout_ms, + bool warm_up_local = false); void _recycle_cache(int64_t tablet_id, const std::vector& rowsets); void submit_download_tasks(io::Path path, int64_t file_size, io::FileSystemSPtr file_system, diff --git a/be/src/exec/sink/load_stream_map_pool.cpp b/be/src/exec/sink/load_stream_map_pool.cpp index 6d87fc19840c14..b4a68e30af50e5 100644 --- a/be/src/exec/sink/load_stream_map_pool.cpp +++ b/be/src/exec/sink/load_stream_map_pool.cpp @@ -99,6 +99,15 @@ void LoadStreamMap::save_tablets_to_commit(int64_t dst_id, } } +void LoadStreamMap::save_segments_for_tablet( + const std::unordered_map& segments_for_tablet) { + std::lock_guard lock(_tablets_to_commit_mutex); + // Unshared DeltaWriters on the same BE each report their own segment count. + for (const auto& [tablet_id, count] : segments_for_tablet) { + _segments_for_tablet[tablet_id] += count; + } +} + bool LoadStreamMap::release() { int num_use = --_use_cnt; if (num_use == 0) { diff --git a/be/src/exec/sink/load_stream_map_pool.h b/be/src/exec/sink/load_stream_map_pool.h index 131d1d1d88b2b7..8679b496a3400a 100644 --- a/be/src/exec/sink/load_stream_map_pool.h +++ b/be/src/exec/sink/load_stream_map_pool.h @@ -85,9 +85,7 @@ class LoadStreamMap { void save_tablets_to_commit(int64_t dst_id, const std::vector& tablets_to_commit); - void save_segments_for_tablet(const std::unordered_map& segments_for_tablet) { - _segments_for_tablet.insert(segments_for_tablet.cbegin(), segments_for_tablet.cend()); - } + void save_segments_for_tablet(const std::unordered_map& segments_for_tablet); // Return true if the last instance is just released. bool release(); diff --git a/be/src/exec/sink/load_stream_stub.cpp b/be/src/exec/sink/load_stream_stub.cpp index 967e35fed49181..0a1c5923b091df 100644 --- a/be/src/exec/sink/load_stream_stub.cpp +++ b/be/src/exec/sink/load_stream_stub.cpp @@ -123,6 +123,12 @@ int LoadStreamReplyHandler::on_received_messages(brpc::StreamId id, butil::IOBuf << status; } } + if (response.has_write_context()) { + auto writer_id = response.write_context().writer_id(); + std::lock_guard lock(stub->_write_context_mutex); + stub->_write_context_responses[writer_id] = std::move(response); + stub->_write_context_cv.notify_all(); + } } return 0; } @@ -308,6 +314,58 @@ Status LoadStreamStub::close_load(const std::vector& tablets_to_commi return Status::OK(); } +Status LoadStreamStub::register_sink_upload_writer(int64_t partition_id, int64_t index_id, + int64_t tablet_id, const std::string& writer_id, + PCloudLoadWriteContext* context) { + PStreamHeader header; + *header.mutable_load_id() = _load_id; + header.set_src_id(_src_id); + header.set_partition_id(partition_id); + header.set_index_id(index_id); + header.set_tablet_id(tablet_id); + header.set_writer_id(writer_id); + header.set_opcode(PStreamHeader::GET_WRITE_CONTEXT); + RETURN_IF_ERROR(_encode_and_send(header)); + MonotonicStopWatch watch; + watch.start(); + std::unique_lock lock(_write_context_mutex); + while (!_write_context_responses.contains(writer_id)) { + RETURN_IF_ERROR(check_cancel()); + if (_is_closed.load()) { + return Status::InternalError("stream closed while getting write context for tablet {}", + tablet_id); + } + if (watch.elapsed_time() / 1000000 >= config::open_load_stream_timeout_ms) { + return Status::TimedOut("getting write context for tablet {}", tablet_id); + } + _write_context_cv.wait_for(lock, 100000); + } + auto response = std::move(_write_context_responses.at(writer_id)); + _write_context_responses.erase(writer_id); + RETURN_IF_ERROR(Status::create(response.status())); + *context = std::move(*response.mutable_write_context()); + return Status::OK(); +} + +Status LoadStreamStub::add_partial_rowset(int64_t partition_id, int64_t index_id, int64_t tablet_id, + const std::string& writer_id, const RowsetMetaPB& meta, + const PCloudLoadMowResult* mow_result) { + RETURN_IF_ERROR(check_cancel()); + PStreamHeader header; + *header.mutable_load_id() = _load_id; + header.set_src_id(_src_id); + header.set_partition_id(partition_id); + header.set_index_id(index_id); + header.set_tablet_id(tablet_id); + header.set_writer_id(writer_id); + header.set_opcode(PStreamHeader::ADD_PARTIAL_ROWSET); + *header.mutable_partial_rowset_meta() = meta; + if (mow_result != nullptr) { + *header.mutable_mow_result() = *mow_result; + } + return _encode_and_send(header); +} + // GET_SCHEMA Status LoadStreamStub::get_schema(const std::vector& tablets) { if (!_is_open.load()) { @@ -421,9 +479,10 @@ Status LoadStreamStub::_encode_and_send(PStreamHeader& header, std::span { // cancel the stream, abort close_wait, mark _is_closed and _is_cancelled void cancel(Status reason); + Status register_sink_upload_writer(int64_t partition_id, int64_t index_id, int64_t tablet_id, + const std::string& writer_id, + PCloudLoadWriteContext* context); + Status add_partial_rowset(int64_t partition_id, int64_t index_id, int64_t tablet_id, + const std::string& writer_id, const RowsetMetaPB& meta, + const PCloudLoadMowResult* mow_result = nullptr); + Status wait_for_schema(int64_t partition_id, int64_t index_id, int64_t tablet_id, int64_t timeout_ms = 60000); @@ -294,6 +301,10 @@ class LoadStreamStub : public std::enable_shared_from_this { std::mutex _send_mutex; butil::IOBuf _buffer; + bthread::Mutex _write_context_mutex; + bthread::ConditionVariable _write_context_cv; + std::unordered_map _write_context_responses; + bthread::Mutex _schema_mutex; bthread::ConditionVariable _schema_cv; std::shared_ptr _tablet_schema_for_index; diff --git a/be/src/exec/sink/writer/vtablet_writer_v2.cpp b/be/src/exec/sink/writer/vtablet_writer_v2.cpp index fa97d272af148c..ffdf71daab3328 100644 --- a/be/src/exec/sink/writer/vtablet_writer_v2.cpp +++ b/be/src/exec/sink/writer/vtablet_writer_v2.cpp @@ -573,6 +573,8 @@ Status VTabletWriterV2::_write_memtable(std::shared_ptr block, int64_t ta .write_file_cache = _write_file_cache, .storage_vault_id {}, .enable_table_memtable_backpressure = _tablet_finder->is_adaptive_random_bucket(), + .cloud_sink_upload = config::is_cloud_mode() && + _state->query_options().enable_cloud_memtable_sink_upload, }; bool index_not_found = true; for (const auto& index : _schema->indexes()) { diff --git a/be/src/load/channel/load_stream.cpp b/be/src/load/channel/load_stream.cpp index d0ac39c7782d69..dbaa7bec9986f8 100644 --- a/be/src/load/channel/load_stream.cpp +++ b/be/src/load/channel/load_stream.cpp @@ -108,11 +108,34 @@ Status TabletStream::init(std::shared_ptr schema, int64_t return _status.status(); } -Status TabletStream::append_data(const PStreamHeader& header, butil::IOBuf* data) { +Status TabletStream::append_data(const PStreamHeader& header, butil::IOBuf* data, + PCloudLoadWriteContext* context) { if (!_status.ok()) { return _status.status(); } + if (header.opcode() == PStreamHeader::GET_WRITE_CONTEXT) { + std::lock_guard lock(_lock); + if (!_segids_mapping.empty()) { + _status.update(Status::InvalidArgument("cannot switch streamed tablet to sink upload")); + return _status.status(); + } + auto st = _load_stream_writer->register_sink_upload_writer(header.writer_id(), context); + _status.update(st); + return st; + } + if (header.opcode() == PStreamHeader::ADD_PARTIAL_ROWSET) { + std::lock_guard lock(_lock); + int64_t num_added_segments = 0; + auto st = _load_stream_writer->add_partial_rowset( + header.writer_id(), header.partial_rowset_meta(), &num_added_segments, + header.has_mow_result() ? &header.mow_result() : nullptr); + // Sink uploads use this as the accepted segment count for close validation, not ID allocation. + _next_segid += cast_set(num_added_segments); + _status.update(st); + return st; + } + // dispatch add_segment request if (header.opcode() == PStreamHeader::ADD_SEGMENT) { return add_segment(header, data); @@ -126,6 +149,11 @@ Status TabletStream::append_data(const PStreamHeader& header, butil::IOBuf* data SegIdMapping* mapping = nullptr; { std::lock_guard lock_guard(_lock); + if (_load_stream_writer->is_sink_upload()) { + _status.update( + Status::InvalidArgument("cannot stream files into a sink-upload rowset")); + return _status.status(); + } if (!_segids_mapping.contains(src_id)) { _segids_mapping[src_id] = std::make_unique(); } @@ -379,7 +407,8 @@ IndexStream::~IndexStream() { } } -Status IndexStream::append_data(const PStreamHeader& header, butil::IOBuf* data) { +Status IndexStream::append_data(const PStreamHeader& header, butil::IOBuf* data, + PCloudLoadWriteContext* context) { SCOPED_TIMER(_append_data_timer); int64_t tablet_id = header.tablet_id(); TabletStreamSharedPtr tablet_stream; @@ -393,7 +422,7 @@ Status IndexStream::append_data(const PStreamHeader& header, butil::IOBuf* data) } } - return tablet_stream->append_data(header, data); + return tablet_stream->append_data(header, data, context); } void IndexStream::_init_tablet_stream(TabletStreamSharedPtr& tablet_stream, int64_t tablet_id, @@ -729,7 +758,8 @@ void LoadStream::_parse_header(butil::IOBuf* const message, PStreamHeader& hdr) VLOG_DEBUG << "header parse result: " << hdr.DebugString(); } -Status LoadStream::_append_data(const PStreamHeader& header, butil::IOBuf* data) { +Status LoadStream::_append_data(const PStreamHeader& header, butil::IOBuf* data, + PCloudLoadWriteContext* context) { SCOPED_TIMER(_append_data_timer); IndexStreamSharedPtr index_stream; @@ -743,7 +773,7 @@ Status LoadStream::_append_data(const PStreamHeader& header, butil::IOBuf* data) index_stream = it->second; } - return index_stream->append_data(header, data); + return index_stream->append_data(header, data, context); } int LoadStream::on_received_messages(StreamId id, butil::IOBuf* const messages[], size_t size) { @@ -808,6 +838,7 @@ void LoadStream::_dispatch(StreamId id, const PStreamHeader& hdr, butil::IOBuf* } switch (hdr.opcode()) { + case PStreamHeader::ADD_PARTIAL_ROWSET: case PStreamHeader::ADD_SEGMENT: { auto st = _append_data(hdr, data); if (!st.ok()) { @@ -859,6 +890,19 @@ void LoadStream::_dispatch(StreamId id, const PStreamHeader& hdr, butil::IOBuf* brpc::StreamClose(closing_id); } } break; + case PStreamHeader::GET_WRITE_CONTEXT: { + PLoadStreamResponse response; + auto* context = response.mutable_write_context(); + context->set_writer_id(hdr.writer_id()); + auto st = _append_data(hdr, data, context); + st.to_protobuf(response.mutable_status()); + butil::IOBuf buf; + buf.append(response.SerializeAsString()); + auto write_status = _write_stream(id, buf); + if (!write_status.ok()) { + LOG(WARNING) << "failed to return cloud write context: " << write_status; + } + } break; case PStreamHeader::GET_SCHEMA: { _report_schema(id, hdr); } break; diff --git a/be/src/load/channel/load_stream.h b/be/src/load/channel/load_stream.h index 22f5845bd1a0fb..860c16fdb718de 100644 --- a/be/src/load/channel/load_stream.h +++ b/be/src/load/channel/load_stream.h @@ -52,7 +52,8 @@ class TabletStream { Status init(std::shared_ptr schema, int64_t index_id, int64_t partition_id, bool is_empty = false); - Status append_data(const PStreamHeader& header, butil::IOBuf* data); + Status append_data(const PStreamHeader& header, butil::IOBuf* data, + PCloudLoadWriteContext* context = nullptr); Status add_segment(const PStreamHeader& header, butil::IOBuf* data); void add_num_segments(int64_t num_segments) { _num_segments += num_segments; } void disable_num_segments_check() { _check_num_segments = false; } @@ -100,7 +101,8 @@ class IndexStream { bool write_file_cache); ~IndexStream(); - Status append_data(const PStreamHeader& header, butil::IOBuf* data); + Status append_data(const PStreamHeader& header, butil::IOBuf* data, + PCloudLoadWriteContext* context = nullptr); void close(const std::vector& tablets_to_commit, std::vector* success_tablet_ids, FailedTablets* failed_tablet_ids); @@ -168,7 +170,8 @@ class LoadStream : public brpc::StreamInputHandler { private: void _parse_header(butil::IOBuf* const message, PStreamHeader& hdr); void _dispatch(StreamId id, const PStreamHeader& hdr, butil::IOBuf* data); - Status _append_data(const PStreamHeader& header, butil::IOBuf* data); + Status _append_data(const PStreamHeader& header, butil::IOBuf* data, + PCloudLoadWriteContext* context = nullptr); void _report_result(StreamId stream, const Status& status, const std::vector& success_tablet_ids, diff --git a/be/src/load/channel/load_stream_writer.cpp b/be/src/load/channel/load_stream_writer.cpp index b507c019fb3b5d..9fc23202731e51 100644 --- a/be/src/load/channel/load_stream_writer.cpp +++ b/be/src/load/channel/load_stream_writer.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -30,6 +31,7 @@ #include "bvar/bvar.h" #include "cloud/cloud_rowset_builder.h" +#include "cloud/cloud_warm_up_manager.h" #include "cloud/config.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" @@ -107,9 +109,111 @@ Status LoadStreamWriter::init(bool is_empty) { return Status::OK(); } +Status LoadStreamWriter::register_sink_upload_writer(const std::string& writer_id, + PCloudLoadWriteContext* context) { + // Control messages run synchronously in LoadStream::_dispatch, which attaches the load. + std::lock_guard lock(_lock); + const auto& ctx = _rowset_writer->context(); + if (!config::is_cloud_mode() || (ctx.tablet_schema->has_inverted_index() && + ctx.tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::V1)) { + return Status::NotSupported("sink upload requires cloud mode and no V1 inverted indexes"); + } + if (_pre_closed || writer_id.empty()) { + return Status::InvalidArgument("invalid sink writer registration for tablet {}", + _req.tablet_id); + } + if (!_sink_upload.exchange(true)) { + _max_segments_per_rowset = config::max_segment_num_per_rowset; + DORIS_CHECK_GT(_max_segments_per_rowset, 0); + } + auto it = _writer_segment_start_ids.find(writer_id); + if (it == _writer_segment_start_ids.end()) { + const int64_t start = + static_cast(_writer_segment_start_ids.size()) * _max_segments_per_rowset; + if (start + _max_segments_per_rowset > INT32_MAX) { + return Status::InvalidArgument("sink upload segment range overflow for tablet {}", + _req.tablet_id); + } + it = _writer_segment_start_ids.emplace(writer_id, static_cast(start)).first; + } + if (ctx.tablet->enable_unique_key_merge_on_write()) { + RETURN_IF_ERROR(static_cast(_rowset_builder.get()) + ->get_mow_snapshot_for_sink(context->mutable_mow_snapshot())); + } + context->set_writer_id(writer_id); + *context->mutable_rowset_meta() = _rowset_writer->rowset_meta()->get_rowset_pb(); + context->mutable_rowset_meta()->set_newest_write_timestamp(ctx.newest_write_timestamp); + context->set_segment_start_id(it->second); + context->set_segment_capacity(_max_segments_per_rowset); + context->set_encrypt_algorithm(ctx.encrypt_algorithm.value()); + context->set_file_cache_expiration_time(ctx.file_cache_expiration_time); + context->set_warm_up_file_cache(config::enable_file_cache && ctx.write_file_cache); + return Status::OK(); +} + +Status LoadStreamWriter::add_partial_rowset(const std::string& writer_id, const RowsetMetaPB& meta, + int64_t* num_added_segments, + const PCloudLoadMowResult* mow_result) { + // Control messages run synchronously in LoadStream::_dispatch, which attaches the load. + std::lock_guard lock(_lock); + *num_added_segments = 0; + auto it = _writer_segment_start_ids.find(writer_id); + if (it == _writer_segment_start_ids.end() || _pre_closed) { + return Status::InvalidArgument("unknown or closed sink writer {} for tablet {}", writer_id, + _req.tablet_id); + } + const bool is_mow = _rowset_builder->tablet()->enable_unique_key_merge_on_write(); + if (is_mow != (mow_result != nullptr)) { + return Status::InvalidArgument("sink upload MOW result does not match tablet {}", + _req.tablet_id); + } + auto previous = _partial_rowset_metas.find(it->second); + if (previous != _partial_rowset_metas.end()) { + if (!google::protobuf::util::MessageDifferencer::Equals(previous->second, meta)) { + return Status::InvalidArgument("conflicting sink writer result {}", writer_id); + } + if (is_mow && !google::protobuf::util::MessageDifferencer::Equals( + _mow_results.at(it->second), *mow_result)) { + return Status::InvalidArgument("conflicting sink MOW result {}", writer_id); + } + return Status::OK(); + } + RETURN_IF_ERROR(CloudRowsetBuilder::validate_partial_rowset_meta( + _rowset_writer->rowset_meta()->get_rowset_pb(), meta, it->second, + _max_segments_per_rowset)); + // Packed mappings name logical files in this rowset, never an uploader's absolute S3 URI. + if (!meta.packed_slice_locations().empty()) { + std::unordered_set paths; + for (auto id : meta.segment_ids()) { + auto path = _rowset_writer->context().segment_path(cast_set(id)); + paths.insert(path); + auto prefix = InvertedIndexDescriptor::get_index_file_path_prefix(path); + paths.insert(InvertedIndexDescriptor::get_index_file_path_v2(std::string(prefix))); + } + for (const auto& [path, location] : meta.packed_slice_locations()) { + if (!paths.contains(path) || location.offset() < 0 || location.size() < 0 || + location.packed_file_path().empty()) { + return Status::InvalidArgument("invalid sink upload packed slice {}", path); + } + } + } + if (is_mow) { + RETURN_IF_ERROR(static_cast(_rowset_builder.get()) + ->merge_sink_mow_bitmap(*mow_result)); + _mow_results.emplace(it->second, *mow_result); + } + _partial_rowset_metas.emplace(it->second, meta); + *num_added_segments = meta.num_segments(); + return Status::OK(); +} + Status LoadStreamWriter::append_data(uint32_t segid, uint64_t offset, butil::IOBuf buf, FileType file_type) { SCOPED_ATTACH_TASK(_resource_ctx); + DBUG_EXECUTE_IF("LoadStreamWriter.append_data.unexpected_transfer", { + return Status::InternalError("unexpected segment transfer in sink-upload load"); + }); io::FileWriter* file_writer = nullptr; auto& file_writers = file_type == FileType::SEGMENT_FILE ? _segment_file_writers : _inverted_file_writers; @@ -327,26 +431,38 @@ Status LoadStreamWriter::_pre_close() { } } - RETURN_IF_ERROR(_rowset_builder->build_rowset()); - if (config::is_cloud_mode()) { - // Forwarded files are owned here, outside CloudRowsetWriter's collections. - auto meta = _rowset_writer->rowset_meta(); - DORIS_CHECK(meta->segments_file_size().empty()); - DORIS_CHECK(meta->inverted_index_file_info().empty()); - std::vector sizes; - sizes.reserve(_segment_file_writers.size()); - for (const auto& writer : _segment_file_writers) { - sizes.push_back(writer->bytes_appended()); + if (_sink_upload.load()) { + if (_writer_segment_start_ids.size() != _partial_rowset_metas.size()) { + return Status::Corruption("missing sink writer results for tablet {}", _req.tablet_id); } - meta->add_segments_file_size(sizes); - std::vector indexes(_inverted_file_writers.size()); - std::vector index_ptrs; - index_ptrs.reserve(indexes.size()); - for (size_t pos = 0; pos < indexes.size(); ++pos) { - indexes[pos].set_index_size(_inverted_file_writers[pos]->bytes_appended()); - index_ptrs.push_back(&indexes[pos]); + RowsetMetaPB meta; + RETURN_IF_ERROR(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + _rowset_writer->rowset_meta()->get_rowset_pb(), _partial_rowset_metas, + _max_segments_per_rowset, &meta)); + RETURN_IF_ERROR(static_cast(_rowset_builder.get()) + ->build_rowset_from_assembled_meta(meta)); + } else { + RETURN_IF_ERROR(_rowset_builder->build_rowset()); + if (config::is_cloud_mode()) { + // Forwarded files are owned here, outside CloudRowsetWriter's collections. + auto meta = _rowset_writer->rowset_meta(); + DORIS_CHECK(meta->segments_file_size().empty()); + DORIS_CHECK(meta->inverted_index_file_info().empty()); + std::vector sizes; + sizes.reserve(_segment_file_writers.size()); + for (const auto& writer : _segment_file_writers) { + sizes.push_back(writer->bytes_appended()); + } + meta->add_segments_file_size(sizes); + std::vector indexes(_inverted_file_writers.size()); + std::vector index_ptrs; + index_ptrs.reserve(indexes.size()); + for (size_t pos = 0; pos < indexes.size(); ++pos) { + indexes[pos].set_index_size(_inverted_file_writers[pos]->bytes_appended()); + index_ptrs.push_back(&indexes[pos]); + } + meta->add_inverted_index_files_info(index_ptrs); } - meta->add_inverted_index_files_info(index_ptrs); } RETURN_IF_ERROR(_rowset_builder->submit_calc_delete_bitmap_task()); _pre_closed = true; @@ -360,6 +476,12 @@ Status LoadStreamWriter::close() { } RETURN_IF_ERROR(_rowset_builder->wait_calc_delete_bitmap()); RETURN_IF_ERROR(_rowset_builder->commit_txn()); + if (_sink_upload.load() && config::enable_file_cache && _req.write_file_cache) { + auto& builder = static_cast(*_rowset_builder); + ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager().warm_up_rowset( + *builder.rowset_meta(), _rowset_builder->tablet_sptr()->table_id(), + /*sync_wait_timeout_ms=*/-1, /*warm_up_local=*/true); + } return Status::OK(); } diff --git a/be/src/load/channel/load_stream_writer.h b/be/src/load/channel/load_stream_writer.h index 8c4f4d79109e88..2873c97b38f0c7 100644 --- a/be/src/load/channel/load_stream_writer.h +++ b/be/src/load/channel/load_stream_writer.h @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -53,6 +54,12 @@ class LoadStreamWriter { // is_empty is true for tablets first seen when closing the load. Status init(bool is_empty = false); + Status register_sink_upload_writer(const std::string& writer_id, + PCloudLoadWriteContext* context); + Status add_partial_rowset(const std::string& writer_id, const RowsetMetaPB& meta, + int64_t* num_added_segments, + const PCloudLoadMowResult* mow_result = nullptr); + bool is_sink_upload() { return _sink_upload.load(); } Status append_data(uint32_t segid, uint64_t offset, butil::IOBuf buf, FileType file_type = FileType::SEGMENT_FILE); @@ -75,6 +82,12 @@ class LoadStreamWriter { // without lock Status _pre_close(); + std::atomic _sink_upload {false}; + int32_t _max_segments_per_rowset = 0; + std::unordered_map _writer_segment_start_ids; + // Partial results keyed by the writer's starting segment ID. + std::map _partial_rowset_metas; + std::map _mow_results; bool _is_init = false; bool _is_canceled = false; bool _pre_closed = false; diff --git a/be/src/load/delta_writer/delta_writer_context.h b/be/src/load/delta_writer/delta_writer_context.h index 06b4659c79bff5..8db1733f4ea9ff 100644 --- a/be/src/load/delta_writer/delta_writer_context.h +++ b/be/src/load/delta_writer/delta_writer_context.h @@ -56,6 +56,7 @@ struct WriteRequest { WriteRequestType write_req_type = WriteRequestType::DATA; std::string storage_vault_id; bool enable_table_memtable_backpressure = false; + bool cloud_sink_upload = false; }; struct TabletAddRowsPayload { diff --git a/be/src/load/delta_writer/delta_writer_v2.cpp b/be/src/load/delta_writer/delta_writer_v2.cpp index d5a40c43fcf8f8..5f70b1b86a7a11 100644 --- a/be/src/load/delta_writer/delta_writer_v2.cpp +++ b/be/src/load/delta_writer/delta_writer_v2.cpp @@ -28,6 +28,7 @@ #include #include +#include "cloud/cloud_storage_engine.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" #include "common/logging.h" @@ -36,20 +37,20 @@ #include "exec/sink/load_stream_stub.h" #include "io/fs/file_writer.h" // IWYU pragma: keep #include "load/memtable/memtable_memory_limiter.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/query_context.h" #include "service/backend_options.h" #include "storage/data_dir.h" #include "storage/index/inverted/inverted_index_desc.h" #include "storage/olap_define.h" -#include "storage/rowset/beta_rowset.h" #include "storage/rowset/beta_rowset_writer_v2.h" +#include "storage/rowset/rowset_factory.h" #include "storage/rowset/rowset_meta.h" #include "storage/rowset/rowset_writer.h" #include "storage/rowset/rowset_writer_context.h" #include "storage/schema.h" #include "storage/schema_change/schema_change.h" -#include "storage/segment/segment.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet_manager.h" #include "storage/tablet/tablet_schema.h" @@ -133,14 +134,20 @@ Status DeltaWriterV2::init() { context.memtable_on_sink_support_index_v2 = true; context.encrypt_algorithm = EncryptionAlgorithmPB::PLAINTEXT; - _rowset_writer = std::make_shared(_streams); - RETURN_IF_ERROR(_rowset_writer->init(context)); + if (_req.cloud_sink_upload) { + RETURN_IF_ERROR(_init_sink_upload_writer(context)); + } else { + _rowset_writer = std::make_shared(_streams); + RETURN_IF_ERROR(_rowset_writer->init(context)); + } RETURN_IF_ERROR(_memtable_writer->init(_rowset_writer, _tablet_schema, _partial_update_info, _workload_group, _streams[0]->enable_unique_mow(_req.index_id))); ExecEnv::GetInstance()->memtable_memory_limiter()->register_writer(_memtable_writer); _is_init = true; - _streams.clear(); + if (!_req.cloud_sink_upload) { + _streams.clear(); + } return Status::OK(); } @@ -198,6 +205,87 @@ Status DeltaWriterV2::close() { return _memtable_writer->close(); } +Status DeltaWriterV2::_init_sink_upload_writer(RowsetWriterContext& context) { + DORIS_CHECK(config::is_cloud_mode()); + DORIS_CHECK_EQ(_streams.size(), 1); + // The local id identifies this writer, including unshared writers on the same BE. + _sink_writer_id = context.rowset_id.to_string(); + PCloudLoadWriteContext target_write_context; + RETURN_IF_ERROR(_streams[0]->register_sink_upload_writer(_req.partition_id, _req.index_id, + _req.tablet_id, _sink_writer_id, + &target_write_context)); + if (context.enable_unique_key_merge_on_write) { + if (!target_write_context.has_mow_snapshot() || + !target_write_context.mow_snapshot().has_version() || + !target_write_context.mow_snapshot().has_delete_bitmap()) { + return Status::NotSupported("target BE does not support MOW sink upload"); + } + RETURN_IF_ERROR( + _init_mow_context_from_snapshot(context, target_write_context.mow_snapshot())); + } + const auto& meta = target_write_context.rowset_meta(); + context.rowset_id.init(meta.rowset_id_v2()); + context.tablet_schema_hash = meta.tablet_schema_hash(); + context.txn_expiration = meta.txn_expiration(); + context.newest_write_timestamp = meta.newest_write_timestamp(); + context.encrypt_algorithm = target_write_context.encrypt_algorithm(); + context.file_cache_expiration_time = target_write_context.file_cache_expiration_time(); + context.write_file_cache = _req.write_file_cache; + // Keep upload-time caching only when this sink is also the target BE. Older targets + // do not advertise warmup, so retain their existing sink-side cache behavior. + // Stream Load may leave the fragment's source backend id unset (-1). + context.disable_file_cache = + target_write_context.warm_up_file_cache() && + ExecEnv::GetInstance()->cluster_info()->backend_id != _streams[0]->dst_id(); + if (meta.has_inverted_index_storage_format()) { + context.inverted_index_storage_format = meta.inverted_index_storage_format(); + } + context.persist_inverted_index_storage_format = meta.has_inverted_index_storage_format(); + auto& engine = static_cast(ExecEnv::GetInstance()->storage_engine()); + context.storage_resource = engine.get_storage_resource(meta.resource_id()); + if (!context.storage_resource) { + return Status::InternalError("sink upload storage resource {} unavailable", + meta.resource_id()); + } + _segment_start_id = target_write_context.segment_start_id(); + context.is_partial_output_writer = true; + context.enable_segcompaction = false; + context.memtable_on_sink_support_index_v2 = false; + // Preserve whole-rowset first-segment packing, not first-segment-per-writer packing. + context.allow_packed_file = _segment_start_id == 0; + _rowset_writer = DORIS_TRY(RowsetFactory::create_rowset_writer(engine, context, false)); + _rowset_writer->set_segment_start_id(_segment_start_id, + target_write_context.segment_capacity()); + _req.txn_expiration = context.txn_expiration; + return Status::OK(); +} + +Status DeltaWriterV2::_init_mow_context_from_snapshot(RowsetWriterContext& context, + const PCloudLoadMowSnapshot& snapshot) { + auto& engine = static_cast(ExecEnv::GetInstance()->storage_engine()); + context.tablet = DORIS_TRY(engine.get_tablet(_req.tablet_id)); + auto ids = std::make_shared(); + std::vector rowsets; + rowsets.reserve(snapshot.rowsets_size()); + for (const auto& meta : snapshot.rowsets()) { + auto rowset_meta = std::make_shared(); + if (!rowset_meta->init_from_pb(meta)) { + return Status::InvalidArgument("invalid sink MOW snapshot rowset"); + } + RowsetSharedPtr rowset; + RETURN_IF_ERROR(RowsetFactory::create_rowset(rowset_meta->tablet_schema(), "", rowset_meta, + &rowset)); + ids->insert(rowset->rowset_id()); + rowsets.push_back(std::move(rowset)); + } + context.mow_context = std::make_shared( + snapshot.version(), _req.txn_id, std::move(ids), std::move(rowsets), + std::make_shared(_req.tablet_id)); + context.mow_context->snapshot_delete_bitmap = std::make_shared( + DeleteBitmap::from_pb(snapshot.delete_bitmap(), _req.tablet_id)); + return Status::OK(); +} + Status DeltaWriterV2::close_wait(int32_t& num_segments, RuntimeProfile* profile) { SCOPED_RAW_TIMER(&_close_wait_time); std::lock_guard l(_lock); @@ -208,12 +296,51 @@ Status DeltaWriterV2::close_wait(int32_t& num_segments, RuntimeProfile* profile) _update_profile(profile); } RETURN_IF_ERROR(_memtable_writer->close_wait(profile)); - num_segments = _rowset_writer->next_segment_id(); + num_segments = _rowset_writer->get_allocated_segment_id() - _segment_start_id; + if (_req.cloud_sink_upload) { + RETURN_IF_ERROR(_finish_sink_upload(profile)); + } _delta_written_success = true; return Status::OK(); } +Status DeltaWriterV2::_finish_sink_upload(RuntimeProfile* profile) { + RowsetSharedPtr partial; + RETURN_IF_ERROR(_rowset_writer->build(partial)); + if (UnixSeconds() >= _req.txn_expiration) { + return Status::TimedOut("sink upload transaction {} expired", _req.txn_id); + } + DBUG_EXECUTE_IF("DeltaWriterV2.sink_upload.after_upload_failure", + { return Status::InternalError("injected failure after sink upload"); }); + PCloudLoadMowResult mow_result; + const auto& mow_context = _rowset_writer->context().mow_context; + if (mow_context != nullptr) { + // CloudRowsetWriter::build has waited for all per-segment bitmap tasks. + mow_result.set_snapshot_version(mow_context->max_version); + *mow_result.mutable_delete_bitmap() = mow_context->delete_bitmap->to_pb(); + DBUG_EXECUTE_IF("DeltaWriterV2.sink_mow.after_bitmap_failure", { + return Status::InternalError("injected failure after sink MOW bitmap calculation"); + }); + if (profile != nullptr) { + profile->add_info_string("CloudMemtableMowBitmap", "true"); + } + } + const auto* result = mow_context != nullptr ? &mow_result : nullptr; + RETURN_IF_ERROR(_streams[0]->add_partial_rowset( + _req.partition_id, _req.index_id, _req.tablet_id, _sink_writer_id, + partial->rowset_meta()->get_rowset_pb(), result)); + DBUG_EXECUTE_IF("DeltaWriterV2.sink_upload.duplicate_result", { + RETURN_IF_ERROR(_streams[0]->add_partial_rowset( + _req.partition_id, _req.index_id, _req.tablet_id, _sink_writer_id, + partial->rowset_meta()->get_rowset_pb(), result)); + }); + if (profile != nullptr) { + profile->add_info_string("CloudMemtableSinkUpload", "true"); + } + return Status::OK(); +} + Status DeltaWriterV2::cancel() { return cancel_with_status(Status::Cancelled("already cancelled")); } diff --git a/be/src/load/delta_writer/delta_writer_v2.h b/be/src/load/delta_writer/delta_writer_v2.h index 337426515c2796..cf7a86cef6a8ef 100644 --- a/be/src/load/delta_writer/delta_writer_v2.h +++ b/be/src/load/delta_writer/delta_writer_v2.h @@ -50,7 +50,6 @@ class StorageEngine; class TupleDescriptor; class SlotDescriptor; class OlapTableSchemaParam; -class BetaRowsetWriterV2; class LoadStreamStub; class WorkloadGroup; @@ -88,6 +87,10 @@ class DeltaWriterV2 { const OlapTableSchemaParam* table_schema_param, const TabletSchema& ori_tablet_schema); int64_t _table_id() const; + Status _init_sink_upload_writer(RowsetWriterContext& context); + Status _finish_sink_upload(RuntimeProfile* profile); + Status _init_mow_context_from_snapshot(RowsetWriterContext& context, + const PCloudLoadMowSnapshot& snapshot); void _update_profile(RuntimeProfile* profile); @@ -95,7 +98,9 @@ class DeltaWriterV2 { bool _is_cancelled = false; WriteRequest _req; std::shared_ptr _workload_group; - std::shared_ptr _rowset_writer; + std::shared_ptr _rowset_writer; + std::string _sink_writer_id; + int32_t _segment_start_id = 0; TabletSchemaSPtr _tablet_schema; bool _delta_written_success = false; diff --git a/be/src/service/http/action/stream_load.cpp b/be/src/service/http/action/stream_load.cpp index 8f47edaba8aa31..8eb1b295220857 100644 --- a/be/src/service/http/action/stream_load.cpp +++ b/be/src/service/http/action/stream_load.cpp @@ -780,6 +780,10 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req, bool value = iequal(http_req->header(HTTP_MEMTABLE_ON_SINKNODE), "true"); request.__set_memtable_on_sink_node(value); } + if (!http_req->header(HTTP_CLOUD_MEMTABLE_SINK_UPLOAD).empty()) { + bool value = iequal(http_req->header(HTTP_CLOUD_MEMTABLE_SINK_UPLOAD), "true"); + request.__set_cloud_memtable_sink_upload(value); + } if (!http_req->header(HTTP_LOAD_STREAM_PER_NODE).empty()) { int stream_per_node = DORIS_TRY( safe_stoi(http_req->header(HTTP_LOAD_STREAM_PER_NODE), HTTP_LOAD_STREAM_PER_NODE)); diff --git a/be/src/service/http/http_common.h b/be/src/service/http/http_common.h index f250e22a278e55..ba54c7bf5baad2 100644 --- a/be/src/service/http/http_common.h +++ b/be/src/service/http/http_common.h @@ -66,6 +66,7 @@ static const std::string HTTP_TWO_PHASE_COMMIT = "two_phase_commit"; static const std::string HTTP_TXN_ID_KEY = "txn_id"; static const std::string HTTP_TXN_OPERATION_KEY = "txn_operation"; static const std::string HTTP_MEMTABLE_ON_SINKNODE = "memtable_on_sink_node"; +static const std::string HTTP_CLOUD_MEMTABLE_SINK_UPLOAD = "cloud_memtable_sink_upload"; static const std::string HTTP_LOAD_STREAM_PER_NODE = "load_stream_per_node"; static const std::string HTTP_WAL_ID_KY = "wal_id"; static const std::string HTTP_AUTH_CODE = "auth_code"; // deprecated diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 0b05f8fdeb4db8..d619068fae44ca 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -470,6 +470,9 @@ struct MowContext { std::shared_ptr rowset_ids; std::vector rowset_ptrs; std::shared_ptr delete_bitmap; + // Sink-upload sinks use rowset_ptrs and this immutable target snapshot instead of + // their local tablet's potentially newer rowsets/deletion state. + std::shared_ptr snapshot_delete_bitmap; }; // used for controll compaction diff --git a/be/src/storage/rowset/beta_rowset_writer.cpp b/be/src/storage/rowset/beta_rowset_writer.cpp index daec6547d644c1..bbca749fcf2542 100644 --- a/be/src/storage/rowset/beta_rowset_writer.cpp +++ b/be/src/storage/rowset/beta_rowset_writer.cpp @@ -71,19 +71,6 @@ using namespace ErrorCode; namespace { -bool is_segment_overlapping(const std::vector& segments_encoded_key_bounds) { - std::string_view last; - for (auto&& segment_encode_key : segments_encoded_key_bounds) { - auto&& cur_min = segment_encode_key.min_key(); - auto&& cur_max = segment_encode_key.max_key(); - if (cur_min <= last) { - return true; - } - last = cur_max; - } - return false; -} - bool copy_key_bounds_with_truncation(const KeyBoundsPB& src, KeyBoundsPB* dst) { DCHECK(dst != nullptr); if (config::random_segments_key_bounds_truncation) { @@ -420,7 +407,9 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { return Status::OK(); } std::vector specified_rowsets; - { + if (_context.mow_context->snapshot_delete_bitmap != nullptr) { + specified_rowsets = _context.mow_context->rowset_ptrs; + } else { std::shared_lock meta_rlock(_context.tablet->get_header_lock()); specified_rowsets = _context.tablet->get_rowset_by_ids(_context.mow_context->rowset_ids.get()); @@ -483,7 +472,7 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { st = BaseTablet::calc_delete_bitmap(_context.tablet, rowset_ptr, segments, specified_rowsets, _context.mow_context->delete_bitmap, _context.mow_context->max_version, nullptr, nullptr, - nullptr); + _context.mow_context->snapshot_delete_bitmap); if (!st.ok()) { return st; } @@ -1064,8 +1053,9 @@ Status BaseBetaRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool ch std::vector segment_rows; std::vector segment_ids; std::optional segments_key_bounds_truncated; - const bool record_segment_ids = _context.write_type == DataWriteType::TYPE_COMPACTION && - (_context.is_partial_output_writer || _segment_start_id != 0); + const bool record_segment_ids = + _context.is_partial_output_writer || + (_context.write_type == DataWriteType::TYPE_COMPACTION && _segment_start_id != 0); { std::lock_guard lock(_segid_statistics_map_mutex); if (record_segment_ids) { diff --git a/be/src/storage/rowset/beta_rowset_writer.h b/be/src/storage/rowset/beta_rowset_writer.h index 918edb057f0bdc..1baf6cc00b5c78 100644 --- a/be/src/storage/rowset/beta_rowset_writer.h +++ b/be/src/storage/rowset/beta_rowset_writer.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include "common/status.h" @@ -46,6 +47,21 @@ namespace doris { class Block; +// NONOVERLAPPING requires key ranges to be ordered and disjoint. +template +bool is_segment_overlapping(const KeyBoundsRange& segments_encoded_key_bounds) { + std::string_view last; + for (auto&& segment_encode_key : segments_encoded_key_bounds) { + auto&& cur_min = segment_encode_key.min_key(); + auto&& cur_max = segment_encode_key.max_key(); + if (cur_min <= last) { + return true; + } + last = cur_max; + } + return false; +} + namespace segment_v2 { class VerticalSegmentWriter; } // namespace segment_v2 diff --git a/be/src/storage/rowset/beta_rowset_writer_v2.h b/be/src/storage/rowset/beta_rowset_writer_v2.h index 8aa40ce4cfd75d..263464384df056 100644 --- a/be/src/storage/rowset/beta_rowset_writer_v2.h +++ b/be/src/storage/rowset/beta_rowset_writer_v2.h @@ -120,8 +120,6 @@ class BetaRowsetWriterV2 : public RowsetWriter { return _segment_creator.get_allocated_segment_id(); }; - int32_t next_segment_id() { return _segment_creator.next_segment_id(); }; - int64_t delete_bitmap_ns() override { return _delete_bitmap_ns; } int64_t segment_writer_ns() override { return _segment_writer_ns; } diff --git a/be/src/storage/rowset/rowset_writer_context.h b/be/src/storage/rowset/rowset_writer_context.h index 6a04d4a04e34fd..a3ed5cca4f9c22 100644 --- a/be/src/storage/rowset/rowset_writer_context.h +++ b/be/src/storage/rowset/rowset_writer_context.h @@ -289,6 +289,7 @@ struct RowsetWriterContext { append_info.rowset_id = rowset_id.to_string(); append_info.first_segment_id = first_segment_id; append_info.txn_id = txn_id; + append_info.write_file_cache = !disable_file_cache; append_info.expiration_time = file_cache_expiration_time; fs = std::make_shared(fs, append_info); } diff --git a/be/src/storage/segment/segment_index_file_cache_loader.cpp b/be/src/storage/segment/segment_index_file_cache_loader.cpp index 46a1ffa9525be3..a9c39873612dba 100644 --- a/be/src/storage/segment/segment_index_file_cache_loader.cpp +++ b/be/src/storage/segment/segment_index_file_cache_loader.cpp @@ -87,7 +87,8 @@ Status read_range_to_file_cache(io::FileReaderSPtr reader, uint64_t offset, uint Status SegmentIndexFileCacheLoader::preload_segment_index_to_file_cache( const RowsetWriterContext& context, uint32_t segment_id, const std::string& segment_path, const SegmentIndexFileCacheInfo& info) { - if (!enable_cloud_index_only_file_cache() || context.is_local_rowset()) { + if (!enable_cloud_index_only_file_cache() || context.is_local_rowset() || + context.disable_file_cache) { return Status::OK(); } diff --git a/be/test/cloud/cloud_warm_up_manager_test.cpp b/be/test/cloud/cloud_warm_up_manager_test.cpp index 4ac284b990bf01..90234a8d107105 100644 --- a/be/test/cloud/cloud_warm_up_manager_test.cpp +++ b/be/test/cloud/cloud_warm_up_manager_test.cpp @@ -30,9 +30,14 @@ #include "cloud/cloud_storage_engine.h" #include "cloud/config.h" +#include "common/config.h" #include "cpp/sync_point.h" +#include "runtime/cluster_info.h" +#include "runtime/exec_env.h" +#include "service/backend_options.h" #include "storage/olap_common.h" #include "storage/rowset/rowset_meta.h" +#include "util/defer_op.h" namespace doris { @@ -83,7 +88,13 @@ class CloudWarmUpManagerTest : public testing::Test { int32_t _origin_thread_pool_size = 0; }; -TEST_F(CloudWarmUpManagerTest, NonPositiveTimeoutQueuesBackgroundCopyAndReturns) { +TEST_F(CloudWarmUpManagerTest, LocalWarmupQueuesBackgroundCopyWithoutEventJobs) { + ClusterInfo cluster_info; + cluster_info.backend_id = 123; + auto* original_cluster_info = ExecEnv::GetInstance()->cluster_info(); + ExecEnv::GetInstance()->set_cluster_info(&cluster_info); + Defer restore_cluster_info { + [&] { ExecEnv::GetInstance()->set_cluster_info(original_cluster_info); }}; config::warm_up_manager_thread_pool_size = 1; CloudWarmUpManager manager(_engine); @@ -118,10 +129,20 @@ TEST_F(CloudWarmUpManagerTest, NonPositiveTimeoutQueuesBackgroundCopyAndReturns) SyncPoint::CallbackGuard warmup_enter_guard; SyncPoint::get_instance()->set_call_back( - "CloudWarmUpManager::_warm_up_rowset.enter", + "CloudWarmUpManager::_do_warm_up_rowset", [&](std::vector&& args) { auto* rs_meta = try_any_cast(args[0]); - auto* timeout_ms = try_any_cast(args[1]); + auto* replicas = try_any_cast*>(args[1]); + auto* timeout_ms = try_any_cast(args[2]); + auto* skip_existence_check = try_any_cast(args[3]); + auto* result = try_any_cast*>(args.back()); + result->second = true; + ASSERT_EQ(1, replicas->size()); + EXPECT_EQ(0, replicas->at(0).job_id); + EXPECT_EQ(123, replicas->at(0).replica.backend_id); + EXPECT_EQ(BackendOptions::get_localhost(), replicas->at(0).replica.host); + EXPECT_EQ(config::brpc_port, replicas->at(0).replica.brpc_port); + EXPECT_FALSE(*skip_existence_check); { std::lock_guard lock(observed_mtx); observed_tablet_id = rs_meta->tablet_id(); @@ -138,7 +159,8 @@ TEST_F(CloudWarmUpManagerTest, NonPositiveTimeoutQueuesBackgroundCopyAndReturns) std::atomic returned = false; std::thread caller([&] { - manager.warm_up_rowset(*rs_meta, /*table_id=*/0, /*sync_wait_timeout_ms=*/-1); + manager.warm_up_rowset(*rs_meta, /*table_id=*/0, /*sync_wait_timeout_ms=*/-1, + /*warm_up_local=*/true); returned = true; }); diff --git a/be/test/load/channel/load_stream_stub_map_test.cpp b/be/test/load/channel/load_stream_stub_map_test.cpp index 430d4f71a0cf75..080d82a705775d 100644 --- a/be/test/load/channel/load_stream_stub_map_test.cpp +++ b/be/test/load/channel/load_stream_stub_map_test.cpp @@ -16,6 +16,8 @@ // under the License. #include +#include + #include "exec/sink/load_stream_map_pool.h" #include "exec/sink/load_stream_stub.h" @@ -56,4 +58,15 @@ TEST_F(LoadStreamMapPoolTest, test) { EXPECT_EQ(0, pool.size()); } +TEST_F(LoadStreamMapPoolTest, AccumulateIndependentWriterSegmentCounts) { + LoadStreamMap map(UniqueId(1, 2), 10, 1, 2, nullptr); + std::thread first([&] { map.save_segments_for_tablet({{100, 2}, {200, 1}}); }); + std::thread second([&] { map.save_segments_for_tablet({{100, 3}, {300, 4}}); }); + first.join(); + second.join(); + EXPECT_EQ(5, map._segments_for_tablet.at(100)); + EXPECT_EQ(1, map._segments_for_tablet.at(200)); + EXPECT_EQ(4, map._segments_for_tablet.at(300)); +} + } // namespace doris diff --git a/be/test/runtime/load_stream_test.cpp b/be/test/runtime/load_stream_test.cpp index ce218d1ceff14b..6e1c46c6052a4f 100644 --- a/be/test/runtime/load_stream_test.cpp +++ b/be/test/runtime/load_stream_test.cpp @@ -33,6 +33,7 @@ #include #include +#include "cloud/cloud_rowset_builder.h" #include "common/config.h" #include "common/status.h" #include "gtest/gtest_pred_impl.h" @@ -1420,4 +1421,250 @@ TEST_F(LoadStreamMgrTest, incremental_close_race_orhpans_streams) { config::enable_debug_points = saved_debug_points; } +TEST_F(LoadStreamMgrTest, SinkUploadDuplicateAndMissingWriterResults) { + WriteRequest req; + req.tablet_id = NORMAL_TABLET_ID; + RuntimeProfile profile("sink-upload-test"); + LoadStreamWriter writer(&req, &profile); + writer._rowset_builder->_tablet = engine_ref->tablet_manager()->get_tablet(NORMAL_TABLET_ID); + writer._writer_segment_start_ids.emplace("writer", 100); + RowsetMetaPB meta; + meta.set_num_segments(1); + meta.add_segment_ids(100); + // Seed an already accepted result to exercise retries without uploading a file. + writer._partial_rowset_metas.emplace(100, meta); + int64_t added = -1; + ASSERT_TRUE(writer.add_partial_rowset("writer", meta, &added).ok()); + EXPECT_EQ(0, added); + EXPECT_FALSE(writer.add_partial_rowset("unknown", meta, &added).ok()); + meta.set_num_rows(7); + EXPECT_FALSE(writer.add_partial_rowset("writer", meta, &added).ok()); + writer._is_init = true; + writer._sink_upload = true; + writer._partial_rowset_metas.clear(); + EXPECT_FALSE(writer.pre_close().ok()); + writer._pre_closed = true; + EXPECT_FALSE(writer.add_partial_rowset("writer", meta, &added).ok()); +} + +class CloudSinkUploadMetaTest : public testing::Test { +protected: + RowsetMetaPB base_meta() { + RowsetMetaPB meta; + meta.set_rowset_id(0); + meta.set_rowset_id_v2("000000000000000000000000000000000000000000000001"); + meta.set_tablet_id(10); + meta.set_txn_id(20); + meta.set_resource_id("vault"); + meta.set_index_id(30); + meta.set_partition_id(40); + meta.mutable_load_id()->set_hi(50); + meta.mutable_load_id()->set_lo(60); + meta.mutable_tablet_schema()->set_keys_type(DUP_KEYS); + meta.mutable_tablet_schema()->set_num_short_key_columns(0); + meta.mutable_tablet_schema()->set_num_rows_per_row_block(1024); + meta.mutable_tablet_schema()->set_compress_kind(COMPRESS_LZ4); + return meta; + } + + RowsetMetaPB partial(int id) { + auto meta = base_meta(); + meta.set_num_segments(1); + meta.add_segment_ids(id); + meta.add_num_segment_rows(id + 1); + meta.set_num_rows(id + 1); + meta.add_segments_file_size(100 + id); + meta.set_data_disk_size(100 + id); + meta.set_index_disk_size(10); + meta.set_total_disk_size(110 + id); + auto* bounds = meta.add_segments_key_bounds(); + bounds->set_min_key("a"); + bounds->set_max_key("z"); + meta.add_inverted_index_file_info()->set_index_size(10); + return meta; + } +}; + +TEST_F(CloudSinkUploadMetaTest, AssembleSparseSegmentsAndPackedLocations) { + auto first = partial(0); + auto* location = &(*first.mutable_packed_slice_locations())["data/10/rowset_0.dat"]; + location->set_packed_file_path("packed/object"); + location->set_offset(64); + location->set_size(100); + RowsetMetaPB merged; + // A writer can finish first even though its allocated range is last. + std::map partials; + partials.emplace(100, partial(102)); + partials.emplace(0, first); + ASSERT_TRUE(CloudRowsetBuilder::assemble_rowset_meta_from_partials(base_meta(), partials, 100, + &merged) + .ok()); + EXPECT_EQ(2, merged.num_segments()); + EXPECT_EQ(0, merged.segment_ids(0)); + EXPECT_EQ(102, merged.segment_ids(1)); + EXPECT_EQ(1, merged.num_segment_rows(0)); + EXPECT_EQ(103, merged.num_segment_rows(1)); + EXPECT_EQ(202, merged.segments_file_size(1)); + EXPECT_EQ(104, merged.num_rows()); + EXPECT_EQ(322, merged.total_disk_size()); + EXPECT_EQ(2, merged.inverted_index_file_info_size()); + EXPECT_EQ(COMMITTED, merged.rowset_state()); + RowsetMetaPB restored; + ASSERT_TRUE(restored.ParseFromString(merged.SerializeAsString())); + EXPECT_EQ(64, restored.packed_slice_locations().at("data/10/rowset_0.dat").offset()); + EXPECT_EQ(102, restored.segment_ids(1)); +} + +TEST_F(CloudSinkUploadMetaTest, RejectInvalidPartialResults) { + RowsetMetaPB merged; + auto check = [&](const RowsetMetaPB& meta) { + EXPECT_FALSE( + CloudRowsetBuilder::validate_partial_rowset_meta(base_meta(), meta, 100, 100).ok()); + }; + check(partial(99)); // Outside this writer's range. + check(partial(200)); + auto meta = partial(100); + meta.set_rowset_id_v2("different-rowset"); + check(meta); + meta = partial(100); + meta.clear_segments_file_size(); + check(meta); + meta = partial(100); + meta.set_num_rows(1); + check(meta); + meta = partial(100); + meta.set_total_disk_size(1); + check(meta); + meta = partial(100); + meta.set_segments_key_bounds_aggregated(true); + check(meta); + meta = partial(100); + meta.mutable_tablet_schema()->add_index()->set_index_type(INVERTED); + meta.clear_inverted_index_file_info(); + check(meta); + EXPECT_FALSE(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + base_meta(), {{0, partial(0)}, {1, partial(1)}}, 1, &merged) + .ok()); +} + +TEST_F(CloudSinkUploadMetaTest, EmptyWriter) { + RowsetMetaPB merged; + ASSERT_TRUE(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + base_meta(), {{0, base_meta()}}, 100, &merged) + .ok()); + EXPECT_EQ(0, merged.num_segments()); + EXPECT_TRUE(merged.empty()); +} + +TEST_F(CloudSinkUploadMetaTest, AssembleVariantSchemaWithoutDuplicateFields) { + auto base = base_meta(); + auto* schema = base.mutable_tablet_schema(); + auto* key = schema->add_column(); + key->set_unique_id(1); + key->set_name("k"); + key->set_type("INT"); + key->set_is_key(true); + key->set_is_nullable(false); + auto* variant = schema->add_column(); + variant->set_unique_id(2); + variant->set_name("v"); + variant->set_type("VARIANT"); + variant->set_is_key(false); + variant->set_is_nullable(true); + auto* index = schema->add_index(); + index->set_index_id(3); + index->set_index_name("idx_k"); + index->set_index_type(INVERTED); + index->add_col_unique_id(1); + schema->add_cluster_key_uids(1); + + auto first = partial(0); + auto second = partial(100); + *first.mutable_tablet_schema() = *schema; + *second.mutable_tablet_schema() = *schema; + RowsetMetaPB merged; + ASSERT_TRUE(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + base, {{0, first}, {100, second}}, 100, &merged) + .ok()); + const auto& merged_schema = merged.tablet_schema(); + ASSERT_EQ(2, merged_schema.column_size()); + EXPECT_EQ(1, merged_schema.column(0).unique_id()); + EXPECT_EQ(2, merged_schema.column(1).unique_id()); + EXPECT_EQ("VARIANT", merged_schema.column(1).type()); + ASSERT_EQ(1, merged_schema.index_size()); + EXPECT_EQ(3, merged_schema.index(0).index_id()); + ASSERT_EQ(1, merged_schema.cluster_key_uids_size()); + EXPECT_EQ(1, merged_schema.cluster_key_uids(0)); +} + +TEST_F(CloudSinkUploadMetaTest, PreserveAggregateAndMorOverlappingLayout) { + for (auto type : {AGG_KEYS, UNIQUE_KEYS}) { + auto base = base_meta(); + base.mutable_tablet_schema()->set_keys_type(type); + auto first = partial(0); + auto second = partial(100); + first.mutable_tablet_schema()->set_keys_type(type); + second.mutable_tablet_schema()->set_keys_type(type); + RowsetMetaPB merged; + ASSERT_TRUE(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + base, {{0, first}, {100, second}}, 100, &merged) + .ok()); + EXPECT_EQ(type, merged.tablet_schema().keys_type()); + EXPECT_EQ(OVERLAPPING, merged.segments_overlap_pb()); + EXPECT_EQ(100, merged.segment_ids(1)); + EXPECT_EQ(102, merged.num_rows()); + } +} + +TEST_F(CloudSinkUploadMetaTest, PreserveDisjointKeyRanges) { + auto first = partial(0); + auto second = partial(100); + first.mutable_segments_key_bounds(0)->set_max_key("b"); + second.mutable_segments_key_bounds(0)->set_min_key("c"); + RowsetMetaPB merged; + auto base = base_meta(); + ASSERT_TRUE(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + base, {{0, first}, {100, second}}, 100, &merged) + .ok()); + EXPECT_EQ(NONOVERLAPPING, merged.segments_overlap_pb()); + + // Primary-key bounds cannot establish the ordering of cluster keys. + base.mutable_tablet_schema()->add_cluster_key_uids(1); + ASSERT_TRUE(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + base, {{0, first}, {100, second}}, 100, &merged) + .ok()); + EXPECT_EQ(OVERLAPPING, merged.segments_overlap_pb()); +} + +TEST(CloudSinkMowTest, RequireMatchingSnapshotAndCompleteBitmap) { + PCloudLoadMowResult result; + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + result.set_snapshot_version(5); + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + auto* bitmap = result.mutable_delete_bitmap(); + EXPECT_TRUE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 6).ok()); + bitmap->add_rowset_ids("020000000000000100000000000000020000000000000003"); + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + bitmap->add_segment_ids(1000); + bitmap->add_versions(0); + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + roaring::Roaring rows; + rows.add(7); + std::string bytes(rows.getSizeInBytes(), '\0'); + rows.write(bytes.data()); + bitmap->add_segment_delete_bitmaps(bytes); + ASSERT_TRUE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + auto decoded = DeleteBitmap::from_pb(*bitmap, 1); + RowsetId rowset_id; + rowset_id.init(bitmap->rowset_ids(0)); + EXPECT_TRUE(decoded.contains({rowset_id, 1000, 0}, 7)); + EXPECT_FALSE(decoded.contains({rowset_id, 0, 0}, 7)); + bitmap->set_versions(0, 5); + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + bitmap->set_versions(0, 0); + bitmap->set_segment_delete_bitmaps(0, "invalid"); + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); +} + } // namespace doris diff --git a/be/test/service/http/stream_load_test.cpp b/be/test/service/http/stream_load_test.cpp index 93dd9d044c7c8b..d3a88e79ce98b6 100644 --- a/be/test/service/http/stream_load_test.cpp +++ b/be/test/service/http/stream_load_test.cpp @@ -27,6 +27,7 @@ #include "event2/http.h" #include "event2/http_struct.h" #include "evhttp.h" +#include "gen_cpp/FrontendService_types.h" #include "load/group_commit/wal/wal_manager.h" #include "load/stream_load/stream_load_context.h" #include "runtime/exec_env.h" @@ -43,6 +44,16 @@ namespace doris { +TEST(StreamLoadRequestTest, SinkUploadPreservesOptionalPresence) { + TStreamLoadPutRequest request; + EXPECT_FALSE(request.__isset.cloud_memtable_sink_upload); + for (bool enabled : {false, true}) { + request.__set_cloud_memtable_sink_upload(enabled); + EXPECT_TRUE(request.__isset.cloud_memtable_sink_upload); + EXPECT_EQ(request.cloud_memtable_sink_upload, enabled); + } +} + class StreamLoadTest : public testing::Test { public: StreamLoadTest() = default; diff --git a/be/test/storage/cloud_file_cache_write_index_only_test.cpp b/be/test/storage/cloud_file_cache_write_index_only_test.cpp index 445768e7d63a73..b6e9ed042ee3dd 100644 --- a/be/test/storage/cloud_file_cache_write_index_only_test.cpp +++ b/be/test/storage/cloud_file_cache_write_index_only_test.cpp @@ -603,6 +603,35 @@ TEST_F(CloudFileCacheWriteIndexOnlyConfigTest, EXPECT_EQ(open_file_count, 0); } +TEST_F(CloudFileCacheWriteIndexOnlyTest, RemoteSinkSkipsIndexPreload) { + auto context = create_context(create_schema(true)); + context.disable_file_cache = true; + int load_count = 0; + SyncPoint::CallbackGuard guard; + SyncPoint::get_instance()->set_call_back( + "SegmentIndexFileCacheLoader::load_segment_index_to_file_cache", + [&](auto&& args) { + ++load_count; + auto* ret = try_any_cast_ret(args); + ret->first = Status::OK(); + ret->second = true; + }, + &guard); + SyncPoint::get_instance()->enable_processing(); + segment_v2::SegmentIndexFileCacheInfo info; + info.segment_file_size = 2; + info.index_ranges.push_back({.offset = 1, .size = 1}); + ASSERT_TRUE(segment_v2::SegmentIndexFileCacheLoader::preload_segment_index_to_file_cache( + context, 0, "remote_sink.dat", info) + .ok()); + EXPECT_EQ(0, load_count); + context.disable_file_cache = false; + ASSERT_TRUE(segment_v2::SegmentIndexFileCacheLoader::preload_segment_index_to_file_cache( + context, 0, "remote_sink.dat", info) + .ok()); + EXPECT_EQ(1, load_count); +} + TEST_F(CloudFileCacheWriteIndexOnlyTest, LoadUsesTheWholeBlockWritePathAndPreloadsAfterAllSegmentFilesClosed) { auto tablet_schema = create_schema(true); diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 40a1b8dfcfb1c6..8e7583ab87c7e2 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -622,6 +622,11 @@ public class Config extends ConfigBase { + "in stream load") public static boolean stream_load_default_memtable_on_sink_node = false; + @ConfField(mutable = true, masterOnly = true, description = "Upload cloud memtable segments from sink BEs " + + "to object storage in HTTP Stream, Routine Load, and Group Commit internal loads. " + + "Ordinary Stream Load uses this default unless the cloud_memtable_sink_upload header is set.") + public static boolean cloud_stream_load_default_memtable_sink_upload = true; + @ConfField(mutable = true, masterOnly = true, description = "Whether to enable forwarding group commit stream " + "load to follower nodes. If true, stream load with " + "group commit mode will be forwarded to a follower FE " + "round robin.") diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index 99cd1439fb64e2..743248cdf2b300 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -386,6 +386,8 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme // Generate loading task and init the plan of task LoadLoadingTask task = createTask(db, table, brokerFileGroups, isEnableMemtableOnSinkNode, batchSize, aggKey, attachment); + task.setCloudMemtableSinkUpload(Boolean.parseBoolean(sessionVariables.getOrDefault( + SessionVariable.ENABLE_CLOUD_MEMTABLE_SINK_UPLOAD, "true"))); idToTasks.put(task.getSignature(), task); // idToTasks contains previous LoadPendingTasks, so idToTasks is just used to save all tasks. // use newLoadingTasks to save new created loading tasks and submit them later. diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java index 05fe3f637e3201..0246533a05d96b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java @@ -110,6 +110,8 @@ public BulkLoadJob(EtlJobType jobType, long dbId, String label, if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); sessionVariables.put(SessionVariable.SQL_MODE, Long.toString(var.getSqlMode())); + sessionVariables.put(SessionVariable.ENABLE_CLOUD_MEMTABLE_SINK_UPLOAD, + Boolean.toString(var.enableCloudMemtableSinkUpload)); sessionVariables.put(SessionVariable.AUTO_PROFILE_THRESHOLD_MS, Long.toString(var.getAutoProfileThresholdMs())); sessionVariables.put(SessionVariable.PROFILE_LEVEL, Long.toString(var.getProfileLevel())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java index 3993ab2dd498b5..b91087d2d727ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java @@ -84,6 +84,7 @@ public class LoadLoadingTask extends LoadTask { private final boolean singleTabletLoadPerSink; private final boolean enableMemTableOnSinkNode; + private boolean cloudMemtableSinkUpload; private final int batchSize; private NereidsLoadingTaskPlanner planner; @@ -147,6 +148,10 @@ public void setLoadBackendSelectionHint(BackendSelection.SelectionHint hint) { loadBackendSelectionHint = hint; } + public void setCloudMemtableSinkUpload(boolean enabled) { + cloudMemtableSinkUpload = enabled; + } + public TUniqueId getLoadId() { return loadId; } @@ -190,6 +195,7 @@ protected void executeOnce() throws Exception { curCoordinator.setExecMemoryLimit(execMemLimit); curCoordinator.setMemTableOnSinkNode(enableMemTableOnSinkNode); + curCoordinator.getQueryOptions().setEnableCloudMemtableSinkUpload(cloudMemtableSinkUpload); if (enableMemTableOnSinkNode) { curCoordinator.setBatchSize(batchSize); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadTaskInfo.java index 2ece54c823ecfb..8b847837ff2f9a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadTaskInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadTaskInfo.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.Separator; import org.apache.doris.catalog.info.PartitionNamesInfo; +import org.apache.doris.common.Config; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.thrift.TFileCompressType; @@ -147,6 +148,10 @@ default boolean isMemtableOnSinkNode() { return false; } + default boolean isCloudMemtableSinkUpload() { + return Config.cloud_stream_load_default_memtable_sink_upload; + } + default int getStreamPerNode() { return 2; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java index 78896eae5aa6a2..d3082d5264470d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java @@ -320,6 +320,7 @@ public TPipelineFragmentParams plan(TUniqueId loadId, int fragmentInstanceIdInde ? taskInfo.isMemtableOnSinkNode() : false; queryOptions.setEnableMemtableOnSinkNode(enableMemtableOnSinkNode); + queryOptions.setEnableCloudMemtableSinkUpload(taskInfo.isCloudMemtableSinkUpload()); queryOptions.setNewVersionUnixTimestamp(true); queryOptions.setNewVersionPercentile(true); queryOptions.setNewVersionBitmapOpCount(true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadTask.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadTask.java index f5ddca41f19a29..dbd89058849686 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadTask.java @@ -90,6 +90,7 @@ public class NereidsStreamLoadTask implements NereidsLoadTaskInfo { private boolean enableProfile = false; private boolean memtableOnSinkNode = false; + private boolean cloudMemtableSinkUpload = true; private int streamPerNode = 2; private byte enclose = 0; @@ -327,6 +328,11 @@ public boolean isMemtableOnSinkNode() { return memtableOnSinkNode; } + @Override + public boolean isCloudMemtableSinkUpload() { + return cloudMemtableSinkUpload; + } + public void setMemtableOnSinkNode(boolean memtableOnSinkNode) { this.memtableOnSinkNode = memtableOnSinkNode; } @@ -368,6 +374,7 @@ public static NereidsStreamLoadTask fromTStreamLoadPutRequest(TStreamLoadPutRequ * setMultiTableBaseTaskInfo */ public void setMultiTableBaseTaskInfo(LoadTaskInfo task) throws UserException { + this.cloudMemtableSinkUpload = Config.cloud_stream_load_default_memtable_sink_upload; this.mergeType = task.getMergeType(); this.columnSeparator = task.getColumnSeparator(); this.whereExpr = task.getWhereExpr() != null ? parseWhereExpr( @@ -516,6 +523,11 @@ private void setOptionalFromTSLPutRequest(TStreamLoadPutRequest request) throws } else { this.memtableOnSinkNode = Config.stream_load_default_memtable_on_sink_node; } + if (request.isSetCloudMemtableSinkUpload()) { + this.cloudMemtableSinkUpload = request.isCloudMemtableSinkUpload(); + } else { + this.cloudMemtableSinkUpload = Config.cloud_stream_load_default_memtable_sink_upload; + } if (request.isSetStreamPerNode()) { this.streamPerNode = request.getStreamPerNode(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 9e40aa4869c539..fa4cb4e619113e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -651,6 +651,8 @@ public String toString() { public static final String ENABLE_MEMTABLE_ON_SINK_NODE = "enable_memtable_on_sink_node"; + public static final String ENABLE_CLOUD_MEMTABLE_SINK_UPLOAD = "enable_cloud_memtable_sink_upload"; + public static final String LOAD_STREAM_PER_NODE = "load_stream_per_node"; public static final String ENABLE_UNIQUE_KEY_PARTIAL_UPDATE = "enable_unique_key_partial_update"; @@ -2694,6 +2696,9 @@ public Map getForceEagerAggHintMap() { @VarAttrDef.VarAttr(name = ENABLE_MEMTABLE_ON_SINK_NODE, needForward = true) public boolean enableMemtableOnSinkNode = true; + @VarAttrDef.VarAttr(name = ENABLE_CLOUD_MEMTABLE_SINK_UPLOAD, needForward = true) + public boolean enableCloudMemtableSinkUpload = true; + @VarAttrDef.VarAttr(name = LOAD_STREAM_PER_NODE) public int loadStreamPerNode = 2; @@ -5554,6 +5559,7 @@ public TQueryOptions toThrift() { tResult.setTruncateCharOrVarcharColumns(truncateCharOrVarcharColumns); tResult.setEnableMemtableOnSinkNode(enableMemtableOnSinkNode); + tResult.setEnableCloudMemtableSinkUpload(enableCloudMemtableSinkUpload); tResult.setInvertedIndexConjunctionOptThreshold(invertedIndexConjunctionOptThreshold); tResult.setInvertedIndexMaxExpansions(invertedIndexMaxExpansions); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 2f98052fd9a321..70e929a5b71bcc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -3205,6 +3205,7 @@ private void httpStreamPutImpl(TStreamLoadPutRequest request, TStreamLoadPutResu } else { ctx.getSessionVariable().enableMemtableOnSinkNode = Config.stream_load_default_memtable_on_sink_node; } + ctx.getSessionVariable().enableCloudMemtableSinkUpload = Config.cloud_stream_load_default_memtable_sink_upload; ctx.getSessionVariable().groupCommit = request.getGroupCommitMode(); ctx.getSessionVariable().setEnableInsertStrict(false); if (request.isSetPartialUpdate() && !request.isPartialUpdate()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java index 9c488169ab4c70..ccfa6fc5aa38d6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java @@ -58,6 +58,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -206,8 +208,9 @@ public void testPendingTaskOnFinishedWithDuplicated() { Assertions.assertEquals(0, idToTasks.size()); } - @Test - public void testPendingTaskOnFinished() throws Exception { + @ParameterizedTest + @CsvSource({"false", "true"}) + public void testPendingTaskOnFinished(boolean sinkUpload) throws Exception { BrokerPendingTaskAttachment attachment = Mockito.mock(BrokerPendingTaskAttachment.class); Env env = Mockito.mock(Env.class); InternalCatalog catalog = Mockito.mock(InternalCatalog.class); @@ -240,6 +243,7 @@ public void testPendingTaskOnFinished() throws Exception { BrokerLoadJob brokerLoadJob = new BrokerLoadJob(); Deencapsulation.setField(brokerLoadJob, "state", JobState.LOADING); + brokerLoadJob.sessionVariables.put("enable_cloud_memtable_sink_upload", Boolean.toString(sinkUpload)); BrokerDesc brokerDesc = Mockito.mock(BrokerDesc.class); Deencapsulation.setField(brokerLoadJob, "brokerDesc", brokerDesc); long taskId = 1L; @@ -306,6 +310,10 @@ public void testPendingTaskOnFinished() throws Exception { Assertions.assertEquals(true, finishedTaskIds.contains(taskId)); Map idToTasks = Deencapsulation.getField(brokerLoadJob, "idToTasks"); Assertions.assertEquals(3, idToTasks.size()); + for (LoadTask task : idToTasks.values()) { + boolean actualSinkUpload = Deencapsulation.getField(task, "cloudMemtableSinkUpload"); + Assertions.assertEquals(sinkUpload, actualSinkUpload); + } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java index 39cff42be39064..a63d1694323ce9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java @@ -17,12 +17,18 @@ package org.apache.doris.planner; +import org.apache.doris.common.Config; import org.apache.doris.common.IdGenerator; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.load.NereidsLoadUtils; +import org.apache.doris.nereids.load.NereidsStreamLoadTask; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileType; +import org.apache.doris.thrift.TStreamLoadPutRequest; +import org.apache.doris.thrift.TUniqueId; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -37,6 +43,30 @@ public void testParseStmt() throws Exception { Assertions.assertEquals(4, expressions.size()); } + @Test + public void testStreamLoadSinkUploadRequestOverridesDefaultConfig() throws Exception { + boolean original = Config.cloud_stream_load_default_memtable_sink_upload; + try { + for (boolean configured : new boolean[] {false, true}) { + Config.cloud_stream_load_default_memtable_sink_upload = configured; + TStreamLoadPutRequest request = new TStreamLoadPutRequest(); + request.setLoadId(new TUniqueId(1, 2)); + request.setTxnId(3); + request.setFileType(TFileType.FILE_STREAM); + request.setFormatType(TFileFormatType.FORMAT_CSV_PLAIN); + Assertions.assertEquals(configured, NereidsStreamLoadTask.fromTStreamLoadPutRequest(request) + .isCloudMemtableSinkUpload()); + for (boolean requested : new boolean[] {false, true}) { + request.setCloudMemtableSinkUpload(requested); + Assertions.assertEquals(requested, NereidsStreamLoadTask.fromTStreamLoadPutRequest(request) + .isCloudMemtableSinkUpload()); + } + } + } finally { + Config.cloud_stream_load_default_memtable_sink_upload = original; + } + } + @Test public void testExprIdGenerator() { IdGenerator exprIdGenerator1 = StatementScopeIdGenerator.getExprIdGenerator(); diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 9d9bd1a496a2f9..b17b90680c8dff 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -1031,6 +1031,27 @@ message PTabletSchemaWithIndex { optional bool enable_unique_key_merge_on_write = 3; } +// All writers of one MOW load compare against the same rowsets and deletion state. +message PCloudLoadMowSnapshot { + optional int64 version = 1; + repeated RowsetMetaPB rowsets = 2; + optional DeleteBitmapPB delete_bitmap = 3; +} + +// A writer owns one stable segment range for the lifetime of a load attempt. +message PCloudLoadWriteContext { + optional string writer_id = 1; + optional RowsetMetaPB rowset_meta = 2; + optional int32 segment_start_id = 3; + optional int32 segment_capacity = 4; + optional EncryptionAlgorithmPB encrypt_algorithm = 5; + // Absolute cache expiration timestamp in seconds; 0 means no TTL. + optional int64 file_cache_expiration_time = 6; + // The target can asynchronously cache completed sink-upload rowsets. + optional bool warm_up_file_cache = 7 [default = false]; + optional PCloudLoadMowSnapshot mow_snapshot = 8; +} + message POpenLoadStreamResponse { optional PStatus status = 1; repeated PTabletSchemaWithIndex tablet_schemas = 2; @@ -1050,6 +1071,12 @@ message PLoadStreamResponse { repeated PTabletSchemaWithIndex tablet_schemas = 5; optional bool eos = 6; repeated PTabletLoadRowsetInfo tablet_load_rowset_num_infos = 7; + optional PCloudLoadWriteContext write_context = 8; +} + +message PCloudLoadMowResult { + optional int64 snapshot_version = 1; + optional DeleteBitmapPB delete_bitmap = 2; } message PStreamHeader { @@ -1058,6 +1085,8 @@ message PStreamHeader { CLOSE_LOAD = 2; ADD_SEGMENT = 3; GET_SCHEMA = 4; + GET_WRITE_CONTEXT = 5; + ADD_PARTIAL_ROWSET = 6; } optional PUniqueId load_id = 1; optional int64 partition_id = 2; @@ -1073,6 +1102,9 @@ message PStreamHeader { optional uint64 offset = 12; optional FileType file_type = 13; optional int64 num_incremental_streams = 14; + optional string writer_id = 15; + optional RowsetMetaPB partial_rowset_meta = 16; + optional PCloudLoadMowResult mow_result = 17; } message PGetWalQueueSizeRequest{ diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index fb3f4c3860b8ba..5b207f1062200a 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -632,6 +632,7 @@ struct TStreamLoadPutRequest { // For cloud 1000: optional string cloud_cluster 1001: optional i64 table_id + 1002: optional bool cloud_memtable_sink_upload } struct TStreamLoadPutResult { diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 3f80d4ae5a6557..eb6d24f20ec3a3 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -540,6 +540,7 @@ struct TQueryOptions { // reads the data instead. // Defaults to false because an old FE never sends this field, and BE checked both cases before. 1006: optional bool force_pushdown_zonemap_minmax = false + 1007: optional bool enable_cloud_memtable_sink_upload = true; } diff --git a/regression-test/data/cloud_p0/test_cloud_memtable_group_commit.out b/regression-test/data/cloud_p0/test_cloud_memtable_group_commit.out new file mode 100644 index 00000000000000..14525f0ad631ca --- /dev/null +++ b/regression-test/data/cloud_p0/test_cloud_memtable_group_commit.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !response -- +Success true async_mode true 2 2 0 + +-- !pending_rows -- +1 10 +2 20 + +-- !rows -- +1 100 +2 20 +3 30 + diff --git a/regression-test/data/cloud_p0/test_cloud_memtable_on_sink.out b/regression-test/data/cloud_p0/test_cloud_memtable_on_sink.out index 6948612c886f8f..ce3cc76caba9fc 100644 --- a/regression-test/data/cloud_p0/test_cloud_memtable_on_sink.out +++ b/regression-test/data/cloud_p0/test_cloud_memtable_on_sink.out @@ -11,9 +11,24 @@ -- !s3_packed_meta_false -- 6 20 2 1 1 0 0,1,2,3,4,5 +-- !s3_rows_true -- +20 1133 2266 + +-- !s3_index_true -- +31 62 +50 100 +50 100 +57 114 + +-- !s3_packed_meta_true -- +6 20 2 1 1 4 0,1,1000,1001,2000,2001 + -- !stream_rows -- 22 1334 2668 +-- !failed_upload_rows -- +22 1334 2668 + -- !agg_false -- 0 17994000 0 11996 3000 1 10 20 1 17997000 1 11997 3000 1 11 21 @@ -120,6 +135,121 @@ 90 180 91 182 +-- !layout_agg -- +true true true true true + +-- !layout_mor -- +true true true true true + +-- !layout_mor_seq -- +true true true true true + +-- !agg_true -- +0 17994000 0 11996 3000 1 10 20 +1 17997000 1 11997 3000 1 11 21 +2 18000000 2 11998 3000 1 12 22 +3 18003000 3 11999 3000 1 13 23 + +-- !mor_true -- +0 0 +1 10 +2 20 +3 30 + +-- !seq_true -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !seq_index_true -- +1 119970 11997 +3 119990 11999 + +-- !seq_lower_true -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !mor_new_version_true -- +0 99 +1 10 +2 20 +3 30 + +-- !agg_new_version_true -- +0 17994010 -1 13000 3000 1 99 20 +1 17997000 1 11997 3000 1 11 21 +2 18000000 2 11998 3000 1 12 22 +3 18003000 3 11999 3000 1 13 23 + +-- !seq_empty_true -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !agg_compacted_true -- +0 17994010 -1 13000 3000 1 99 20 +1 17997000 1 11997 3000 1 11 21 +2 18000000 2 11998 3000 1 12 22 +3 18003000 3 11999 3000 1 13 23 + +-- !mor_compacted_true -- +0 99 +1 10 +2 20 +3 30 + +-- !seq_compacted_true -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !agg_broker_true -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 200 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + +-- !mor_broker_true -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 100 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + -- !agg_stream -- 50 203 diff --git a/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_cache.out b/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_cache.out new file mode 100644 index 00000000000000..3fd2f58bdbb155 --- /dev/null +++ b/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_cache.out @@ -0,0 +1,41 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !load_plain_remote -- +Success 2 0 + +-- !layout_plain_remote -- +1 false + +-- !cache_plain_remote -- +true true + +-- !load_plain_local -- +Success 2 0 + +-- !layout_plain_local -- +1 false + +-- !cache_plain_local -- +true true + +-- !load_packed_remote -- +Success 2 0 + +-- !layout_packed_remote -- +1 true + +-- !cache_packed_remote -- +true true + +-- !load_packed_local -- +Success 2 0 + +-- !layout_packed_local -- +1 true + +-- !cache_packed_local -- +true true + +-- !rows -- +1 40 4 +2 80 4 + diff --git a/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_mow.out b/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_mow.out new file mode 100644 index 00000000000000..f52daf0cf0ab28 --- /dev/null +++ b/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_mow.out @@ -0,0 +1,147 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !initial -- +0 0 +1 10 +2 20 +3 30 + +-- !initial_seq -- +0 119960 11997 +1 119970 11998 +2 119980 11999 +3 119990 12000 + +-- !index -- +1 119970 11998 +3 119990 12000 + +-- !layout_cloud_mow_direct -- +true true true true + +-- !layout_cloud_mow_direct_seq -- +true true true true + +-- !lower_seq -- +0 119960 11997 +1 119970 11998 +2 119980 11999 +3 119990 12000 + +-- !new_version -- +0 99 +1 10 +2 20 +3 30 + +-- !empty -- +0 119960 11997 +1 119970 11998 +2 119980 11999 +3 119990 12000 + +-- !delete -- +0 119960 11997 +1 119970 11998 +3 119990 12000 + +-- !lower_after_delete -- +0 119960 11997 +1 119970 11998 +3 119990 12000 + +-- !reinsert -- +0 119960 11997 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !compacted -- +0 99 +1 10 +2 20 +3 30 + +-- !compacted_seq -- +0 119960 11997 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !stream -- +0 150000 15000 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !concurrent_compaction -- +0 210000 21000 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !failed_invisible -- +0 210000 21000 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !cluster -- +0 -11996 11997 +1 -11997 11998 +2 -11998 11999 +3 -11999 12000 + +-- !cluster_update -- +0 999 20000 +1 -11997 11998 +2 -11998 11999 +3 -11999 12000 + +-- !cluster_compacted -- +0 999 20000 +1 -11997 11998 +2 -11998 11999 +3 -11999 12000 + +-- !broker_true -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 100 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + +-- !broker_false -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 100 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + diff --git a/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_unshared.out b/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_unshared.out new file mode 100644 index 00000000000000..7829dc48776f74 --- /dev/null +++ b/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_unshared.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !rows -- +100000 4999950000 9999900000 + +-- !rowset_layout -- +true true true 100000 + +-- !empty_input -- +100000 4999950000 9999900000 + +-- !second_load -- +100010 4999950045 9999900090 + diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_group_commit.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_group_commit.groovy new file mode 100644 index 00000000000000..c45db6eed22a66 --- /dev/null +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_group_commit.groovy @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.suite.client.BackendClientImpl +import org.apache.doris.thrift.TGetRealtimeExecStatusRequest +import org.apache.doris.thrift.TNetworkAddress +import org.apache.doris.thrift.TStatusCode +import org.apache.doris.thrift.TUniqueId + +suite("test_cloud_memtable_group_commit", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(3) + options.enableDebugPoints() + options.feConfigs += ['stream_load_default_memtable_on_sink_node=true', + 'cloud_stream_load_default_memtable_sink_upload=true'] + docker(options) { + sql "DROP TABLE IF EXISTS cloud_memtable_group_commit" + sql """ + CREATE TABLE cloud_memtable_group_commit (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "group_commit_interval_ms"="200", "disable_auto_compaction"="true") + """ + sql "INSERT INTO cloud_memtable_group_commit VALUES (1,10),(2,20)" + // Internal Group Commit plans use a new session, not this SQL connection's variables. + sql "SET GLOBAL enable_profile=true" + sql "SET GLOBAL profile_level=2" + def tablet = sql_return_maparray("SHOW TABLETS FROM cloud_memtable_group_commit")[0] + def backends = sql_return_maparray("SHOW BACKENDS") + def block = "VTabletWriterV2.close.profile_ready" + try { + GetDebugPoint().enableDebugPointForAllBEs(block, [timeout: "120"]) + def label = "" + streamLoad { + table "cloud_memtable_group_commit" + set "column_separator", "," + set "group_commit", "async_mode" + // The internal load must use the FE defaults rather than this outer request's flag. + set "memtable_on_sink_node", "false" + set "cloud_memtable_sink_upload", "false" + unset "label" + inputText "1,100\n3,30\n" + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + logger.info("Cloud memtable Group Commit response: {}", result) + def json = parseJson(result) + label = json.Label + order_qt_response """ + SELECT '${json.Status}', ${json.GroupCommit == true}, '${json.GroupCommitMode}', + ${label.startsWith('group_commit_')}, ${json.NumberTotalRows}, + ${json.NumberLoadedRows}, ${json.NumberFilteredRows} + """ + } + } + def idParts = label.substring('group_commit_'.length()).split('_') + def queryId = new TUniqueId(Long.parseUnsignedLong(idParts[0], 16), + Long.parseUnsignedLong(idParts[1], 16)) + def request = new TGetRealtimeExecStatusRequest().setId(queryId).setReqType("profile") + // Probe the BEs for this internal query; its writer remains alive at the debug point. + awaitUntil(60) { + backends.any { be -> + def backend = new BackendClientImpl( + new TNetworkAddress(be.Host, be.BePort as int), be.HttpPort as int) + try { + backend.client.inputProtocol.transport.setTimeout(5000) + def response = backend.client.getRealtimeExecStatus(request) + if (response.status.statusCode == TStatusCode.NOT_FOUND) { + return false + } + if (response.status.statusCode != TStatusCode.OK) { + throw new IllegalStateException("Get BE profile failed: ${response.status}") + } + def profile = response.reportExecStatusParams.queryProfile + def nodes = profile.fragmentIdToProfile.values().flatten() + .collectMany { it.profile.nodes } + return nodes.any { it.name == "DeltaWriterV2 ${tablet.TabletId}" } && + nodes.any { + it.infoStrings['CloudMemtableSinkUpload'] == 'true' && + it.infoStrings['CloudMemtableMowBitmap'] == 'true' + } + } finally { + backend.close() + } + } + } + order_qt_pending_rows "SELECT * FROM cloud_memtable_group_commit" + } finally { + GetDebugPoint().disableDebugPointForAllBEs(block) + } + // Publication can only finish after releasing the writer. + awaitUntil(60) { + (sql "SELECT COUNT(*) FROM cloud_memtable_group_commit")[0][0] == 3 + } + order_qt_rows "SELECT * FROM cloud_memtable_group_commit" + } +} diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_mow_forward.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_mow_forward.groovy index 1b84ca7214a838..1fc69b1b97754d 100644 --- a/regression-test/suites/cloud_p0/test_cloud_memtable_mow_forward.groovy +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_mow_forward.groovy @@ -31,7 +31,8 @@ suite("test_cloud_memtable_mow_forward", "p0, docker") { options.enableDebugPoints() options.beConfigs += ['small_file_threshold_bytes=1048576', 'enable_merge_on_write_correctness_check=true'] - options.feConfigs += ['stream_load_default_memtable_on_sink_node=true'] + options.feConfigs += ['stream_load_default_memtable_on_sink_node=true', + 'cloud_stream_load_default_memtable_sink_upload=false'] docker(options) { def backends = sql_return_maparray("SHOW BACKENDS") def awaitGroupProfile = { label, tabletId -> @@ -77,179 +78,190 @@ suite("test_cloud_memtable_mow_forward", "p0, docker") { """ sql "SET enable_memtable_on_sink_node=false" sql "INSERT INTO cloud_mow_forward_source SELECT number FROM numbers('number'='12000')" + sql "SET enable_cloud_memtable_sink_upload=false" sql "SET parallel_pipeline_task_num=4" sql "SET profile_level=2" sql "SET enable_sql_cache=false" sql "SET enable_file_cache=false" - [false, true].each { packed -> - setBeConfigTemporary(['enable_packed_file': packed.toString()]) { - sql "DROP TABLE IF EXISTS cloud_mow_forward" - sql """ - CREATE TABLE cloud_mow_forward ( - k BIGINT NOT NULL, v BIGINT, seq BIGINT NOT NULL, - INDEX idx_k(k) USING INVERTED - ) UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", - "function_column.sequence_col"="seq", "inverted_index_storage_format"="V2", - "disable_auto_compaction"="true", "group_commit_interval_ms"="200") - """ - sql "SET enable_memtable_on_sink_node=false" - sql "INSERT INTO cloud_mow_forward VALUES (0,-1,-1),(1,-1,-1),(2,-1,-1),(3,-1,-1)" - sql "INSERT INTO cloud_mow_forward VALUES (0,0,0),(1,0,0),(2,0,0),(3,0,0)" - sql "SET enable_memtable_on_sink_node=true" - sql "SET enable_profile=true" - sql """ - /* cloud_mow_forward_${packed} */ - INSERT INTO cloud_mow_forward SELECT n%4,n*10,n+1 FROM cloud_mow_forward_source - """ - new ProfileAction(context).getProfileBySql("cloud_mow_forward_${packed}", ["DeltaWriterV2"]) - sql "SET enable_profile=false" - quickTest("initial_${packed}", "SELECT * FROM cloud_mow_forward", true) - checkUniqueKeys() - quickTest("index_${packed}", "SELECT * FROM cloud_mow_forward WHERE k IN (1,3)", true) - def tablet = sql_return_maparray("SHOW TABLETS FROM cloud_mow_forward")[0] - def partition = sql_return_maparray("SHOW PARTITIONS FROM cloud_mow_forward")[0] - def ms = cluster.getAllMetaservices()[0] - getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { - code, body -> - assertEquals(200, code) - def meta = parseJson(body) - quickTest("layout_${packed}", """ - SELECT ${meta.num_segments as int} > 1, - ${(meta.packed_slice_locations ?: [:]).size() > 0}, - ${meta.segments_file_size.size() == (meta.num_segments as int)}, - ${meta.segments_file_size.every { (it as long) > 0 }}, - ${meta.inverted_index_file_info.size() == (meta.num_segments as int)}, - ${meta.inverted_index_file_info.every { (it.index_size as long) > 0 }} - """, true) - } - sql "INSERT INTO cloud_mow_forward VALUES (0,-1,1)" - sql "INSERT INTO cloud_mow_forward (k,v,seq,__DORIS_DELETE_SIGN__) VALUES (2,130000,13000,1)" - sql "INSERT INTO cloud_mow_forward VALUES (2,-1,1)" - quickTest("deleted_${packed}", "SELECT * FROM cloud_mow_forward", true) - sql "INSERT INTO cloud_mow_forward VALUES (2,140000,14000)" - streamLoad { - table "cloud_mow_forward" - set "column_separator", "," - set "memtable_on_sink_node", "true" - set "group_commit", "off_mode" - inputText "0,150000,15000\n3,-1,1\n" - check { result, exception, startTime, endTime -> - if (exception != null) { throw exception } - def response = parseJson(result) - quickTest("stream_status_${packed}", "SELECT '${response.Status}'", true) + // Any accidental switch to sink upload must fail before its result is committed. + GetDebugPoint().enableDebugPointForAllBEs("DeltaWriterV2.sink_upload.after_upload_failure") + try { + [false, true].each { packed -> + setBeConfigTemporary(['enable_packed_file': packed.toString()]) { + sql "DROP TABLE IF EXISTS cloud_mow_forward" + sql """ + CREATE TABLE cloud_mow_forward ( + k BIGINT NOT NULL, v BIGINT, seq BIGINT NOT NULL, + INDEX idx_k(k) USING INVERTED + ) UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "function_column.sequence_col"="seq", "inverted_index_storage_format"="V2", + "disable_auto_compaction"="true", "group_commit_interval_ms"="200") + """ + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_mow_forward VALUES (0,-1,-1),(1,-1,-1),(2,-1,-1),(3,-1,-1)" + sql "INSERT INTO cloud_mow_forward VALUES (0,0,0),(1,0,0),(2,0,0),(3,0,0)" + sql "SET enable_memtable_on_sink_node=true" + sql "SET enable_profile=true" + sql """ + /* cloud_mow_forward_${packed} */ + INSERT INTO cloud_mow_forward SELECT n%4,n*10,n+1 FROM cloud_mow_forward_source + """ + new ProfileAction(context).getProfileBySql("cloud_mow_forward_${packed}", ["DeltaWriterV2"]) + sql "SET enable_profile=false" + quickTest("initial_${packed}", "SELECT * FROM cloud_mow_forward", true) + checkUniqueKeys() + quickTest("index_${packed}", "SELECT * FROM cloud_mow_forward WHERE k IN (1,3)", true) + def tablet = sql_return_maparray("SHOW TABLETS FROM cloud_mow_forward")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM cloud_mow_forward")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + code, body -> + assertEquals(200, code) + def meta = parseJson(body) + quickTest("layout_${packed}", """ + SELECT ${meta.num_segments as int} > 1, + ${(meta.packed_slice_locations ?: [:]).size() > 0}, + ${meta.segments_file_size.size() == (meta.num_segments as int)}, + ${meta.segments_file_size.every { (it as long) > 0 }}, + ${meta.inverted_index_file_info.size() == (meta.num_segments as int)}, + ${meta.inverted_index_file_info.every { (it.index_size as long) > 0 }} + """, true) } - } - quickTest("stream_${packed}", "SELECT * FROM cloud_mow_forward", true) - def originalEnableProfile = sql("SHOW GLOBAL VARIABLES LIKE 'enable_profile'")[0][1] - def originalProfileLevel = sql("SHOW GLOBAL VARIABLES LIKE 'profile_level'")[0][1] - def profileBlock = "VTabletWriterV2.close.profile_ready" - def groupLabel = "" - try { - sql "SET GLOBAL enable_profile=true" - sql "SET GLOBAL profile_level=2" - GetDebugPoint().enableDebugPointForAllBEs(profileBlock, [timeout: "120"]) + sql "INSERT INTO cloud_mow_forward VALUES (0,-1,1)" + sql "INSERT INTO cloud_mow_forward (k,v,seq,__DORIS_DELETE_SIGN__) VALUES (2,130000,13000,1)" + sql "INSERT INTO cloud_mow_forward VALUES (2,-1,1)" + quickTest("deleted_${packed}", "SELECT * FROM cloud_mow_forward", true) + sql "INSERT INTO cloud_mow_forward VALUES (2,140000,14000)" streamLoad { table "cloud_mow_forward" set "column_separator", "," - set "group_commit", "async_mode" - set "memtable_on_sink_node", "false" - unset "label" - inputText "1,160000,16000\n" + set "memtable_on_sink_node", "true" + set "group_commit", "off_mode" + inputText "0,150000,15000\n3,-1,1\n" check { result, exception, startTime, endTime -> if (exception != null) { throw exception } def response = parseJson(result) - groupLabel = response.Label - quickTest("group_status_${packed}", "SELECT '${response.Status}', '${response.GroupCommit}'", true) + sql "SELECT assert_true('${response.Status}' = 'Success', 'forwarded Stream Load failed')" + quickTest("stream_status_${packed}", "SELECT '${response.Status}'", true) } } - awaitGroupProfile(groupLabel, tablet.TabletId) - } finally { - GetDebugPoint().disableDebugPointForAllBEs(profileBlock) - sql "SET GLOBAL enable_profile=${originalEnableProfile}" - sql "SET GLOBAL profile_level=${originalProfileLevel}" - } - awaitUntil(60) { - (sql "SELECT v FROM cloud_mow_forward WHERE k=1")[0][0] == 160000 + quickTest("stream_${packed}", "SELECT * FROM cloud_mow_forward", true) + def originalEnableProfile = sql("SHOW GLOBAL VARIABLES LIKE 'enable_profile'")[0][1] + def originalProfileLevel = sql("SHOW GLOBAL VARIABLES LIKE 'profile_level'")[0][1] + def profileBlock = "VTabletWriterV2.close.profile_ready" + def groupLabel = "" + try { + sql "SET GLOBAL enable_profile=true" + sql "SET GLOBAL profile_level=2" + GetDebugPoint().enableDebugPointForAllBEs(profileBlock, [timeout: "120"]) + streamLoad { + table "cloud_mow_forward" + set "column_separator", "," + set "group_commit", "async_mode" + set "memtable_on_sink_node", "false" + set "cloud_memtable_sink_upload", "true" + unset "label" + inputText "1,160000,16000\n" + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + sql "SELECT assert_true('${response.Status}' = 'Success', 'Group Commit Stream Load failed')" + groupLabel = response.Label + quickTest("group_status_${packed}", "SELECT '${response.Status}', '${response.GroupCommit}'", true) + } + } + awaitGroupProfile(groupLabel, tablet.TabletId) + } finally { + GetDebugPoint().disableDebugPointForAllBEs(profileBlock) + sql "SET GLOBAL enable_profile=${originalEnableProfile}" + sql "SET GLOBAL profile_level=${originalProfileLevel}" + } + awaitUntil(60) { + (sql "SELECT v FROM cloud_mow_forward WHERE k=1")[0][0] == 160000 + } + quickTest("group_${packed}", "SELECT * FROM cloud_mow_forward", true) + checkUniqueKeys() + trigger_and_wait_compaction("cloud_mow_forward", "full") + quickTest("compacted_${packed}", "SELECT * FROM cloud_mow_forward", true) + checkUniqueKeys() } - quickTest("group_${packed}", "SELECT * FROM cloud_mow_forward", true) - checkUniqueKeys() - trigger_and_wait_compaction("cloud_mow_forward", "full") - quickTest("compacted_${packed}", "SELECT * FROM cloud_mow_forward", true) - checkUniqueKeys() } - } - // MOW without sink upload forwards files and calculates bitmaps on the target BE. - sql "DROP TABLE IF EXISTS cloud_memtable_mow_fallback" - sql """ - CREATE TABLE cloud_memtable_mow_fallback (k BIGINT NOT NULL, v BIGINT) - UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true") - """ - sql "INSERT INTO cloud_memtable_mow_fallback VALUES (1,10)" - sql "INSERT INTO cloud_memtable_mow_fallback VALUES (1,20)" - order_qt_mow_fallback "SELECT * FROM cloud_memtable_mow_fallback" + // MOW without sink upload forwards files and calculates bitmaps on the target BE. + sql "DROP TABLE IF EXISTS cloud_memtable_mow_fallback" + sql """ + CREATE TABLE cloud_memtable_mow_fallback (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true") + """ + sql "INSERT INTO cloud_memtable_mow_fallback VALUES (1,10)" + sql "INSERT INTO cloud_memtable_mow_fallback VALUES (1,20)" + order_qt_mow_fallback "SELECT * FROM cloud_memtable_mow_fallback" - sql "DROP TABLE IF EXISTS cloud_mow_forward_broker" - sql """ - CREATE TABLE cloud_mow_forward_broker (k BIGINT NOT NULL, v BIGINT) - UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", - "disable_auto_compaction"="true") - """ - sql "INSERT INTO cloud_mow_forward_broker VALUES (50,-1)" - def brokerLabel = "mow_forward_" + UUID.randomUUID().toString().replace('-', '_') - sql "SET enable_profile=true" - try { + sql "DROP TABLE IF EXISTS cloud_mow_forward_broker" sql """ - LOAD LABEL ${brokerLabel} ( - DATA INFILE("s3://${getS3BucketName()}/regression/load/data/basic_data.csv") - INTO TABLE cloud_mow_forward_broker COLUMNS TERMINATED BY "|" FORMAT AS "CSV" - (k, c01, c02, c03, c04, c05, c06, c07, c08, c09, - c10, c11, c12, c13, c14, c15, c16, c17, c18) SET (v=k*2) - ) WITH S3 ( - "AWS_ACCESS_KEY"="${getS3AK()}", "AWS_SECRET_KEY"="${getS3SK()}", - "AWS_ENDPOINT"="${getS3Endpoint()}", "AWS_REGION"="${getS3Region()}", - "provider"="${getS3Provider()}" - ) PROPERTIES ("load_parallelism"="1") + CREATE TABLE cloud_mow_forward_broker (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "disable_auto_compaction"="true") """ - waitForBrokerLoadDone(brokerLabel) - def load = sql_return_maparray("SHOW LOAD WHERE LABEL = '${brokerLabel}'")[0] - new ProfileAction(context).getProfile(load.JobId.toString(), ["DeltaWriterV2"]) - } finally { - sql "SET enable_profile=false" - } - sql """ - SELECT assert_true(COUNT(*) = 0, 'cloud_mow_forward_broker contains duplicate keys') FROM ( - SELECT k, COUNT(*) AS a FROM cloud_mow_forward_broker GROUP BY k HAVING a > 1 - ) duplicates - """ - order_qt_broker_rows "SELECT k,v FROM cloud_mow_forward_broker" + sql "INSERT INTO cloud_mow_forward_broker VALUES (50,-1)" + def brokerLabel = "mow_forward_" + UUID.randomUUID().toString().replace('-', '_') + sql "SET enable_profile=true" + try { + sql """ + LOAD LABEL ${brokerLabel} ( + DATA INFILE("s3://${getS3BucketName()}/regression/load/data/basic_data.csv") + INTO TABLE cloud_mow_forward_broker COLUMNS TERMINATED BY "|" FORMAT AS "CSV" + (k, c01, c02, c03, c04, c05, c06, c07, c08, c09, + c10, c11, c12, c13, c14, c15, c16, c17, c18) SET (v=k*2) + ) WITH S3 ( + "AWS_ACCESS_KEY"="${getS3AK()}", "AWS_SECRET_KEY"="${getS3SK()}", + "AWS_ENDPOINT"="${getS3Endpoint()}", "AWS_REGION"="${getS3Region()}", + "provider"="${getS3Provider()}" + ) PROPERTIES ("load_parallelism"="1") + """ + waitForBrokerLoadDone(brokerLabel) + def load = sql_return_maparray("SHOW LOAD WHERE LABEL = '${brokerLabel}'")[0] + new ProfileAction(context).getProfile(load.JobId.toString(), ["DeltaWriterV2"]) + } finally { + sql "SET enable_profile=false" + } + sql """ + SELECT assert_true(COUNT(*) = 0, 'cloud_mow_forward_broker contains duplicate keys') FROM ( + SELECT k, COUNT(*) AS a FROM cloud_mow_forward_broker GROUP BY k HAVING a > 1 + ) duplicates + """ + order_qt_broker_rows "SELECT k,v FROM cloud_mow_forward_broker" - sql "DROP TABLE IF EXISTS cloud_memtable_partial_update" - sql """ - CREATE TABLE cloud_memtable_partial_update (k INT NOT NULL, v INT, untouched INT) - UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true") - """ - sql "INSERT INTO cloud_memtable_partial_update VALUES (1,10,100)" - streamLoad { - table "cloud_memtable_partial_update" - set "column_separator", "," - set "columns", "k,v" - set "partial_columns", "true" - set "memtable_on_sink_node", "true" - set "group_commit", "off_mode" - inputText "1,20\n" - check { result, exception, startTime, endTime -> - if (exception != null) { throw exception } - def response = parseJson(result) - sql "SELECT assert_true('${response.Status}' = 'Success', 'partial-update Stream Load failed')" + sql "DROP TABLE IF EXISTS cloud_memtable_partial_update" + sql """ + CREATE TABLE cloud_memtable_partial_update (k INT NOT NULL, v INT, untouched INT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true") + """ + sql "INSERT INTO cloud_memtable_partial_update VALUES (1,10,100)" + streamLoad { + table "cloud_memtable_partial_update" + set "column_separator", "," + set "columns", "k,v" + set "partial_columns", "true" + set "memtable_on_sink_node", "true" + set "cloud_memtable_sink_upload", "false" + set "group_commit", "off_mode" + inputText "1,20\n" + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + sql "SELECT assert_true('${response.Status}' = 'Success', 'partial-update Stream Load failed')" + } } + sql """ + SELECT assert_true(COUNT(*) = 1 AND SUM(v) = 20 AND SUM(untouched) = 100, 'partial update result mismatch') + FROM cloud_memtable_partial_update WHERE k = 1 + """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("DeltaWriterV2.sink_upload.after_upload_failure") } - sql """ - SELECT assert_true(COUNT(*) = 1 AND SUM(v) = 20 AND SUM(untouched) = 100, 'partial update result mismatch') - FROM cloud_memtable_partial_update WHERE k = 1 - """ } } diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_on_sink.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_on_sink.groovy index f34dcc078b3745..aee48339813798 100644 --- a/regression-test/suites/cloud_p0/test_cloud_memtable_on_sink.groovy +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_on_sink.groovy @@ -30,6 +30,8 @@ suite("test_cloud_memtable_on_sink", "p0, docker") { options.beConfigs += ['enable_packed_file=true', 'small_file_threshold_bytes=1048576', 'enable_adaptive_batch_size=false'] + // Ordinary Stream Load inherits the FE default unless its header overrides it. + options.feConfigs += ['cloud_stream_load_default_memtable_sink_upload = true'] docker(options) { def loadS3 = { table, label -> sql """ @@ -50,152 +52,165 @@ suite("test_cloud_memtable_on_sink", "p0, docker") { } def testDuplicate = { - sql "SET enable_sql_cache = false" - sql "DROP TABLE IF EXISTS test_cloud_duplicate_memtable_on_sink_source" - sql "DROP TABLE IF EXISTS test_cloud_duplicate_memtable_on_sink" - - sql """ - CREATE TABLE test_cloud_duplicate_memtable_on_sink_source ( - k BIGINT NOT NULL, - v BIGINT NOT NULL - ) - DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 3 - PROPERTIES ("replication_num" = "1") - """ - sql """ - CREATE TABLE test_cloud_duplicate_memtable_on_sink ( - k BIGINT NOT NULL, - v BIGINT NOT NULL - ) - DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES ("replication_num" = "1") - """ + [false, true].each { sinkUpload -> + sql "SET enable_cloud_memtable_sink_upload = ${sinkUpload}" + if (sinkUpload) { + GetDebugPoint().enableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + GetDebugPoint().enableDebugPointForAllBEs("DeltaWriterV2.sink_upload.duplicate_result") + } - sql "SET enable_memtable_on_sink_node = false" - sql """ - INSERT INTO test_cloud_duplicate_memtable_on_sink_source - SELECT number, number * 2 FROM numbers("number" = "100000") - """ + sql "SET enable_sql_cache = false" + sql "DROP TABLE IF EXISTS test_cloud_duplicate_memtable_on_sink_source" + sql "DROP TABLE IF EXISTS test_cloud_duplicate_memtable_on_sink" - try { - sql "SET enable_memtable_on_sink_node = true" - sql "SET profile_level = 2" - sql "SET enable_profile = true" sql """ - /* cloud_duplicate_memtable_on_sink_profile_false */ - INSERT INTO test_cloud_duplicate_memtable_on_sink - SELECT k, v FROM test_cloud_duplicate_memtable_on_sink_source + CREATE TABLE test_cloud_duplicate_memtable_on_sink_source ( + k BIGINT NOT NULL, + v BIGINT NOT NULL + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 3 + PROPERTIES ("replication_num" = "1") """ - def required = ["DeltaWriterV2"] - def profileString = new ProfileAction(context).getProfileBySql( - "cloud_duplicate_memtable_on_sink_profile_false", required) - logger.info("memtable-on-sink profile:\n{}", profileString) - } finally { - sql "SET enable_profile = false" + sql """ + CREATE TABLE test_cloud_duplicate_memtable_on_sink ( + k BIGINT NOT NULL, + v BIGINT NOT NULL + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + sql "SET enable_memtable_on_sink_node = false" - } + sql """ + INSERT INTO test_cloud_duplicate_memtable_on_sink_source + SELECT number, number * 2 FROM numbers("number" = "100000") + """ - sql """ - SELECT assert_true( - COUNT(*) = 100000 - AND SUM(k) = 4999950000 - AND SUM(v) = 9999900000, - 'cloud duplicate memtable-on-sink result mismatch') - FROM test_cloud_duplicate_memtable_on_sink - """ + try { + sql "SET enable_memtable_on_sink_node = true" + sql "SET profile_level = 2" + sql "SET enable_profile = true" + sql """ + /* cloud_duplicate_memtable_on_sink_profile_${sinkUpload} */ + INSERT INTO test_cloud_duplicate_memtable_on_sink + SELECT k, v FROM test_cloud_duplicate_memtable_on_sink_source + """ + def required = ["DeltaWriterV2"] + if (sinkUpload) { + required += "CloudMemtableSinkUpload: true" + } + def profileString = new ProfileAction(context).getProfileBySql( + "cloud_duplicate_memtable_on_sink_profile_${sinkUpload}", required) + logger.info("memtable-on-sink profile:\n{}", profileString) + } finally { + sql "SET enable_profile = false" + sql "SET enable_memtable_on_sink_node = false" + } - sql "DROP TABLE IF EXISTS test_cloud_duplicate_memtable_on_sink_s3" - sql """ - CREATE TABLE test_cloud_duplicate_memtable_on_sink_s3 ( - k BIGINT NOT NULL, - v BIGINT NOT NULL, - INDEX idx_v (v) USING INVERTED - ) - DUPLICATE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES ( - "replication_num" = "1", - "inverted_index_storage_format" = "V2", - "disable_auto_compaction" = "true" - ) - """ + sql """ + SELECT assert_true( + COUNT(*) = 100000 + AND SUM(k) = 4999950000 + AND SUM(v) = 9999900000, + 'cloud duplicate memtable-on-sink result mismatch') + FROM test_cloud_duplicate_memtable_on_sink + """ - def label = "cloud_duplicate_memtable_on_sink_s3_" + UUID.randomUUID().toString().replace('-', '_') - try { - sql "SET enable_memtable_on_sink_node = true" - sql "SET enable_profile = true" - // The three scanners read 6, 6, and 8 rows. Flush after the first four rows on - // each BE, then flush the remaining rows at close to produce two segments each. - sql "SET broker_load_batch_size = 4" - GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush", [execute: 1]) - def load = loadS3("test_cloud_duplicate_memtable_on_sink_s3", label) - assertEquals("FINISHED", load.State, "S3 load did not finish: ${load}") + sql "DROP TABLE IF EXISTS test_cloud_duplicate_memtable_on_sink_s3" + sql """ + CREATE TABLE test_cloud_duplicate_memtable_on_sink_s3 ( + k BIGINT NOT NULL, + v BIGINT NOT NULL, + INDEX idx_v (v) USING INVERTED + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2", + "disable_auto_compaction" = "true" + ) + """ + + def label = "cloud_duplicate_memtable_on_sink_s3_" + UUID.randomUUID().toString().replace('-', '_') + try { + sql "SET enable_memtable_on_sink_node = true" + sql "SET enable_profile = true" + // The three scanners read 6, 6, and 8 rows. Flush after the first four rows on + // each BE, then flush the remaining rows at close to produce two segments each. + sql "SET broker_load_batch_size = 4" + GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush", [execute: 1]) + def load = loadS3("test_cloud_duplicate_memtable_on_sink_s3", label) + assertEquals("FINISHED", load.State, "S3 load did not finish: ${load}") - def profileString = new ProfileAction(context).getProfile( - load.JobId.toString(), ["DeltaWriterV2", "NumScanners"]) - logger.info("S3 memtable-on-sink profile:\n{}", profileString) + def profileString = new ProfileAction(context).getProfile( + load.JobId.toString(), ["DeltaWriterV2", "NumScanners"]) + logger.info("S3 memtable-on-sink profile:\n{}", profileString) - // Only inspect per-BE pipelines, excluding the merged profile's duplicate counters. - def pipelines = profileString.split(/(?m)(?=^[ \t]*(?:Pipeline \d+|FragmentLevelProfile:)\(host=)/) - .findAll { it.trim().startsWith("Pipeline ") } - def backends = sql_return_maparray("SHOW BACKENDS") - backends.each { backend -> - def pipeline = pipelines.find { - it.readLines()[0].contains("hostname:${backend.Host},") && it.contains("DeltaWriterV2") + // Only inspect per-BE pipelines, excluding the merged profile's duplicate counters. + def pipelines = profileString.split(/(?m)(?=^[ \t]*(?:Pipeline \d+|FragmentLevelProfile:)\(host=)/) + .findAll { it.trim().startsWith("Pipeline ") } + def backends = sql_return_maparray("SHOW BACKENDS") + backends.each { backend -> + def pipeline = pipelines.find { + it.readLines()[0].contains("hostname:${backend.Host},") && it.contains("DeltaWriterV2") + } + assertNotNull(pipeline, "Missing S3 sink writer on BE ${backend.Host}") + assertTrue(pipeline.contains("FILE_SCAN_OPERATOR"), "Missing S3 scanner on BE ${backend.Host}") + assertTrue((pipeline =~ /(?m)^\s*- NumScanners: 1\s*$/).find(), + "Expected one S3 scanner on BE ${backend.Host}") + assertTrue((pipeline =~ /(?m)^\s*- SegmentNum: 2\s*$/).find(), + "Expected two flushed segments on BE ${backend.Host}") } - assertNotNull(pipeline, "Missing S3 sink writer on BE ${backend.Host}") - assertTrue(pipeline.contains("FILE_SCAN_OPERATOR"), "Missing S3 scanner on BE ${backend.Host}") - assertTrue((pipeline =~ /(?m)^\s*- NumScanners: 1\s*$/).find(), - "Expected one S3 scanner on BE ${backend.Host}") - assertTrue((pipeline =~ /(?m)^\s*- SegmentNum: 2\s*$/).find(), - "Expected two flushed segments on BE ${backend.Host}") + } finally { + GetDebugPoint().disableDebugPointForAllBEs("MemTable.need_flush") + sql "SET enable_profile = false" + sql "SET enable_memtable_on_sink_node = false" } - } finally { - GetDebugPoint().disableDebugPointForAllBEs("MemTable.need_flush") - sql "SET enable_profile = false" - sql "SET enable_memtable_on_sink_node = false" - } - // Read both segment data and V2 indexes without the uploader's file cache. - sql "SET enable_file_cache = false" - quickTest("s3_rows_false", """ - SELECT COUNT(*), SUM(k), SUM(v) - FROM test_cloud_duplicate_memtable_on_sink_s3 - """, true) - quickTest("s3_index_false", """ - SELECT k, v FROM test_cloud_duplicate_memtable_on_sink_s3 WHERE v IN (62, 100, 114) - """, true) - def tablet = sql_return_maparray("SHOW TABLETS FROM test_cloud_duplicate_memtable_on_sink_s3")[0] - def partition = sql_return_maparray("SHOW PARTITIONS FROM test_cloud_duplicate_memtable_on_sink_s3")[0] - def ms = cluster.getAllMetaservices()[0] - getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { - responseCode, body -> - assertEquals(200, responseCode) - logger.info("S3 memtable-on-sink rowset meta: {}", body) - def rowsetMeta = parseJson(body) - def locations = rowsetMeta.packed_slice_locations - def segmentIds = rowsetMeta.segment_ids ?: (0..<(rowsetMeta.num_segments as int)).toList() - logger.info("S3 forwarded rowset layout: {}", [ - tablet_id: tablet.TabletId, version: partition.VisibleVersion, - rowset_id: rowsetMeta.rowset_id_v2, segment_ids: segmentIds, - num_segment_rows: rowsetMeta.num_segment_rows, - segments_file_size: rowsetMeta.segments_file_size, - packed_files: locations.keySet().sort() - ]) - // Only the destination rowset's first segment and its V2 index are packed. - // The other five segments retain independent files. - quickTest("s3_packed_meta_false", """ - SELECT ${rowsetMeta.num_segments as int}, ${rowsetMeta.num_rows as long}, - ${locations.size()}, - ${locations.keySet().count { it.endsWith('_0.dat') }}, - ${locations.keySet().count { it.endsWith('_0.idx') }}, - ${segmentIds.count { (it as long) >= 1000 }}, '${segmentIds.join(",")}' - """, true) + // Read both segment data and V2 indexes without the uploader's file cache. + sql "SET enable_file_cache = false" + quickTest("s3_rows_${sinkUpload}", """ + SELECT COUNT(*), SUM(k), SUM(v) + FROM test_cloud_duplicate_memtable_on_sink_s3 + """, true) + quickTest("s3_index_${sinkUpload}", """ + SELECT k, v FROM test_cloud_duplicate_memtable_on_sink_s3 WHERE v IN (62, 100, 114) + """, true) + def tablet = sql_return_maparray("SHOW TABLETS FROM test_cloud_duplicate_memtable_on_sink_s3")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM test_cloud_duplicate_memtable_on_sink_s3")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + responseCode, body -> + assertEquals(200, responseCode) + logger.info("S3 memtable-on-sink rowset meta: {}", body) + def rowsetMeta = parseJson(body) + def locations = rowsetMeta.packed_slice_locations + def segmentIds = rowsetMeta.segment_ids ?: (0..<(rowsetMeta.num_segments as int)).toList() + logger.info("S3 rowset layout (sinkUpload={}): {}", sinkUpload, [ + tablet_id: tablet.TabletId, version: partition.VisibleVersion, + rowset_id: rowsetMeta.rowset_id_v2, segment_ids: segmentIds, + num_segment_rows: rowsetMeta.num_segment_rows, + segments_file_size: rowsetMeta.segments_file_size, + packed_files: locations.keySet().sort() + ]) + // Only the destination rowset's first segment and its V2 index are packed. + // The other five segments retain independent files. + quickTest("s3_packed_meta_${sinkUpload}", """ + SELECT ${rowsetMeta.num_segments as int}, ${rowsetMeta.num_rows as long}, + ${locations.size()}, + ${locations.keySet().count { it.endsWith('_0.dat') }}, + ${locations.keySet().count { it.endsWith('_0.idx') }}, + ${segmentIds.count { (it as long) >= 1000 }}, '${segmentIds.join(",")}' + """, true) + } } - // Stream Load also uses the file-forwarding path when memtable-on-sink is enabled. + GetDebugPoint().disableDebugPointForAllBEs("DeltaWriterV2.sink_upload.duplicate_result") + + // Stream Load inherits the FE sink-upload default independently of SQL session options. // This hook is only reached by the memtable-on-sink receiving writer. // A successful fallback would fail this check instead of silently passing. def failClose = "LoadStreamWriter.close.cancelled" @@ -221,23 +236,67 @@ suite("test_cloud_memtable_on_sink", "p0, docker") { FROM test_cloud_duplicate_memtable_on_sink_s3 """ - streamLoad { - table "test_cloud_duplicate_memtable_on_sink_s3" - set "column_separator", "," - set "memtable_on_sink_node", "true" - set "group_commit", "off_mode" - inputStream new ByteArrayInputStream("100,200\n101,202\n".getBytes()) - time 30000 - check { result, exception, startTime, endTime -> - if (exception != null) { - throw exception + def failUpload = "DeltaWriterV2.sink_upload.after_upload_failure" + try { + GetDebugPoint().enableDebugPointForAllBEs(failUpload) + [null, "true"].each { sinkUploadHeader -> + streamLoad { + table "test_cloud_duplicate_memtable_on_sink_s3" + set "column_separator", "," + set "memtable_on_sink_node", "true" + if (sinkUploadHeader != null) { + set "cloud_memtable_sink_upload", sinkUploadHeader + } + set "group_commit", "off_mode" + inputText "100,200\n101,202\n" + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + sql "SELECT assert_true('${response.Status}' = 'Fail', 'Stream Load should reach sink upload')" + assertTrue(response.Message.contains("injected failure after sink upload")) + } + } + } + // Allow file bytes while keeping the sink-upload failure hook enabled. + GetDebugPoint().disableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + streamLoad { + table "test_cloud_duplicate_memtable_on_sink_s3" + set "column_separator", "," + set "memtable_on_sink_node", "true" + set "cloud_memtable_sink_upload", "false" + set "group_commit", "off_mode" + inputStream new ByteArrayInputStream("100,200\n101,202\n".getBytes()) + time 30000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + assertEquals("Success", parseJson(result).Status) } - assertEquals("Success", parseJson(result).Status) } + } finally { + GetDebugPoint().disableDebugPointForAllBEs(failUpload) + GetDebugPoint().enableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") } order_qt_stream_rows """ SELECT COUNT(*), SUM(k), SUM(v) FROM test_cloud_duplicate_memtable_on_sink_s3 """ + + sql "SET enable_memtable_on_sink_node = true" + sql "SET enable_cloud_memtable_sink_upload = true" + // An uploaded partial rowset must remain invisible if its result cannot be sent. + try { + GetDebugPoint().enableDebugPointForAllBEs("DeltaWriterV2.sink_upload.after_upload_failure") + test { + sql "INSERT INTO test_cloud_duplicate_memtable_on_sink_s3 VALUES (999, 1998)" + exception "injected failure after sink upload" + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs("DeltaWriterV2.sink_upload.after_upload_failure") + } + order_qt_failed_upload_rows """ + SELECT COUNT(*), SUM(k), SUM(v) FROM test_cloud_duplicate_memtable_on_sink_s3 + """ } def testAggregateAndMor = { @@ -260,103 +319,131 @@ suite("test_cloud_memtable_on_sink", "p0, docker") { """ def morQuery = "SELECT k, v FROM cloud_memtable_mor" def seqQuery = "SELECT k, v, seq FROM cloud_memtable_mor_seq" - def loadModelS3 = { table -> + def checkLayout = { table, tag -> + def tablet = sql_return_maparray("SHOW TABLETS FROM ${table}")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM ${table}")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + code, body -> + assertEquals(200, code) + def meta = parseJson(body) + logger.info("{} rowset layout: {}", table, meta) + def ids = meta.segment_ids ?: [] + quickTest(tag, """ + SELECT ${meta.segments_overlap_pb == 'OVERLAPPING'}, + ${ids.size() == (meta.num_segments as int)}, + ${ids.toSet().size() == ids.size()}, + ${ids.any { (it as long) >= 1000 }}, + ${(meta.packed_slice_locations ?: [:]).size() > 0} + """, true) + } + } + def loadModelS3 = { table, sinkUpload -> def label = "agg_mor_" + UUID.randomUUID().toString().replace('-', '_') sql "SET enable_profile=true" def load = loadS3(table, label) - def required = ["DeltaWriterV2"] + def required = sinkUpload ? ["DeltaWriterV2", "CloudMemtableSinkUpload: true"] : ["DeltaWriterV2"] new ProfileAction(context).getProfile(load.JobId.toString(), required) sql "SET enable_profile=false" } - sql "DROP TABLE IF EXISTS cloud_memtable_agg" - sql "DROP TABLE IF EXISTS cloud_memtable_mor" - sql "DROP TABLE IF EXISTS cloud_memtable_mor_seq" - sql "DROP TABLE IF EXISTS cloud_memtable_agg_broker" - sql "DROP TABLE IF EXISTS cloud_memtable_mor_broker" - sql """ - CREATE TABLE cloud_memtable_agg ( - k BIGINT NOT NULL, s BIGINT SUM, lo BIGINT MIN, hi BIGINT MAX, - b BITMAP BITMAP_UNION, h HLL HLL_UNION, - r BIGINT REPLACE, rn BIGINT REPLACE_IF_NOT_NULL - ) AGGREGATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true") - """ - sql """ - CREATE TABLE cloud_memtable_mor (k BIGINT NOT NULL, v BIGINT) - UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="false", - "disable_auto_compaction"="true") - """ - sql """ - CREATE TABLE cloud_memtable_mor_seq ( - k BIGINT NOT NULL, v BIGINT, seq BIGINT NOT NULL, - INDEX idx_k (k) USING INVERTED - ) UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="false", - "function_column.sequence_col"="seq", "inverted_index_storage_format"="V2", - "disable_auto_compaction"="true") - """ - sql "SET enable_profile=true" - sql """ - /* cloud_memtable_agg_false */ - INSERT INTO cloud_memtable_agg - SELECT n % 4, n, n, n, to_bitmap(n), hll_hash(CAST(n % 4 AS STRING)), - n % 4 + 10, IF(n % 8 < 4, n % 4 + 20, NULL) - FROM cloud_memtable_models_source - """ - sql """ - /* cloud_memtable_mor_false */ - INSERT INTO cloud_memtable_mor SELECT n % 4, n % 4 * 10 FROM cloud_memtable_models_source - """ - sql """ - /* cloud_memtable_mor_seq_false */ - INSERT INTO cloud_memtable_mor_seq SELECT n % 4, n * 10, n FROM cloud_memtable_models_source - """ - ['agg', 'mor', 'mor_seq'].each { model -> - def required = ["DeltaWriterV2"] - new ProfileAction(context).getProfileBySql("cloud_memtable_${model}_false", required) - } - sql "SET enable_profile=false" - quickTest("agg_false", aggQuery, true) - quickTest("mor_false", morQuery, true) - quickTest("seq_false", seqQuery, true) - quickTest("seq_index_false", seqQuery + " WHERE k IN (1,3)", true) + [false, true].each { sinkUpload -> + sql "SET enable_cloud_memtable_sink_upload=${sinkUpload}" + if (sinkUpload) { + GetDebugPoint().enableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + } + sql "DROP TABLE IF EXISTS cloud_memtable_agg" + sql "DROP TABLE IF EXISTS cloud_memtable_mor" + sql "DROP TABLE IF EXISTS cloud_memtable_mor_seq" + sql "DROP TABLE IF EXISTS cloud_memtable_agg_broker" + sql "DROP TABLE IF EXISTS cloud_memtable_mor_broker" + sql """ + CREATE TABLE cloud_memtable_agg ( + k BIGINT NOT NULL, s BIGINT SUM, lo BIGINT MIN, hi BIGINT MAX, + b BITMAP BITMAP_UNION, h HLL HLL_UNION, + r BIGINT REPLACE, rn BIGINT REPLACE_IF_NOT_NULL + ) AGGREGATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true") + """ + sql """ + CREATE TABLE cloud_memtable_mor (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="false", + "disable_auto_compaction"="true") + """ + sql """ + CREATE TABLE cloud_memtable_mor_seq ( + k BIGINT NOT NULL, v BIGINT, seq BIGINT NOT NULL, + INDEX idx_k (k) USING INVERTED + ) UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="false", + "function_column.sequence_col"="seq", "inverted_index_storage_format"="V2", + "disable_auto_compaction"="true") + """ + sql "SET enable_profile=true" + sql """ + /* cloud_memtable_agg_${sinkUpload} */ + INSERT INTO cloud_memtable_agg + SELECT n % 4, n, n, n, to_bitmap(n), hll_hash(CAST(n % 4 AS STRING)), + n % 4 + 10, IF(n % 8 < 4, n % 4 + 20, NULL) + FROM cloud_memtable_models_source + """ + sql """ + /* cloud_memtable_mor_${sinkUpload} */ + INSERT INTO cloud_memtable_mor SELECT n % 4, n % 4 * 10 FROM cloud_memtable_models_source + """ + sql """ + /* cloud_memtable_mor_seq_${sinkUpload} */ + INSERT INTO cloud_memtable_mor_seq SELECT n % 4, n * 10, n FROM cloud_memtable_models_source + """ + ['agg', 'mor', 'mor_seq'].each { model -> + def required = sinkUpload ? ["DeltaWriterV2", "CloudMemtableSinkUpload: true"] : ["DeltaWriterV2"] + new ProfileAction(context).getProfileBySql("cloud_memtable_${model}_${sinkUpload}", required) + if (sinkUpload) { + checkLayout("cloud_memtable_${model}", "layout_${model}") + } + } + sql "SET enable_profile=false" + quickTest("agg_${sinkUpload}", aggQuery, true) + quickTest("mor_${sinkUpload}", morQuery, true) + quickTest("seq_${sinkUpload}", seqQuery, true) + quickTest("seq_index_${sinkUpload}", seqQuery + " WHERE k IN (1,3)", true) - // A newer transaction with a lower Sequence must not replace the business-newer row. - sql "INSERT INTO cloud_memtable_mor_seq VALUES (0,-1,1)" - quickTest("seq_lower_false", seqQuery, true) - sql "INSERT INTO cloud_memtable_mor VALUES (0,99)" - quickTest("mor_new_version_false", morQuery, true) - sql """ - INSERT INTO cloud_memtable_agg - VALUES (0,10,-1,13000,bitmap_empty(),hll_empty(),99,NULL) - """ - quickTest("agg_new_version_false", aggQuery, true) - sql "INSERT INTO cloud_memtable_mor_seq SELECT n % 4,n*10,n FROM cloud_memtable_models_source WHERE n<0" - quickTest("seq_empty_false", seqQuery, true) - ['agg', 'mor', 'mor_seq'].each { model -> - trigger_and_wait_compaction("cloud_memtable_${model}", "full") - } - quickTest("agg_compacted_false", aggQuery, true) - quickTest("mor_compacted_false", morQuery, true) - quickTest("seq_compacted_false", seqQuery, true) + // A newer transaction with a lower Sequence must not replace the business-newer row. + sql "INSERT INTO cloud_memtable_mor_seq VALUES (0,-1,1)" + quickTest("seq_lower_${sinkUpload}", seqQuery, true) + sql "INSERT INTO cloud_memtable_mor VALUES (0,99)" + quickTest("mor_new_version_${sinkUpload}", morQuery, true) + sql """ + INSERT INTO cloud_memtable_agg + VALUES (0,10,-1,13000,bitmap_empty(),hll_empty(),99,NULL) + """ + quickTest("agg_new_version_${sinkUpload}", aggQuery, true) + sql "INSERT INTO cloud_memtable_mor_seq SELECT n % 4,n*10,n FROM cloud_memtable_models_source WHERE n<0" + quickTest("seq_empty_${sinkUpload}", seqQuery, true) + ['agg', 'mor', 'mor_seq'].each { model -> + trigger_and_wait_compaction("cloud_memtable_${model}", "full") + } + quickTest("agg_compacted_${sinkUpload}", aggQuery, true) + quickTest("mor_compacted_${sinkUpload}", morQuery, true) + quickTest("seq_compacted_${sinkUpload}", seqQuery, true) - sql """ - CREATE TABLE cloud_memtable_agg_broker (k BIGINT NOT NULL, v BIGINT SUM) - AGGREGATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true") - """ - sql """ - CREATE TABLE cloud_memtable_mor_broker (k BIGINT NOT NULL, v BIGINT) - UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="false", - "disable_auto_compaction"="true") - """ - loadModelS3("cloud_memtable_agg_broker") - loadModelS3("cloud_memtable_mor_broker") - quickTest("agg_broker_false", "SELECT k,v FROM cloud_memtable_agg_broker", true) - quickTest("mor_broker_false", "SELECT k,v FROM cloud_memtable_mor_broker", true) + sql """ + CREATE TABLE cloud_memtable_agg_broker (k BIGINT NOT NULL, v BIGINT SUM) + AGGREGATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true") + """ + sql """ + CREATE TABLE cloud_memtable_mor_broker (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="false", + "disable_auto_compaction"="true") + """ + loadModelS3("cloud_memtable_agg_broker", sinkUpload) + loadModelS3("cloud_memtable_mor_broker", sinkUpload) + quickTest("agg_broker_${sinkUpload}", "SELECT k,v FROM cloud_memtable_agg_broker", true) + quickTest("mor_broker_${sinkUpload}", "SELECT k,v FROM cloud_memtable_mor_broker", true) + } streamLoad { table "cloud_memtable_agg_broker" @@ -388,6 +475,63 @@ suite("test_cloud_memtable_on_sink", "p0, docker") { order_qt_mor_final_compacted seqQuery } + def testVariant = { + sql "DROP TABLE IF EXISTS cloud_memtable_variant_reference" + sql "DROP TABLE IF EXISTS cloud_memtable_variant" + sql """ + CREATE TABLE cloud_memtable_variant_reference (k BIGINT NOT NULL, v VARIANT) + DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true") + """ + sql "CREATE TABLE cloud_memtable_variant LIKE cloud_memtable_variant_reference" + def input = """ + SELECT n, parse_to_variant(IF(n % 2 = 0, + CONCAT('{"a":', n, ',"nested":{"b":', n * 2, '}}'), + CONCAT('{"a":', n, ',"extra":"', n, '"}'))) + FROM cloud_memtable_models_source WHERE n < 1200 + """ + def newPath = """SELECT 1200, parse_to_variant('{"a":1200,"later":2400}')""" + def projection = """ + SELECT k, CAST(v['a'] AS BIGINT), CAST(v['nested']['b'] AS BIGINT), + CAST(v['extra'] AS STRING), CAST(v['later'] AS BIGINT) + """ + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_memtable_variant_reference ${input}" + sql "INSERT INTO cloud_memtable_variant_reference ${newPath}" + def expected = sql "${projection} FROM cloud_memtable_variant_reference ORDER BY k" + sql "SET enable_memtable_on_sink_node=true" + sql "SET enable_cloud_memtable_sink_upload=true" + sql "SET enable_profile=true" + GetDebugPoint().enableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + try { + sql """ + /* cloud_memtable_variant_profile */ + INSERT INTO cloud_memtable_variant ${input} + """ + new ProfileAction(context).getProfileBySql("cloud_memtable_variant_profile", + ["DeltaWriterV2", "CloudMemtableSinkUpload: true"]) + sql "SET enable_profile=false" + def tablet = sql_return_maparray("SHOW TABLETS FROM cloud_memtable_variant")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM cloud_memtable_variant")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + code, body -> + assertEquals(200, code) + def ids = parseJson(body).segment_ids + assertTrue(ids.any { (it as long) >= 1000 }, + "VARIANT schema assembly must include multiple sink writers: ${ids}") + } + // A later rowset introduces another extracted path before compaction. + sql "INSERT INTO cloud_memtable_variant ${newPath}" + assertEquals(expected, sql("${projection} FROM cloud_memtable_variant ORDER BY k")) + trigger_and_wait_compaction("cloud_memtable_variant", "full") + assertEquals(expected, sql("${projection} FROM cloud_memtable_variant ORDER BY k")) + } finally { + sql "SET enable_profile=false" + GetDebugPoint().disableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + } + } + def originalBatchSize = sql("SHOW VARIABLES LIKE 'broker_load_batch_size'")[0][1] try { testDuplicate() @@ -395,11 +539,15 @@ suite("test_cloud_memtable_on_sink", "p0, docker") { sql "SET broker_load_batch_size=${originalBatchSize}" sql "SET enable_profile=false" GetDebugPoint().disableDebugPointForAllBEs("MemTable.need_flush") + GetDebugPoint().disableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + GetDebugPoint().disableDebugPointForAllBEs("DeltaWriterV2.sink_upload.duplicate_result") } try { testAggregateAndMor() } finally { sql "SET enable_profile=false" + GetDebugPoint().disableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") } + testVariant() } } diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_cache.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_cache.groovy new file mode 100644 index 00000000000000..02dd58cd435a8d --- /dev/null +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_cache.groovy @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.Http + +suite("test_cloud_memtable_sink_upload_cache", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(3) + options.enableDebugPoints() + options.feConfigs += ['cloud_stream_load_default_memtable_sink_upload=true'] + options.beConfigs += ['enable_file_cache=true', 'small_file_threshold_bytes=1048576'] + docker(options) { + sql "DROP TABLE IF EXISTS cloud_memtable_sink_upload_cache" + sql """ + CREATE TABLE cloud_memtable_sink_upload_cache ( + k BIGINT NOT NULL, v BIGINT, INDEX idx_k(k) USING INVERTED + ) DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true", + "inverted_index_storage_format"="V2") + """ + def tablet = sql_return_maparray("SHOW TABLETS FROM cloud_memtable_sink_upload_cache")[0] + def backends = sql_return_maparray("SHOW BACKENDS") + def target = backends.find { it.BackendId == tablet.BackendId } + def remote = backends.find { it.BackendId != tablet.BackendId } + def ms = cluster.getAllMetaservices()[0] + def cached = { be, file -> + !Http.GET("http://${be.Host}:${be.HttpPort}/api/file_cache?op=list_cache&value=${file}", true).isEmpty() + } + def warmupStats = { + Http.GET("http://${target.Host}:${target.HttpPort}/api/warmup_event_driven_stats", true) + .data.find { it.job_id == 0 } + } + def skipWarmup = "FileCacheBlockDownloader::download_segment_file.skip_warmup" + GetDebugPoint().enableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + try { + [false, true].each { packed -> + setBeConfigTemporary(['enable_packed_file': packed.toString()]) { + [false, true].each { local -> + def sink = local ? target : remote + def tag = "${packed ? 'packed' : 'plain'}_${local ? 'local' : 'remote'}" + def before = warmupStats() + long finishedSegments = before?.finish?.seg?.num?.get('1h') ?: 0 + long finishedIndexes = before?.finish?.idx?.num?.get('1h') ?: 0 + // For the same-BE case, cache must come from uploading, even without warmup. + if (local) { + GetDebugPoint().enableDebugPointForAllBEs(skipWarmup) + } + try { + streamLoad { + table "cloud_memtable_sink_upload_cache" + directToBe sink.Host, sink.HttpPort as int + set "column_separator", "," + set "memtable_on_sink_node", "true" + set "group_commit", "off_mode" + inputStream new ByteArrayInputStream("1,10\n2,20\n".getBytes()) + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + quickTest("load_${tag}", """ + SELECT '${response.Status}', ${response.NumberLoadedRows}, + ${response.NumberFilteredRows} + """, true) + } + } + def partition = sql_return_maparray( + "SHOW PARTITIONS FROM cloud_memtable_sink_upload_cache")[0] + def meta + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, + partition.VisibleVersion) { code, body -> + assertEquals(200, code) + meta = parseJson(body) + } + def ids = meta.segment_ids + quickTest("layout_${tag}", """ + SELECT ${meta.num_segments as int}, + ${(meta.packed_slice_locations ?: [:]).size() > 0} + """, true) + def files = ids.collectMany { id -> + ["${meta.rowset_id_v2}_${id}.dat", "${meta.rowset_id_v2}_${id}.idx"] + } + // Wait for download callbacks, then inspect cache keys before querying data. + // list_cache alone also lists blocks whose downloads have not completed. + awaitUntil(60) { + def stats = warmupStats() + stats != null && + stats.finish.seg.num['1h'] >= finishedSegments + ids.size() && + stats.finish.idx.num['1h'] >= finishedIndexes + ids.size() && + files.every { cached(target, it) } + } + if (packed) { + // Drain sink-side async cache writes before asserting their absence. + awaitUntil(60) { + // BRPC serves HTML to Java's user agent, and plain text to curl. + def (code, out, err) = curl("GET", + "http://${sink.Host}:${sink.BrpcPort}/vars/packed_file_cache_async_write_count") + assertEquals(0, code, err) + out.trim().split(':')[-1].trim().toLong() == 0 + } + } + quickTest("cache_${tag}", """ + SELECT ${files.every { cached(target, it) }}, + ${files.every { !cached(remote, it) }} + """, true) + } finally { + if (local) { + GetDebugPoint().disableDebugPointForAllBEs(skipWarmup) + } + } + } + } + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + } + order_qt_rows "SELECT k, SUM(v), COUNT(*) FROM cloud_memtable_sink_upload_cache GROUP BY k" + } +} diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_mow.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_mow.groovy new file mode 100644 index 00000000000000..5992161726f235 --- /dev/null +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_mow.groovy @@ -0,0 +1,282 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction +import org.apache.doris.regression.suite.ClusterOptions +import java.util.concurrent.TimeUnit + +suite("test_cloud_memtable_sink_upload_mow", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(3) + options.enableDebugPoints() + options.beConfigs += ['share_delta_writers=false', 'enable_packed_file=true', + 'small_file_threshold_bytes=1048576', + 'enable_merge_on_write_correctness_check=true'] + options.feConfigs += ['cloud_stream_load_default_memtable_sink_upload=true', + 'min_bytes_per_broker_scanner=100'] + docker(options) { + sql "SET enable_sql_cache=false" + def checkUniqueKeys = { table -> + sql """ + SELECT assert_true(COUNT(*) = 0, '${table} contains duplicate keys') FROM ( + SELECT k, COUNT(*) AS a FROM ${table} GROUP BY k HAVING a > 1 + ) duplicates + """ + } + sql "DROP TABLE IF EXISTS cloud_mow_source" + sql "DROP TABLE IF EXISTS cloud_mow_direct" + sql "DROP TABLE IF EXISTS cloud_mow_direct_seq" + sql "DROP TABLE IF EXISTS cloud_mow_broker" + sql """ + CREATE TABLE cloud_mow_source (n BIGINT NOT NULL) + DUPLICATE KEY(n) DISTRIBUTED BY HASH(n) BUCKETS 12 + PROPERTIES ("replication_num"="1") + """ + sql """ + CREATE TABLE cloud_mow_direct (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "disable_auto_compaction"="true") + """ + sql """ + CREATE TABLE cloud_mow_direct_seq ( + k BIGINT NOT NULL, v BIGINT, seq BIGINT NOT NULL, INDEX idx_k(k) USING INVERTED + ) UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "function_column.sequence_col"="seq", "inverted_index_storage_format"="V2", + "disable_auto_compaction"="true") + """ + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_mow_source SELECT number FROM numbers('number'='12000')" + // Two versions leave deleted old rows in the snapshot. + sql "INSERT INTO cloud_mow_direct VALUES (0,-1),(1,-1),(2,-1),(3,-1)" + sql "INSERT INTO cloud_mow_direct VALUES (0,0),(1,0),(2,0),(3,0)" + sql "INSERT INTO cloud_mow_direct_seq VALUES (0,-1,-1),(1,-1,-1),(2,-1,-1),(3,-1,-1)" + sql "INSERT INTO cloud_mow_direct_seq VALUES (0,0,0),(1,0,0),(2,0,0),(3,0,0)" + sql "SET enable_memtable_on_sink_node=true" + sql "SET enable_cloud_memtable_sink_upload=true" + sql "SET parallel_pipeline_task_num=4" + sql "SET enable_file_cache=false" + sql "SET profile_level=2" + sql "SET enable_profile=true" + GetDebugPoint().enableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + GetDebugPoint().enableDebugPointForAllBEs("DeltaWriterV2.sink_upload.duplicate_result") + sql """ + /* cloud_mow_direct_profile */ + INSERT INTO cloud_mow_direct SELECT n%4,n%4*10 FROM cloud_mow_source + """ + sql """ + /* cloud_mow_direct_seq_profile */ + INSERT INTO cloud_mow_direct_seq SELECT n%4,n*10,n+1 FROM cloud_mow_source + """ + ['cloud_mow_direct_profile', 'cloud_mow_direct_seq_profile'].each { tag -> + new ProfileAction(context).getProfileBySql(tag, + ["CloudMemtableSinkUpload: true", "CloudMemtableMowBitmap: true"]) + } + sql "SET enable_profile=false" + GetDebugPoint().disableDebugPointForAllBEs("DeltaWriterV2.sink_upload.duplicate_result") + order_qt_initial "SELECT * FROM cloud_mow_direct" + order_qt_initial_seq "SELECT * FROM cloud_mow_direct_seq" + order_qt_index "SELECT * FROM cloud_mow_direct_seq WHERE k IN (1,3)" + ['cloud_mow_direct', 'cloud_mow_direct_seq'].each { table -> + checkUniqueKeys(table) + def tablet = sql_return_maparray("SHOW TABLETS FROM ${table}")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM ${table}")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + code, body -> + assertEquals(200, code) + def meta = parseJson(body) + logger.info("MOW rowset layout: {}", meta) + def ids = meta.segment_ids ?: [] + quickTest("layout_${table}", """ + SELECT ${ids.size() == (meta.num_segments as int)}, + ${ids.toSet().size() == ids.size()}, + ${ids.any { (it as long) >= 1000 }}, + ${(meta.packed_slice_locations ?: [:]).size() > 0} + """, true) + } + } + sql "INSERT INTO cloud_mow_direct_seq VALUES (0,-1,1)" + order_qt_lower_seq "SELECT * FROM cloud_mow_direct_seq" + sql "INSERT INTO cloud_mow_direct VALUES (0,99)" + order_qt_new_version "SELECT * FROM cloud_mow_direct" + sql "INSERT INTO cloud_mow_direct_seq SELECT n%4,n*10,n FROM cloud_mow_source WHERE n<0" + order_qt_empty "SELECT * FROM cloud_mow_direct_seq" + sql "INSERT INTO cloud_mow_direct_seq (k,v,seq,__DORIS_DELETE_SIGN__) VALUES (2,130000,13000,1)" + order_qt_delete "SELECT * FROM cloud_mow_direct_seq" + sql "INSERT INTO cloud_mow_direct_seq VALUES (2,-1,1)" + order_qt_lower_after_delete "SELECT * FROM cloud_mow_direct_seq" + sql "INSERT INTO cloud_mow_direct_seq VALUES (2,140000,14000)" + order_qt_reinsert "SELECT * FROM cloud_mow_direct_seq" + trigger_and_wait_compaction("cloud_mow_direct", "full") + trigger_and_wait_compaction("cloud_mow_direct_seq", "full") + order_qt_compacted "SELECT * FROM cloud_mow_direct" + order_qt_compacted_seq "SELECT * FROM cloud_mow_direct_seq" + checkUniqueKeys("cloud_mow_direct") + checkUniqueKeys("cloud_mow_direct_seq") + + streamLoad { + table "cloud_mow_direct_seq" + set "column_separator", "," + set "memtable_on_sink_node", "true" + inputStream new ByteArrayInputStream("0,150000,15000\n3,-1,1\n".getBytes()) + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + assertEquals("Success", parseJson(result).Status) + } + } + order_qt_stream "SELECT * FROM cloud_mow_direct_seq" + + // Freeze the snapshot, then replace its rowsets with a concurrent load and compaction. + def block = "CloudRowsetBuilder.sink_mow.snapshot_ready" + def backends = sql_return_maparray("SHOW BACKENDS") + GetDebugPoint().enableDebugPointForAllBEs(block, [timeout: "120"]) + def pending = thread { + sql "SET enable_memtable_on_sink_node=true" + sql "SET enable_cloud_memtable_sink_upload=true" + sql "INSERT INTO cloud_mow_direct_seq VALUES (0,200000,20000)" + } + try { + awaitUntil(60) { + backends.any { be -> + def (code, out, err) = curl("GET", + "http://${be.Host}:${be.BrpcPort}/vars/cloud_memtable_mow_snapshot_waiters") + assertEquals(0, code, err) + def ready = out =~ /cloud_memtable_mow_snapshot_waiters\s*:\s*(\d+)/ + ready.find() && ready.group(1).toLong() > 0 + } + } + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_mow_direct_seq VALUES (0,210000,21000)" + trigger_and_wait_compaction("cloud_mow_direct_seq", "full") + } finally { + GetDebugPoint().disableDebugPointForAllBEs(block) + sql "SET enable_memtable_on_sink_node=true" + } + pending.get(120, TimeUnit.SECONDS) + order_qt_concurrent_compaction "SELECT * FROM cloud_mow_direct_seq" + checkUniqueKeys("cloud_mow_direct_seq") + + GetDebugPoint().enableDebugPointForAllBEs("DeltaWriterV2.sink_mow.after_bitmap_failure") + try { + test { + sql "INSERT INTO cloud_mow_direct_seq VALUES (99,990000,99000)" + exception "injected failure after sink MOW bitmap calculation" + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs("DeltaWriterV2.sink_mow.after_bitmap_failure") + } + order_qt_failed_invisible "SELECT * FROM cloud_mow_direct_seq" + + // Cluster keys encode a physical row ID in the primary-key index. + sql "DROP TABLE IF EXISTS cloud_mow_cluster" + sql """ + CREATE TABLE cloud_mow_cluster (k BIGINT NOT NULL, v BIGINT NOT NULL, seq BIGINT NOT NULL) + UNIQUE KEY(k) ORDER BY(v) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "function_column.sequence_col"="seq", "disable_auto_compaction"="true") + """ + sql "INSERT INTO cloud_mow_cluster SELECT n%4,-n,n+1 FROM cloud_mow_source" + order_qt_cluster "SELECT * FROM cloud_mow_cluster" + sql "INSERT INTO cloud_mow_cluster VALUES (0,999,20000),(1,999,1)" + order_qt_cluster_update "SELECT * FROM cloud_mow_cluster" + trigger_and_wait_compaction("cloud_mow_cluster", "full") + order_qt_cluster_compacted "SELECT * FROM cloud_mow_cluster" + checkUniqueKeys("cloud_mow_cluster") + + setBeConfigTemporary(['enable_packed_file': 'false']) { + sql "DROP TABLE IF EXISTS cloud_mow_plain_reference" + sql "DROP TABLE IF EXISTS cloud_mow_plain" + sql "CREATE TABLE cloud_mow_plain_reference LIKE cloud_mow_direct_seq" + sql "CREATE TABLE cloud_mow_plain LIKE cloud_mow_direct_seq" + try { + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_mow_plain_reference VALUES (0,-1,-1),(1,-1,-1)" + sql "INSERT INTO cloud_mow_plain VALUES (0,-1,-1),(1,-1,-1)" + sql "INSERT INTO cloud_mow_plain_reference SELECT n%4,n*10,n+1 FROM cloud_mow_source" + def expected = sql "SELECT k,v,seq FROM cloud_mow_plain_reference ORDER BY k" + sql "SET enable_memtable_on_sink_node=true" + sql "SET enable_profile=true" + sql """ + /* cloud_mow_plain_profile */ + INSERT INTO cloud_mow_plain SELECT n%4,n*10,n+1 FROM cloud_mow_source + """ + new ProfileAction(context).getProfileBySql("cloud_mow_plain_profile", + ["CloudMemtableSinkUpload: true", "CloudMemtableMowBitmap: true"]) + sql "SET enable_profile=false" + checkUniqueKeys("cloud_mow_plain") + assertEquals(expected, sql("SELECT k,v,seq FROM cloud_mow_plain ORDER BY k")) + def tablet = sql_return_maparray("SHOW TABLETS FROM cloud_mow_plain")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM cloud_mow_plain")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + code, body -> + assertEquals(200, code) + def meta = parseJson(body) + assertTrue((meta.packed_slice_locations ?: [:]).isEmpty(), + "Unpacked MOW rowset contains packed mappings: ${meta}") + } + trigger_and_wait_compaction("cloud_mow_plain", "full") + checkUniqueKeys("cloud_mow_plain") + assertEquals(expected, sql("SELECT k,v,seq FROM cloud_mow_plain ORDER BY k")) + } finally { + sql "SET enable_profile=false" + sql "SET enable_memtable_on_sink_node=true" + } + } + + sql """ + CREATE TABLE cloud_mow_broker (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true") + """ + // Switch the same table from sink upload to forwarding and overwrite a sink-uploaded key. + [true, false].each { sinkUpload -> + if (!sinkUpload) { + sql "INSERT INTO cloud_mow_broker VALUES (50,-1)" + GetDebugPoint().disableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + } + sql "SET enable_cloud_memtable_sink_upload=${sinkUpload}" + def label = "mow_broker_" + UUID.randomUUID().toString().replace('-', '_') + sql "SET enable_profile=true" + sql """ + LOAD LABEL ${label} ( + DATA INFILE("s3://${getS3BucketName()}/regression/load/data/basic_data.csv") + INTO TABLE cloud_mow_broker COLUMNS TERMINATED BY "|" FORMAT AS "CSV" + (k, c01, c02, c03, c04, c05, c06, c07, c08, c09, + c10, c11, c12, c13, c14, c15, c16, c17, c18) SET (v=k*2) + ) WITH S3 ( + "AWS_ACCESS_KEY"="${getS3AK()}", "AWS_SECRET_KEY"="${getS3SK()}", + "AWS_ENDPOINT"="${getS3Endpoint()}", "AWS_REGION"="${getS3Region()}", + "provider"="${getS3Provider()}" + ) PROPERTIES ("load_parallelism"="1") + """ + waitForBrokerLoadDone(label) + def load = sql_return_maparray("SHOW LOAD WHERE LABEL = '${label}'")[0] + assertEquals("FINISHED", load.State, "Broker load did not finish: ${load}") + new ProfileAction(context).getProfile(load.JobId.toString(), + sinkUpload ? ["CloudMemtableSinkUpload: true", "CloudMemtableMowBitmap: true"] + : ["DeltaWriterV2"]) + sql "SET enable_profile=false" + quickTest("broker_${sinkUpload}", "SELECT * FROM cloud_mow_broker", true) + checkUniqueKeys("cloud_mow_broker") + } + } +} diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_unshared.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_unshared.groovy new file mode 100644 index 00000000000000..b147958b1f7a05 --- /dev/null +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_unshared.groovy @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction +import org.apache.doris.regression.suite.ClusterOptions + +suite("test_cloud_memtable_sink_upload_unshared", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(3) + options.beConfigs += ['share_delta_writers=false', 'enable_packed_file=false', + 'max_segment_num_per_rowset=1000'] + docker(options) { + sql "DROP TABLE IF EXISTS test_cloud_sink_upload_source" + sql "DROP TABLE IF EXISTS test_cloud_sink_upload_unshared" + sql """ + CREATE TABLE test_cloud_sink_upload_source (k BIGINT, v BIGINT) + DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 12 + PROPERTIES ("replication_num"="1") + """ + sql """ + CREATE TABLE test_cloud_sink_upload_unshared (k BIGINT, v BIGINT) + DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true") + """ + sql "SET enable_sql_cache=false" + sql "SET enable_memtable_on_sink_node=false" + sql """ + INSERT INTO test_cloud_sink_upload_source + SELECT number, number * 2 FROM numbers("number"="100000") + """ + sql "SET enable_memtable_on_sink_node=true" + sql "SET enable_cloud_memtable_sink_upload=true" + sql "SET parallel_pipeline_task_num=4" + sql "SET profile_level=2" + sql "SET enable_profile=true" + sql """ + /* cloud_sink_upload_unshared_profile */ + INSERT INTO test_cloud_sink_upload_unshared + SELECT * FROM test_cloud_sink_upload_source + """ + def profileString = new ProfileAction(context).getProfileBySql( + "cloud_sink_upload_unshared_profile", ["CloudMemtableSinkUpload: true"]) + logger.info("unshared sink-upload profile:\n{}", profileString) + sql "SET enable_profile=false" + sql "SET enable_file_cache=false" + order_qt_rows """ + SELECT COUNT(*), SUM(k), SUM(v) FROM test_cloud_sink_upload_unshared + """ + def tablet = sql_return_maparray("SHOW TABLETS FROM test_cloud_sink_upload_unshared")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM test_cloud_sink_upload_unshared")[0] + def ms = cluster.getAllMetaservices()[0] + def backendCount = sql_return_maparray("SHOW BACKENDS").size() + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + responseCode, body -> + assertEquals(200, responseCode) + def rowsetMeta = parseJson(body) + def segmentIds = rowsetMeta.segment_ids ?: [] + logger.info("Unshared sink-upload rowset layout: {}", [ + tablet_id: tablet.TabletId, version: partition.VisibleVersion, + rowset_id: rowsetMeta.rowset_id_v2, segment_ids: segmentIds, + num_segment_rows: rowsetMeta.num_segment_rows, + segments_file_size: rowsetMeta.segments_file_size + ]) + // One shared writer per BE cannot produce more nonempty ranges than BEs. + def writerRanges = segmentIds.collect { (it as long).intdiv(1000) }.toSet() + assertTrue(writerRanges.size() > backendCount, + "Expected multiple writers on one source BE for tablet ${tablet.TabletId}: ${segmentIds}") + order_qt_rowset_layout """ + SELECT ${segmentIds.size() == (rowsetMeta.num_segments as int)}, + ${segmentIds.toSet().size() == segmentIds.size()}, + ${segmentIds.any { (it as long) >= 1000 }}, + ${rowsetMeta.num_rows as long} + """ + } + sql """ + INSERT INTO test_cloud_sink_upload_unshared + SELECT * FROM test_cloud_sink_upload_source WHERE k < 0 + """ + order_qt_empty_input """ + SELECT COUNT(*), SUM(k), SUM(v) FROM test_cloud_sink_upload_unshared + """ + // Verify that another transaction can coexist with the sparse-id rowset. + sql """ + INSERT INTO test_cloud_sink_upload_unshared + SELECT * FROM test_cloud_sink_upload_source WHERE k < 10 + """ + order_qt_second_load """ + SELECT COUNT(*), SUM(k), SUM(v) FROM test_cloud_sink_upload_unshared + """ + } +}