diff --git a/be/src/cloud/cloud_rowset_builder.cpp b/be/src/cloud/cloud_rowset_builder.cpp index d9065c340ece9e..7fc272032bb5e0 100644 --- a/be/src/cloud/cloud_rowset_builder.cpp +++ b/be/src/cloud/cloud_rowset_builder.cpp @@ -17,18 +17,24 @@ #include "cloud/cloud_rowset_builder.h" +#include +#include + #include #include "cloud/cloud_meta_mgr.h" +#include "cloud/cloud_rowset_writer.h" #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" #include "cloud/cloud_tablet_mgr.h" +#include "exec/common/variant_util.h" #include "io/fs/file_system.h" #include "storage/rowset/group_rowset_writer.h" #include "storage/rowset/rowset_factory.h" #include "storage/rowset/rowset_writer_context.h" #include "storage/storage_policy.h" #include "storage/tablet_info.h" +#include "util/defer_op.h" namespace doris { using namespace ErrorCode; @@ -249,6 +255,236 @@ Status CloudRowsetBuilder::commit_rowset(const std::string& job_id, int64_t tabl return _engine.meta_mgr().commit_rowset(*rowset_meta(), job_id, table_id); } +Status CloudRowsetBuilder::get_mow_snapshot_for_sink(PCloudLoadMowSnapshot* snapshot) { + DORIS_CHECK(_tablet->enable_unique_key_merge_on_write()); + if (_mow_snapshot_for_sink == nullptr) { + RETURN_IF_ERROR(cloud_tablet()->sync_rowsets()); + auto context = _rowset_writer->context().mow_context; + DORIS_CHECK(context != nullptr); + DeleteBitmap snapshot_bitmap(_tablet->tablet_id()); + { + std::unique_lock sync_lock(cloud_tablet()->get_sync_meta_lock()); + std::shared_lock lock(_tablet->get_header_lock()); + if (_tablet->tablet_state() != TABLET_RUNNING) { + return Status::NotSupported("sink MOW load requires a running tablet {}", + _tablet->tablet_id()); + } + _max_version_in_flush_phase = _tablet->max_version_unlocked(); + _rowset_ids->clear(); + RETURN_IF_ERROR(_tablet->get_all_rs_id_unlocked(_max_version_in_flush_phase, + _rowset_ids.get())); + context->max_version = _max_version_in_flush_phase; + context->rowset_ptrs = _tablet->get_rowset_by_ids(_rowset_ids.get()); + std::vector rowset_segments; + for (const auto& rowset : context->rowset_ptrs) { + std::vector ids; + for (auto segment : rowset->segments()) { + ids.push_back(cast_set(segment.id())); + } + rowset_segments.emplace_back(rowset->rowset_id(), std::move(ids)); + } + _tablet->tablet_meta()->delete_bitmap().subset_and_agg( + rowset_segments, 0, _max_version_in_flush_phase, &snapshot_bitmap); + } + _mow_snapshot_for_sink = std::make_unique(); + _mow_snapshot_for_sink->set_version(_max_version_in_flush_phase); + // Keep the rowset references in MowContext until the load has finished. + for (const auto& rowset : context->rowset_ptrs) { + auto* meta = _mow_snapshot_for_sink->add_rowsets(); + *meta = rowset->rowset_meta()->get_rowset_pb(); + meta->clear_tablet_schema(); + rowset->tablet_schema()->to_schema_pb(meta->mutable_tablet_schema()); + } + *_mow_snapshot_for_sink->mutable_delete_bitmap() = snapshot_bitmap.to_pb(); + } + *snapshot = *_mow_snapshot_for_sink; + DBUG_EXECUTE_IF("CloudRowsetBuilder.sink_mow.snapshot_ready", { + // Expose readiness while blocked, independent of asynchronous log flushing. + static bvar::Adder waiters("cloud_memtable_mow_snapshot_waiters"); + waiters << 1; + Defer release([] { waiters << -1; }); + DBUG_BLOCK; + }); + return Status::OK(); +} + +Status CloudRowsetBuilder::validate_sink_mow_result(const PCloudLoadMowResult& result, + int64_t snapshot_version) { + if (!result.has_snapshot_version() || result.snapshot_version() != snapshot_version || + !result.has_delete_bitmap()) { + return Status::InvalidArgument("missing or mismatched sink MOW snapshot result"); + } + const auto& bitmap = result.delete_bitmap(); + const auto count = bitmap.rowset_ids_size(); + if (bitmap.segment_ids_size() != count || bitmap.versions_size() != count || + bitmap.segment_delete_bitmaps_size() != count) { + return Status::InvalidArgument("misaligned sink MOW delete bitmap"); + } + for (int pos = 0; pos < count; ++pos) { + if (bitmap.versions(pos) != DeleteBitmap::TEMP_VERSION_COMMON) { + return Status::InvalidArgument("sink MOW bitmap must use the temporary version"); + } + const auto& bytes = bitmap.segment_delete_bitmaps(pos); + const auto size = + roaring::api::roaring_bitmap_portable_deserialize_size(bytes.data(), bytes.size()); + if (size == 0 || size != bytes.size()) { + return Status::Corruption("invalid serialized sink MOW bitmap"); + } + } + return Status::OK(); +} + +Status CloudRowsetBuilder::merge_sink_mow_bitmap(const PCloudLoadMowResult& result) { + DORIS_CHECK(_mow_snapshot_for_sink != nullptr); + RETURN_IF_ERROR(validate_sink_mow_result(result, _mow_snapshot_for_sink->version())); + _delete_bitmap->merge(DeleteBitmap::from_pb(result.delete_bitmap(), _tablet->tablet_id())); + return Status::OK(); +} + +Status CloudRowsetBuilder::validate_partial_rowset_meta(const RowsetMetaPB& base_meta, + const RowsetMetaPB& partial_meta, + int32_t segment_start_id, + int32_t segment_capacity) { + const auto count = partial_meta.num_segments(); + if (partial_meta.rowset_id_v2() != base_meta.rowset_id_v2() || + partial_meta.tablet_id() != base_meta.tablet_id() || + partial_meta.txn_id() != base_meta.txn_id() || + partial_meta.resource_id() != base_meta.resource_id() || + partial_meta.index_id() != base_meta.index_id() || + partial_meta.partition_id() != base_meta.partition_id() || + partial_meta.tablet_schema_hash() != base_meta.tablet_schema_hash() || + partial_meta.table_id() != base_meta.table_id() || + partial_meta.db_id() != base_meta.db_id() || + UniqueId(partial_meta.load_id()) != UniqueId(base_meta.load_id())) { + return Status::InvalidArgument("sink upload rowset identity mismatch for tablet {}", + base_meta.tablet_id()); + } + if (!partial_meta.has_tablet_schema() || count < 0 || count > segment_capacity || + partial_meta.segment_ids_size() != count || partial_meta.num_segment_rows_size() != count || + partial_meta.segments_key_bounds_size() != count || + partial_meta.segments_file_size_size() != count || + partial_meta.segments_key_bounds_aggregated() || + (partial_meta.inverted_index_file_info_size() != 0 && + partial_meta.inverted_index_file_info_size() != count)) { + return Status::InvalidArgument("misaligned sink upload metadata for tablet {}", + base_meta.tablet_id()); + } + const bool has_index = + std::ranges::any_of(partial_meta.tablet_schema().index(), [](const auto& index) { + return index.index_type() == IndexType::INVERTED || + index.index_type() == IndexType::ANN; + }); + if (has_index && partial_meta.inverted_index_file_info_size() != count) { + return Status::InvalidArgument("missing sink upload index metadata for tablet {}", + base_meta.tablet_id()); + } + int64_t partial_rows = 0; + for (int pos = 0; pos < count; ++pos) { + const int64_t id = partial_meta.segment_ids(pos); + if (id < segment_start_id || + id >= static_cast(segment_start_id) + segment_capacity || + (pos > 0 && id <= partial_meta.segment_ids(pos - 1)) || + partial_meta.num_segment_rows(pos) < 0 || partial_meta.segments_file_size(pos) <= 0) { + return Status::InvalidArgument("invalid sink upload segment {} for tablet {}", id, + base_meta.tablet_id()); + } + partial_rows += partial_meta.num_segment_rows(pos); + } + if (partial_rows != partial_meta.num_rows() || partial_meta.data_disk_size() < 0 || + partial_meta.index_disk_size() < 0 || + partial_meta.total_disk_size() != + partial_meta.data_disk_size() + partial_meta.index_disk_size()) { + return Status::InvalidArgument("invalid sink upload statistics for tablet {}", + base_meta.tablet_id()); + } + return Status::OK(); +} + +Status CloudRowsetBuilder::assemble_rowset_meta_from_partials( + const RowsetMetaPB& base_meta, const std::map& partial_rowset_metas, + int32_t max_segments_per_rowset, RowsetMetaPB* result) { + *result = base_meta; + int64_t rows = 0; + int64_t data_size = 0; + int64_t index_size = 0; + auto schema = std::make_shared(); + schema->init_from_pb(base_meta.tablet_schema()); + const bool has_variant = schema->num_variant_columns() > 0; + std::vector schemas; + for (const auto& [segment_start_id, partial_meta] : partial_rowset_metas) { + const auto count = partial_meta.num_segments(); + if (count > 0 && result->segment_ids_size() > 0 && + partial_meta.segment_ids(0) <= result->segment_ids(result->segment_ids_size() - 1)) { + return Status::InvalidArgument("overlapping sink upload segment ranges"); + } + result->mutable_segment_ids()->MergeFrom(partial_meta.segment_ids()); + result->mutable_num_segment_rows()->MergeFrom(partial_meta.num_segment_rows()); + result->mutable_segments_file_size()->MergeFrom(partial_meta.segments_file_size()); + result->mutable_segments_key_bounds()->MergeFrom(partial_meta.segments_key_bounds()); + result->mutable_inverted_index_file_info()->MergeFrom( + partial_meta.inverted_index_file_info()); + for (const auto& [path, location] : partial_meta.packed_slice_locations()) { + if (!result->mutable_packed_slice_locations()->emplace(path, location).second) { + return Status::InvalidArgument("duplicate packed slice {}", path); + } + } + rows += partial_meta.num_rows(); + data_size += partial_meta.data_disk_size(); + index_size += partial_meta.index_disk_size(); + result->set_segments_key_bounds_truncated(result->segments_key_bounds_truncated() || + partial_meta.segments_key_bounds_truncated()); + if (has_variant) { + auto partial_schema = std::make_shared(); + partial_schema->init_from_pb(partial_meta.tablet_schema()); + schemas.push_back(std::move(partial_schema)); + } + } + if (result->segment_ids_size() > max_segments_per_rowset) { + return Status::InvalidArgument("too many sink upload segments for tablet {}", + base_meta.tablet_id()); + } + if (has_variant && !schemas.empty()) { + TabletSchemaSPtr merged_schema; + schemas.push_back(schema); + RETURN_IF_ERROR(variant_util::get_least_common_schema(schemas, nullptr, merged_schema)); + result->clear_tablet_schema(); + merged_schema->to_schema_pb(result->mutable_tablet_schema()); + } + result->set_num_segments(result->segment_ids_size()); + result->set_num_rows(rows); + result->set_data_disk_size(data_size); + result->set_index_disk_size(index_size); + result->set_total_disk_size(data_size + index_size); + result->set_empty(rows == 0); + result->set_segments_overlap_pb( + schema->cluster_key_uids().empty() && + !is_segment_overlapping(result->segments_key_bounds()) + ? NONOVERLAPPING + : OVERLAPPING); + result->set_enable_segments_file_size(true); + result->set_enable_inverted_index_file_info(true); + result->set_creation_time(UnixSeconds()); + result->set_newest_write_timestamp(UnixSeconds()); + result->set_rowset_state(COMMITTED); + return Status::OK(); +} + +Status CloudRowsetBuilder::build_rowset_from_assembled_meta(const RowsetMetaPB& meta) { + return static_cast(_rowset_writer.get()) + ->build_from_assembled_meta(meta, _rowset); +} + +Status CloudRowsetBuilder::commit_txn() { + DCHECK(is_data_builder()); + if (!_skip_writing_rowset_metadata) { + RETURN_IF_ERROR(commit_rowset("", _tablet->table_id())); + } + RETURN_IF_ERROR(set_txn_related_info()); + update_tablet_stats(); + _is_committed = true; + return Status::OK(); +} + Status CloudRowsetBuilder::set_txn_related_info() { if (_tablet->enable_unique_key_merge_on_write() || _tablet->is_row_binlog_tablet()) { // For empty rowsets when skip_writing_empty_rowset_metadata=true, diff --git a/be/src/cloud/cloud_rowset_builder.h b/be/src/cloud/cloud_rowset_builder.h index c65e192444865d..d77dcbfdb306fc 100644 --- a/be/src/cloud/cloud_rowset_builder.h +++ b/be/src/cloud/cloud_rowset_builder.h @@ -17,11 +17,15 @@ #pragma once +#include + #include "storage/rowset_builder.h" namespace doris { class CloudTablet; +class PCloudLoadMowSnapshot; +class PCloudLoadMowResult; class CloudStorageEngine; class CloudRowsetBuilder : public BaseRowsetBuilder { @@ -33,6 +37,24 @@ class CloudRowsetBuilder : public BaseRowsetBuilder { Status init() override; + Status commit_txn() override; + + static Status validate_partial_rowset_meta(const RowsetMetaPB& base_meta, + const RowsetMetaPB& partial_meta, + int32_t segment_start_id, int32_t segment_capacity); + // Inputs have already passed validate_partial_rowset_meta at the stream boundary. + static Status assemble_rowset_meta_from_partials( + const RowsetMetaPB& base_meta, + const std::map& partial_rowset_metas, + int32_t max_segments_per_rowset, RowsetMetaPB* result); + + Status build_rowset_from_assembled_meta(const RowsetMetaPB& meta); + Status get_mow_snapshot_for_sink(PCloudLoadMowSnapshot* snapshot); + Status merge_sink_mow_bitmap(const PCloudLoadMowResult& result); + + static Status validate_sink_mow_result(const PCloudLoadMowResult& result, + int64_t snapshot_version); + virtual void update_tablet_stats(); const RowsetMetaSharedPtr& rowset_meta(); @@ -54,6 +76,7 @@ class CloudRowsetBuilder : public BaseRowsetBuilder { Status check_tablet_version_count(); CloudStorageEngine& _engine; + std::unique_ptr _mow_snapshot_for_sink; // whether to skip writing rowset metadata to meta service. // This is used for empty rowset when config::skip_writing_empty_rowset_metadata is true. diff --git a/be/src/cloud/cloud_rowset_writer.cpp b/be/src/cloud/cloud_rowset_writer.cpp index 9753ca3dd99cd9..313ccf3eb3528b 100644 --- a/be/src/cloud/cloud_rowset_writer.cpp +++ b/be/src/cloud/cloud_rowset_writer.cpp @@ -20,8 +20,7 @@ #include "common/logging.h" #include "common/status.h" #include "io/cache/block_file_cache_factory.h" -#include "io/fs/packed_file_manager.h" -#include "io/fs/packed_file_writer.h" +#include "io/fs/file_writer.h" #include "storage/rowset/rowset_factory.h" namespace doris { @@ -103,8 +102,25 @@ Status CloudRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool check RETURN_IF_ERROR(BaseBetaRowsetWriter::_build_rowset_meta(rowset_meta, check_segment_num, completed_segment_ids)); - // Collect packed file segment index information for interim rowsets as well. - return _collect_all_packed_slice_locations(rowset_meta); + // Temporary bitmap rowsets collect their own segment after its file is closed. + // The complete collections are safe to traverse only after all flushes finish. + if (completed_segment_ids == nullptr) { + return _collect_all_packed_slice_locations(rowset_meta); + } + return Status::OK(); +} + +Status CloudRowsetWriter::build_from_assembled_meta(const RowsetMetaPB& meta, + RowsetSharedPtr& rowset) { + DORIS_CHECK(_seg_files.get_file_writers().empty()); + if (!_rowset_meta->init_from_pb(meta)) { + return Status::InvalidArgument("invalid assembled rowset metadata for tablet {}", + _context.tablet_id); + } + RETURN_IF_ERROR(RowsetFactory::create_rowset(_rowset_meta->tablet_schema(), + _context.tablet_path, _rowset_meta, &rowset)); + _already_built = true; + return Status::OK(); } Status CloudRowsetWriter::build(RowsetSharedPtr& rowset) { @@ -116,8 +132,6 @@ Status CloudRowsetWriter::build(RowsetSharedPtr& rowset) { // TODO(plat1ko): check_segment_footer RETURN_IF_ERROR(_build_rowset_meta(_rowset_meta.get())); - // At this point all writers have been closed, so collecting packed file indices is safe. - RETURN_IF_ERROR(_collect_all_packed_slice_locations(_rowset_meta.get())); // If the current load is a partial update, new segments may be appended to the tmp rowset after the tmp rowset // has been committed if conflicts occur due to concurrent partial updates. However, when the recycler do recycling, // it will generate the paths for the segments to be recycled on the object storage based on the number of segments @@ -187,8 +201,7 @@ Status CloudRowsetWriter::_collect_all_packed_slice_locations(RowsetMeta* rowset const auto& file_writers = _seg_files.get_file_writers(); for (const auto& [seg_id, writer_ptr] : file_writers) { auto segment_path = _context.segment_path(seg_id); - RETURN_IF_ERROR( - _collect_packed_slice_location(writer_ptr.get(), segment_path, rowset_meta)); + RETURN_IF_ERROR(rowset_meta->collect_packed_slice_location(*writer_ptr, segment_path)); } // Collect inverted index file packed indices @@ -200,43 +213,12 @@ Status CloudRowsetWriter::_collect_all_packed_slice_locations(RowsetMeta* rowset InvertedIndexDescriptor::get_index_file_path_prefix(segment_path); std::string index_path = InvertedIndexDescriptor::get_index_file_path_v2(std::string(index_prefix_view)); - RETURN_IF_ERROR(_collect_packed_slice_location(idx_writer_ptr->get_file_writer(), - index_path, rowset_meta)); + RETURN_IF_ERROR(rowset_meta->collect_packed_slice_location( + *idx_writer_ptr->get_file_writer(), index_path)); } } return Status::OK(); } -Status CloudRowsetWriter::_collect_packed_slice_location(io::FileWriter* file_writer, - const std::string& file_path, - RowsetMeta* rowset_meta) { - VLOG_NOTICE << "collect packed slice location for file: " << file_path; - // Check if file writer is closed - if (file_writer->state() != io::FileWriter::State::CLOSED) { - // Writer is still open; index will be collected after it is closed. - return Status::OK(); - } - - // Check if file is actually in packed file (not direct write for large files) - if (!file_writer->is_in_packed_file()) { - return Status::OK(); - } - - // Ask the writer, which holds a reference to its own slice location. Looking it up by - // path in PackedFileManager would race with the retention based cleanup of the index. - io::PackedSliceLocation index; - RETURN_IF_ERROR( - static_cast(file_writer)->get_packed_slice_location(&index)); - if (index.packed_file_path.empty()) { - return Status::OK(); // File not in packed file, skip - } - - rowset_meta->add_packed_slice_location(file_path, index.packed_file_path, index.offset, - index.size, index.packed_file_size); - LOG(INFO) << "collect packed file index: " << file_path << " -> " << index.packed_file_path - << ", offset: " << index.offset << ", size: " << index.size; - return Status::OK(); -} - } // namespace doris diff --git a/be/src/cloud/cloud_rowset_writer.h b/be/src/cloud/cloud_rowset_writer.h index 4841fbf6cf7132..1e8a4fa7a8a02a 100644 --- a/be/src/cloud/cloud_rowset_writer.h +++ b/be/src/cloud/cloud_rowset_writer.h @@ -32,14 +32,14 @@ class CloudRowsetWriter : public BaseBetaRowsetWriter { Status build(RowsetSharedPtr& rowset) override; + Status build_from_assembled_meta(const RowsetMetaPB& meta, RowsetSharedPtr& rowset); + private: Status _build_rowset_meta(RowsetMeta* rowset_meta, bool check_segment_num = false, std::vector* completed_segment_ids = nullptr) override; Status _collect_all_packed_slice_locations(RowsetMeta* rowset_meta); - Status _collect_packed_slice_location(io::FileWriter* file_writer, const std::string& file_path, - RowsetMeta* rowset_meta); CloudStorageEngine& _engine; }; diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index bbfc8f5c1569a5..e1a0c43480eee8 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -684,11 +684,11 @@ std::vector CloudWarmUpManager::get_replica_info(int64_t tablet_ } void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, - int64_t sync_wait_timeout_ms) { + int64_t sync_wait_timeout_ms, bool warm_up_local) { if (sync_wait_timeout_ms <= 0) { auto rs_meta_pb = std::make_shared(rs_meta.get_rowset_pb()); - auto st = _thread_pool_token->submit_func([this, rs_meta_pb, table_id, - sync_wait_timeout_ms]() { + auto st = _thread_pool_token->submit_func([this, rs_meta_pb, table_id, sync_wait_timeout_ms, + warm_up_local]() { RowsetMeta async_rs_meta; bool init_succeed = async_rs_meta.init_from_pb(*rs_meta_pb); TEST_SYNC_POINT_CALLBACK("CloudWarmUpManager::warm_up_rowset.async_init_from_pb", @@ -697,7 +697,7 @@ void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, LOG(WARNING) << "Failed to init rowset meta when warming up rowset asynchronously"; return; } - _warm_up_rowset(async_rs_meta, table_id, sync_wait_timeout_ms); + _warm_up_rowset(async_rs_meta, table_id, sync_wait_timeout_ms, warm_up_local); }); if (!st.ok()) { LOG(WARNING) << "Failed to submit warm up rowset task: " << st; @@ -711,7 +711,7 @@ void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, bool finished = false; std::unique_lock lock(mu); auto st = _thread_pool_token->submit_func([&, this]() { - _warm_up_rowset(rs_meta, table_id, sync_wait_timeout_ms); + _warm_up_rowset(rs_meta, table_id, sync_wait_timeout_ms, warm_up_local); std::unique_lock l(mu); finished = true; cv.notify_one(); @@ -728,18 +728,28 @@ void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, } void CloudWarmUpManager::_warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, - int64_t sync_wait_timeout_ms) { + int64_t sync_wait_timeout_ms, bool warm_up_local) { TEST_SYNC_POINT_CALLBACK("CloudWarmUpManager::_warm_up_rowset.enter", &rs_meta, &sync_wait_timeout_ms); bool cache_hit = false; - auto replicas = get_replica_info(rs_meta.tablet_id(), table_id, false, cache_hit); + std::vector replicas; + if (warm_up_local) { + TReplicaInfo replica; + replica.__set_backend_id(ExecEnv::GetInstance()->cluster_info()->backend_id); + replica.__set_host(BackendOptions::get_localhost()); + replica.__set_brpc_port(config::brpc_port); + replicas.push_back(JobReplicaInfo {0, std::move(replica)}); + } else { + replicas = get_replica_info(rs_meta.tablet_id(), table_id, false, cache_hit); + } if (replicas.empty()) { VLOG_DEBUG << "There is no need to warmup tablet=" << rs_meta.tablet_id() << ", skipping rowset=" << rs_meta.rowset_id().to_string(); g_file_cache_event_driven_warm_up_skipped_rowset_num << 1; return; } - Status st = _do_warm_up_rowset(rs_meta, table_id, replicas, sync_wait_timeout_ms, !cache_hit); + Status st = _do_warm_up_rowset(rs_meta, table_id, replicas, sync_wait_timeout_ms, + !warm_up_local && !cache_hit); if (cache_hit && !st.ok() && st.is()) { replicas = get_replica_info(rs_meta.tablet_id(), table_id, true, cache_hit); st = _do_warm_up_rowset(rs_meta, table_id, replicas, sync_wait_timeout_ms, true); @@ -780,6 +790,9 @@ Status CloudWarmUpManager::_do_warm_up_rowset(RowsetMeta& rs_meta, int64_t table std::vector& replicas, int64_t sync_wait_timeout_ms, bool skip_existence_check) { + TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudWarmUpManager::_do_warm_up_rowset", Status::OK(), + &rs_meta, &replicas, &sync_wait_timeout_ms, + &skip_existence_check); auto tablet_id = rs_meta.tablet_id(); int64_t now_ts = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) diff --git a/be/src/cloud/cloud_warm_up_manager.h b/be/src/cloud/cloud_warm_up_manager.h index ba6e408da5ac81..b1f7ce1aaddf41 100644 --- a/be/src/cloud/cloud_warm_up_manager.h +++ b/be/src/cloud/cloud_warm_up_manager.h @@ -126,7 +126,10 @@ class CloudWarmUpManager { // @param rs_meta Metadata of the rowset to be warmed up. // @param sync_wait_timeout_ms Timeout in milliseconds to wait for the warm-up // to complete. Non-positive value means no waiting. - void warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, int64_t sync_wait_timeout_ms = -1); + // @param warm_up_local Only warms this BE's cache if true. Used when memtable-on-sink + // is enabled and the sink BE uploads files directly to object storage. + void warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, int64_t sync_wait_timeout_ms = -1, + bool warm_up_local = false); void recycle_cache(int64_t tablet_id, const std::vector& rowsets); @@ -183,7 +186,8 @@ class CloudWarmUpManager { std::vector get_replica_info(int64_t tablet_id, int64_t table_id, bool bypass_cache, bool& cache_hit); - void _warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, int64_t sync_wait_timeout_ms); + void _warm_up_rowset(RowsetMeta& rs_meta, int64_t table_id, int64_t sync_wait_timeout_ms, + bool warm_up_local = false); void _recycle_cache(int64_t tablet_id, const std::vector& rowsets); void submit_download_tasks(io::Path path, int64_t file_size, io::FileSystemSPtr file_system, diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index 5fa5c4c2932294..cc1d359d02df24 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -1305,7 +1305,7 @@ Status PipelineFragmentContext::_create_data_sink(ObjectPool* pool, const TDataS int child_node_id = pipeline->operators().back()->node_id(); if (state->query_options().enable_memtable_on_sink_node && !_has_inverted_index_v1_or_partial_update(thrift_sink.olap_table_sink) && - !_has_row_binlog(thrift_sink.olap_table_sink) && !config::is_cloud_mode()) { + !_has_row_binlog(thrift_sink.olap_table_sink)) { _sink = std::make_shared( pool, next_sink_operator_id(), child_node_id + 1, row_desc, output_exprs); } else { diff --git a/be/src/exec/sink/load_stream_map_pool.cpp b/be/src/exec/sink/load_stream_map_pool.cpp index 6d87fc19840c14..b4a68e30af50e5 100644 --- a/be/src/exec/sink/load_stream_map_pool.cpp +++ b/be/src/exec/sink/load_stream_map_pool.cpp @@ -99,6 +99,15 @@ void LoadStreamMap::save_tablets_to_commit(int64_t dst_id, } } +void LoadStreamMap::save_segments_for_tablet( + const std::unordered_map& segments_for_tablet) { + std::lock_guard lock(_tablets_to_commit_mutex); + // Unshared DeltaWriters on the same BE each report their own segment count. + for (const auto& [tablet_id, count] : segments_for_tablet) { + _segments_for_tablet[tablet_id] += count; + } +} + bool LoadStreamMap::release() { int num_use = --_use_cnt; if (num_use == 0) { diff --git a/be/src/exec/sink/load_stream_map_pool.h b/be/src/exec/sink/load_stream_map_pool.h index 131d1d1d88b2b7..8679b496a3400a 100644 --- a/be/src/exec/sink/load_stream_map_pool.h +++ b/be/src/exec/sink/load_stream_map_pool.h @@ -85,9 +85,7 @@ class LoadStreamMap { void save_tablets_to_commit(int64_t dst_id, const std::vector& tablets_to_commit); - void save_segments_for_tablet(const std::unordered_map& segments_for_tablet) { - _segments_for_tablet.insert(segments_for_tablet.cbegin(), segments_for_tablet.cend()); - } + void save_segments_for_tablet(const std::unordered_map& segments_for_tablet); // Return true if the last instance is just released. bool release(); diff --git a/be/src/exec/sink/load_stream_stub.cpp b/be/src/exec/sink/load_stream_stub.cpp index e46eec24333868..0a1c5923b091df 100644 --- a/be/src/exec/sink/load_stream_stub.cpp +++ b/be/src/exec/sink/load_stream_stub.cpp @@ -123,6 +123,12 @@ int LoadStreamReplyHandler::on_received_messages(brpc::StreamId id, butil::IOBuf << status; } } + if (response.has_write_context()) { + auto writer_id = response.write_context().writer_id(); + std::lock_guard lock(stub->_write_context_mutex); + stub->_write_context_responses[writer_id] = std::move(response); + stub->_write_context_cv.notify_all(); + } } return 0; } @@ -170,7 +176,8 @@ Status LoadStreamStub::open(BrpcClientCache* client_cache, const NodeInfo& node_info, int64_t txn_id, const OlapTableSchemaParam& schema, const std::vector& tablets_for_schema, int total_streams, - int64_t idle_timeout_ms, bool enable_profile) { + int64_t idle_timeout_ms, bool enable_profile, int64_t txn_expiration, + const std::string& storage_vault_id, bool write_file_cache) { std::unique_lock lock(_open_mutex); if (_is_init.load()) { return _status; @@ -194,6 +201,9 @@ Status LoadStreamStub::open(BrpcClientCache* client_cache, request.set_src_id(_src_id); request.set_txn_id(txn_id); request.set_enable_profile(enable_profile); + request.set_txn_expiration(txn_expiration); + request.set_storage_vault_id(storage_vault_id); + request.set_write_file_cache(write_file_cache); if (_is_incremental) { request.set_total_streams(0); } else if (total_streams > 0) { @@ -304,6 +314,58 @@ Status LoadStreamStub::close_load(const std::vector& tablets_to_commi return Status::OK(); } +Status LoadStreamStub::register_sink_upload_writer(int64_t partition_id, int64_t index_id, + int64_t tablet_id, const std::string& writer_id, + PCloudLoadWriteContext* context) { + PStreamHeader header; + *header.mutable_load_id() = _load_id; + header.set_src_id(_src_id); + header.set_partition_id(partition_id); + header.set_index_id(index_id); + header.set_tablet_id(tablet_id); + header.set_writer_id(writer_id); + header.set_opcode(PStreamHeader::GET_WRITE_CONTEXT); + RETURN_IF_ERROR(_encode_and_send(header)); + MonotonicStopWatch watch; + watch.start(); + std::unique_lock lock(_write_context_mutex); + while (!_write_context_responses.contains(writer_id)) { + RETURN_IF_ERROR(check_cancel()); + if (_is_closed.load()) { + return Status::InternalError("stream closed while getting write context for tablet {}", + tablet_id); + } + if (watch.elapsed_time() / 1000000 >= config::open_load_stream_timeout_ms) { + return Status::TimedOut("getting write context for tablet {}", tablet_id); + } + _write_context_cv.wait_for(lock, 100000); + } + auto response = std::move(_write_context_responses.at(writer_id)); + _write_context_responses.erase(writer_id); + RETURN_IF_ERROR(Status::create(response.status())); + *context = std::move(*response.mutable_write_context()); + return Status::OK(); +} + +Status LoadStreamStub::add_partial_rowset(int64_t partition_id, int64_t index_id, int64_t tablet_id, + const std::string& writer_id, const RowsetMetaPB& meta, + const PCloudLoadMowResult* mow_result) { + RETURN_IF_ERROR(check_cancel()); + PStreamHeader header; + *header.mutable_load_id() = _load_id; + header.set_src_id(_src_id); + header.set_partition_id(partition_id); + header.set_index_id(index_id); + header.set_tablet_id(tablet_id); + header.set_writer_id(writer_id); + header.set_opcode(PStreamHeader::ADD_PARTIAL_ROWSET); + *header.mutable_partial_rowset_meta() = meta; + if (mow_result != nullptr) { + *header.mutable_mow_result() = *mow_result; + } + return _encode_and_send(header); +} + // GET_SCHEMA Status LoadStreamStub::get_schema(const std::vector& tablets) { if (!_is_open.load()) { @@ -417,9 +479,10 @@ Status LoadStreamStub::_encode_and_send(PStreamHeader& header, std::span* client_cache const NodeInfo& node_info, int64_t txn_id, const OlapTableSchemaParam& schema, const std::vector& tablets_for_schema, int total_streams, - int64_t idle_timeout_ms, bool enable_profile) { + int64_t idle_timeout_ms, bool enable_profile, int64_t txn_expiration, + const std::string& storage_vault_id, bool write_file_cache) { bool get_schema = true; auto status = Status::OK(); bool first_stream = true; @@ -599,10 +665,12 @@ Status LoadStreamStubs::open(BrpcClientCache* client_cache Status st; if (get_schema) { st = stream->open(client_cache, node_info, txn_id, schema, tablets_for_schema, - total_streams, idle_timeout_ms, enable_profile); + total_streams, idle_timeout_ms, enable_profile, txn_expiration, + storage_vault_id, write_file_cache); } else { st = stream->open(client_cache, node_info, txn_id, schema, {}, total_streams, - idle_timeout_ms, enable_profile); + idle_timeout_ms, enable_profile, txn_expiration, storage_vault_id, + write_file_cache); } // Simulate one stream open failure within LoadStreamStubs. // This causes the successfully opened streams to be cancelled, diff --git a/be/src/exec/sink/load_stream_stub.h b/be/src/exec/sink/load_stream_stub.h index 8951c483220c01..ca75f3551ba2bf 100644 --- a/be/src/exec/sink/load_stream_stub.h +++ b/be/src/exec/sink/load_stream_stub.h @@ -144,7 +144,8 @@ class LoadStreamStub : public std::enable_shared_from_this { Status open(BrpcClientCache* client_cache, const NodeInfo& node_info, int64_t txn_id, const OlapTableSchemaParam& schema, const std::vector& tablets_for_schema, int total_streams, - int64_t idle_timeout_ms, bool enable_profile); + int64_t idle_timeout_ms, bool enable_profile, int64_t txn_expiration, + const std::string& storage_vault_id, bool write_file_cache); // for mock this class in UT #ifdef BE_TEST @@ -176,6 +177,13 @@ class LoadStreamStub : public std::enable_shared_from_this { // cancel the stream, abort close_wait, mark _is_closed and _is_cancelled void cancel(Status reason); + Status register_sink_upload_writer(int64_t partition_id, int64_t index_id, int64_t tablet_id, + const std::string& writer_id, + PCloudLoadWriteContext* context); + Status add_partial_rowset(int64_t partition_id, int64_t index_id, int64_t tablet_id, + const std::string& writer_id, const RowsetMetaPB& meta, + const PCloudLoadMowResult* mow_result = nullptr); + Status wait_for_schema(int64_t partition_id, int64_t index_id, int64_t tablet_id, int64_t timeout_ms = 60000); @@ -293,6 +301,10 @@ class LoadStreamStub : public std::enable_shared_from_this { std::mutex _send_mutex; butil::IOBuf _buffer; + bthread::Mutex _write_context_mutex; + bthread::ConditionVariable _write_context_cv; + std::unordered_map _write_context_responses; + bthread::Mutex _schema_mutex; bthread::ConditionVariable _schema_cv; std::shared_ptr _tablet_schema_for_index; @@ -331,7 +343,8 @@ class LoadStreamStubs { Status open(BrpcClientCache* client_cache, const NodeInfo& node_info, int64_t txn_id, const OlapTableSchemaParam& schema, const std::vector& tablets_for_schema, int total_streams, - int64_t idle_timeout_ms, bool enable_profile); + int64_t idle_timeout_ms, bool enable_profile, int64_t txn_expiration, + const std::string& storage_vault_id, bool write_file_cache); bool is_incremental() const { return _is_incremental; } diff --git a/be/src/exec/sink/writer/vtablet_writer_v2.cpp b/be/src/exec/sink/writer/vtablet_writer_v2.cpp index 8a5fe58500a1d9..ffdf71daab3328 100644 --- a/be/src/exec/sink/writer/vtablet_writer_v2.cpp +++ b/be/src/exec/sink/writer/vtablet_writer_v2.cpp @@ -149,9 +149,18 @@ Status VTabletWriterV2::_init(RuntimeState* state, RuntimeProfile* profile) { _load_id.set_lo(table_sink.load_id.lo); signal::set_signal_task_id(_load_id); _txn_id = table_sink.txn_id; + if (config::is_cloud_mode()) { + if (!table_sink.__isset.txn_timeout_s || table_sink.txn_timeout_s <= 0) { + return Status::InternalError("The txn_timeout_s of TDataSink is invalid"); + } + _txn_expiration = UnixSeconds() + table_sink.txn_timeout_s; + } _num_replicas = table_sink.num_replicas; _tuple_desc_id = table_sink.tuple_id; _write_file_cache = table_sink.write_file_cache; + if (table_sink.__isset.storage_vault_id) { + _storage_vault_id = table_sink.storage_vault_id; + } _schema.reset(new OlapTableSchemaParam()); RETURN_IF_ERROR(_schema->init(table_sink.schema)); _schema->set_timestamp_ms(state->timestamp_ms()); @@ -318,7 +327,8 @@ Status VTabletWriterV2::_open_streams_to_backend(int64_t dst_id, LoadStreamStubs { tablets_for_schema.clear(); }); auto st = streams.open(_state->exec_env()->brpc_streaming_client_cache(), *node_info, _txn_id, *_schema, tablets_for_schema, _total_streams, idle_timeout_ms, - _state->enable_profile()); + _state->enable_profile(), _txn_expiration, _storage_vault_id, + _write_file_cache); if (!st.ok()) { LOG(WARNING) << "failed to open stream to backend " << dst_id << ", load_id=" << print_id(_load_id) << ", err=" << st; @@ -563,6 +573,8 @@ Status VTabletWriterV2::_write_memtable(std::shared_ptr block, int64_t ta .write_file_cache = _write_file_cache, .storage_vault_id {}, .enable_table_memtable_backpressure = _tablet_finder->is_adaptive_random_bucket(), + .cloud_sink_upload = config::is_cloud_mode() && + _state->query_options().enable_cloud_memtable_sink_upload, }; bool index_not_found = true; for (const auto& index : _schema->indexes()) { @@ -791,6 +803,9 @@ Status VTabletWriterV2::close(Status exec_status) { _row_distribution.output_profile_info(_operator_profile); } + // Keep the fragment alive so tests can fetch the completed writer profile over Thrift. + DBUG_EXECUTE_IF("VTabletWriterV2.close.profile_ready", DBUG_BLOCK); + LOG(INFO) << "finished to close olap table sink. load_id=" << print_id(_load_id) << ", txn_id=" << _txn_id; } else { diff --git a/be/src/exec/sink/writer/vtablet_writer_v2.h b/be/src/exec/sink/writer/vtablet_writer_v2.h index a3d86a613bbde1..7c6269e9e8726f 100644 --- a/be/src/exec/sink/writer/vtablet_writer_v2.h +++ b/be/src/exec/sink/writer/vtablet_writer_v2.h @@ -179,6 +179,7 @@ class VTabletWriterV2 final : public AsyncResultWriter { // unique load id PUniqueId _load_id; int64_t _txn_id = -1; + int64_t _txn_expiration = 0; int _num_replicas = -1; int _tuple_desc_id = -1; @@ -197,6 +198,7 @@ class VTabletWriterV2 final : public AsyncResultWriter { int _num_local_sink = -1; bool _is_high_priority = false; bool _write_file_cache = false; + std::string _storage_vault_id; // TODO(zc): think about cache this data std::shared_ptr _schema; diff --git a/be/src/load/channel/load_stream.cpp b/be/src/load/channel/load_stream.cpp index 64c8beb0a2a073..dbaa7bec9986f8 100644 --- a/be/src/load/channel/load_stream.cpp +++ b/be/src/load/channel/load_stream.cpp @@ -26,6 +26,7 @@ #include #include "bvar/bvar.h" +#include "cloud/cloud_meta_mgr.h" #include "cloud/config.h" #include "common/signal_handler.h" #include "load/channel/load_channel.h" @@ -57,11 +58,16 @@ bvar::LatencyRecorder g_load_stream_flush_wait_ms("load_stream_flush_wait_ms"); bvar::Adder g_load_stream_flush_running_threads("load_stream_flush_wait_threads"); TabletStream::TabletStream(const PUniqueId& load_id, int64_t id, int64_t txn_id, - LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile) + LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile, + int64_t txn_expiration, std::string storage_vault_id, + bool write_file_cache) : _id(id), _next_segid(0), _load_id(load_id), _txn_id(txn_id), + _txn_expiration(txn_expiration), + _storage_vault_id(std::move(storage_vault_id)), + _write_file_cache(write_file_cache), _load_stream_mgr(load_stream_mgr) { load_stream_mgr->create_token(_flush_token); _profile = profile->create_child(fmt::format("TabletStream {}", id), true, true); @@ -77,16 +83,17 @@ inline std::ostream& operator<<(std::ostream& ostr, const TabletStream& tablet_s } Status TabletStream::init(std::shared_ptr schema, int64_t index_id, - int64_t partition_id) { + int64_t partition_id, bool is_empty) { WriteRequest req { .tablet_id = _id, .txn_id = _txn_id, + .txn_expiration = _txn_expiration, .index_id = index_id, .partition_id = partition_id, .load_id = _load_id, .table_schema_param = schema, - // TODO(plat1ko): write_file_cache - .storage_vault_id {}, + .write_file_cache = _write_file_cache, + .storage_vault_id = _storage_vault_id, }; _load_stream_writer = std::make_shared(&req, _profile); @@ -94,18 +101,41 @@ Status TabletStream::init(std::shared_ptr schema, int64_t _status.update(Status::Uninitialized("fault injection")); return _status.status(); }); - _status.update(_load_stream_writer->init()); + _status.update(_load_stream_writer->init(is_empty)); if (!_status.ok()) { LOG(INFO) << "failed to init rowset builder due to " << *this; } return _status.status(); } -Status TabletStream::append_data(const PStreamHeader& header, butil::IOBuf* data) { +Status TabletStream::append_data(const PStreamHeader& header, butil::IOBuf* data, + PCloudLoadWriteContext* context) { if (!_status.ok()) { return _status.status(); } + if (header.opcode() == PStreamHeader::GET_WRITE_CONTEXT) { + std::lock_guard lock(_lock); + if (!_segids_mapping.empty()) { + _status.update(Status::InvalidArgument("cannot switch streamed tablet to sink upload")); + return _status.status(); + } + auto st = _load_stream_writer->register_sink_upload_writer(header.writer_id(), context); + _status.update(st); + return st; + } + if (header.opcode() == PStreamHeader::ADD_PARTIAL_ROWSET) { + std::lock_guard lock(_lock); + int64_t num_added_segments = 0; + auto st = _load_stream_writer->add_partial_rowset( + header.writer_id(), header.partial_rowset_meta(), &num_added_segments, + header.has_mow_result() ? &header.mow_result() : nullptr); + // Sink uploads use this as the accepted segment count for close validation, not ID allocation. + _next_segid += cast_set(num_added_segments); + _status.update(st); + return st; + } + // dispatch add_segment request if (header.opcode() == PStreamHeader::ADD_SEGMENT) { return add_segment(header, data); @@ -119,6 +149,11 @@ Status TabletStream::append_data(const PStreamHeader& header, butil::IOBuf* data SegIdMapping* mapping = nullptr; { std::lock_guard lock_guard(_lock); + if (_load_stream_writer->is_sink_upload()) { + _status.update( + Status::InvalidArgument("cannot stream files into a sink-upload rowset")); + return _status.status(); + } if (!_segids_mapping.contains(src_id)) { _segids_mapping[src_id] = std::make_unique(); } @@ -346,10 +381,15 @@ Status TabletStream::close() { IndexStream::IndexStream(const PUniqueId& load_id, int64_t id, int64_t txn_id, std::shared_ptr schema, - LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile) + LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile, + int64_t txn_expiration, std::string storage_vault_id, + bool write_file_cache) : _id(id), _load_id(load_id), _txn_id(txn_id), + _txn_expiration(txn_expiration), + _storage_vault_id(std::move(storage_vault_id)), + _write_file_cache(write_file_cache), _schema(schema), _load_stream_mgr(load_stream_mgr) { _profile = profile->create_child(fmt::format("IndexStream {}", id), true, true); @@ -367,7 +407,8 @@ IndexStream::~IndexStream() { } } -Status IndexStream::append_data(const PStreamHeader& header, butil::IOBuf* data) { +Status IndexStream::append_data(const PStreamHeader& header, butil::IOBuf* data, + PCloudLoadWriteContext* context) { SCOPED_TIMER(_append_data_timer); int64_t tablet_id = header.tablet_id(); TabletStreamSharedPtr tablet_stream; @@ -375,21 +416,22 @@ Status IndexStream::append_data(const PStreamHeader& header, butil::IOBuf* data) std::lock_guard lock_guard(_lock); auto it = _tablet_streams_map.find(tablet_id); if (it == _tablet_streams_map.end()) { - _init_tablet_stream(tablet_stream, tablet_id, header.partition_id()); + _init_tablet_stream(tablet_stream, tablet_id, header.partition_id(), false); } else { tablet_stream = it->second; } } - return tablet_stream->append_data(header, data); + return tablet_stream->append_data(header, data, context); } void IndexStream::_init_tablet_stream(TabletStreamSharedPtr& tablet_stream, int64_t tablet_id, - int64_t partition_id) { - tablet_stream = std::make_shared(_load_id, tablet_id, _txn_id, _load_stream_mgr, - _profile); + int64_t partition_id, bool is_empty) { + tablet_stream = + std::make_shared(_load_id, tablet_id, _txn_id, _load_stream_mgr, _profile, + _txn_expiration, _storage_vault_id, _write_file_cache); _tablet_streams_map[tablet_id] = tablet_stream; - auto st = tablet_stream->init(_schema, _id, partition_id); + auto st = tablet_stream->init(_schema, _id, partition_id, is_empty); if (!st.ok()) { LOG(WARNING) << "tablet stream init failed " << *tablet_stream; } @@ -414,7 +456,8 @@ void IndexStream::close(const std::vector& tablets_to_commit, TabletStreamSharedPtr tablet_stream; auto it = _tablet_streams_map.find(tablet.tablet_id()); if (it == _tablet_streams_map.end()) { - _init_tablet_stream(tablet_stream, tablet.tablet_id(), tablet.partition_id()); + // A tablet first seen at close received no files from any sender. + _init_tablet_stream(tablet_stream, tablet.tablet_id(), tablet.partition_id(), true); } else { tablet_stream = it->second; } @@ -430,8 +473,28 @@ void IndexStream::close(const std::vector& tablets_to_commit, tablet_stream->pre_close(); } + const bool is_cloud = config::is_cloud_mode(); + std::vector close_statuses; + if (is_cloud) { + close_statuses.resize(_tablet_streams_map.size()); + std::vector> tasks; + tasks.reserve(_tablet_streams_map.size()); + size_t i = 0; + for (auto& [_, tablet_stream] : _tablet_streams_map) { + tasks.emplace_back([tablet_stream, &close_statuses, i] { + close_statuses[i] = tablet_stream->close(); + // A tablet failure must not stop the remaining tablets from closing. + return Status::OK(); + }); + ++i; + } + auto st = cloud::bthread_fork_join(tasks, 10); + DORIS_CHECK(st.ok()) << st; + } + + size_t i = 0; for (auto& [_, tablet_stream] : _tablet_streams_map) { - auto st = tablet_stream->close(); + auto st = is_cloud ? std::move(close_statuses[i++]) : tablet_stream->close(); if (st.ok()) { success_tablet_ids->push_back(tablet_stream->id()); } else { @@ -489,7 +552,9 @@ Status LoadStream::init(const POpenLoadStreamRequest* request) { RETURN_IF_ERROR(_schema->init(request->schema())); for (auto& index : request->schema().indexes()) { _index_streams_map[index.id()] = std::make_shared( - _load_id, index.id(), _txn_id, _schema, _load_stream_mgr, _profile.get()); + _load_id, index.id(), _txn_id, _schema, _load_stream_mgr, _profile.get(), + request->txn_expiration(), request->storage_vault_id(), + request->write_file_cache()); } LOG(INFO) << "succeed to init load stream " << *this; return Status::OK(); @@ -693,7 +758,8 @@ void LoadStream::_parse_header(butil::IOBuf* const message, PStreamHeader& hdr) VLOG_DEBUG << "header parse result: " << hdr.DebugString(); } -Status LoadStream::_append_data(const PStreamHeader& header, butil::IOBuf* data) { +Status LoadStream::_append_data(const PStreamHeader& header, butil::IOBuf* data, + PCloudLoadWriteContext* context) { SCOPED_TIMER(_append_data_timer); IndexStreamSharedPtr index_stream; @@ -707,7 +773,7 @@ Status LoadStream::_append_data(const PStreamHeader& header, butil::IOBuf* data) index_stream = it->second; } - return index_stream->append_data(header, data); + return index_stream->append_data(header, data, context); } int LoadStream::on_received_messages(StreamId id, butil::IOBuf* const messages[], size_t size) { @@ -772,6 +838,7 @@ void LoadStream::_dispatch(StreamId id, const PStreamHeader& hdr, butil::IOBuf* } switch (hdr.opcode()) { + case PStreamHeader::ADD_PARTIAL_ROWSET: case PStreamHeader::ADD_SEGMENT: { auto st = _append_data(hdr, data); if (!st.ok()) { @@ -823,6 +890,19 @@ void LoadStream::_dispatch(StreamId id, const PStreamHeader& hdr, butil::IOBuf* brpc::StreamClose(closing_id); } } break; + case PStreamHeader::GET_WRITE_CONTEXT: { + PLoadStreamResponse response; + auto* context = response.mutable_write_context(); + context->set_writer_id(hdr.writer_id()); + auto st = _append_data(hdr, data, context); + st.to_protobuf(response.mutable_status()); + butil::IOBuf buf; + buf.append(response.SerializeAsString()); + auto write_status = _write_stream(id, buf); + if (!write_status.ok()) { + LOG(WARNING) << "failed to return cloud write context: " << write_status; + } + } break; case PStreamHeader::GET_SCHEMA: { _report_schema(id, hdr); } break; diff --git a/be/src/load/channel/load_stream.h b/be/src/load/channel/load_stream.h index 4c8865e160306d..860c16fdb718de 100644 --- a/be/src/load/channel/load_stream.h +++ b/be/src/load/channel/load_stream.h @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -45,12 +46,14 @@ using FailedTablets = std::vector>; class TabletStream { public: TabletStream(const PUniqueId& load_id, int64_t id, int64_t txn_id, - LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile); + LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile, int64_t txn_expiration, + std::string storage_vault_id, bool write_file_cache); Status init(std::shared_ptr schema, int64_t index_id, - int64_t partition_id); + int64_t partition_id, bool is_empty = false); - Status append_data(const PStreamHeader& header, butil::IOBuf* data); + Status append_data(const PStreamHeader& header, butil::IOBuf* data, + PCloudLoadWriteContext* context = nullptr); Status add_segment(const PStreamHeader& header, butil::IOBuf* data); void add_num_segments(int64_t num_segments) { _num_segments += num_segments; } void disable_num_segments_check() { _check_num_segments = false; } @@ -78,6 +81,9 @@ class TabletStream { AtomicStatus _status; PUniqueId _load_id; int64_t _txn_id; + int64_t _txn_expiration; + std::string _storage_vault_id; + bool _write_file_cache; RuntimeProfile* _profile = nullptr; RuntimeProfile::Counter* _append_data_timer = nullptr; RuntimeProfile::Counter* _add_segment_timer = nullptr; @@ -91,10 +97,12 @@ class IndexStream { public: IndexStream(const PUniqueId& load_id, int64_t id, int64_t txn_id, std::shared_ptr schema, LoadStreamMgr* load_stream_mgr, - RuntimeProfile* profile); + RuntimeProfile* profile, int64_t txn_expiration, std::string storage_vault_id, + bool write_file_cache); ~IndexStream(); - Status append_data(const PStreamHeader& header, butil::IOBuf* data); + Status append_data(const PStreamHeader& header, butil::IOBuf* data, + PCloudLoadWriteContext* context = nullptr); void close(const std::vector& tablets_to_commit, std::vector* success_tablet_ids, FailedTablets* failed_tablet_ids); @@ -103,7 +111,7 @@ class IndexStream { private: void _init_tablet_stream(TabletStreamSharedPtr& tablet_stream, int64_t tablet_id, - int64_t partition_id); + int64_t partition_id, bool is_empty); private: int64_t _id; @@ -111,6 +119,9 @@ class IndexStream { bthread::Mutex _lock; PUniqueId _load_id; int64_t _txn_id; + int64_t _txn_expiration; + std::string _storage_vault_id; + bool _write_file_cache; std::shared_ptr _schema; std::unordered_map _tablet_partitions; RuntimeProfile* _profile = nullptr; @@ -159,7 +170,8 @@ class LoadStream : public brpc::StreamInputHandler { private: void _parse_header(butil::IOBuf* const message, PStreamHeader& hdr); void _dispatch(StreamId id, const PStreamHeader& hdr, butil::IOBuf* data); - Status _append_data(const PStreamHeader& header, butil::IOBuf* data); + Status _append_data(const PStreamHeader& header, butil::IOBuf* data, + PCloudLoadWriteContext* context = nullptr); void _report_result(StreamId stream, const Status& status, const std::vector& success_tablet_ids, diff --git a/be/src/load/channel/load_stream_writer.cpp b/be/src/load/channel/load_stream_writer.cpp index df67245ef375d7..9fc23202731e51 100644 --- a/be/src/load/channel/load_stream_writer.cpp +++ b/be/src/load/channel/load_stream_writer.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,8 @@ #include #include "bvar/bvar.h" +#include "cloud/cloud_rowset_builder.h" +#include "cloud/cloud_warm_up_manager.h" #include "cloud/config.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" @@ -75,9 +78,13 @@ bvar::Adder g_load_stream_file_writer_cnt("load_stream_file_writer_coun LoadStreamWriter::LoadStreamWriter(WriteRequest* context, RuntimeProfile* profile) : _req(*context), _rowset_writer(nullptr) { g_load_stream_writer_cnt << 1; - // TODO(plat1ko): CloudStorageEngine - _rowset_builder = std::make_unique( - ExecEnv::GetInstance()->storage_engine().to_local(), *context, profile); + auto& engine = ExecEnv::GetInstance()->storage_engine(); + if (config::is_cloud_mode()) { + _rowset_builder = + std::make_unique(engine.to_cloud(), *context, profile); + } else { + _rowset_builder = std::make_unique(engine.to_local(), *context, profile); + } _resource_ctx = thread_context()->resource_ctx(); // from load stream } @@ -87,18 +94,126 @@ LoadStreamWriter::~LoadStreamWriter() { g_load_stream_writer_cnt << -1; } -Status LoadStreamWriter::init() { +Status LoadStreamWriter::init(bool is_empty) { DBUG_EXECUTE_IF("LoadStreamWriter.init.failure", { return Status::InternalError("fault injection"); }); + if (config::is_cloud_mode()) { + // Set this before initialization so empty tablets also skip PREPARE_ROWSET. + static_cast(_rowset_builder.get()) + ->set_skip_writing_rowset_metadata(is_empty && + config::skip_writing_empty_rowset_metadata); + } RETURN_IF_ERROR(_rowset_builder->init()); _rowset_writer = _rowset_builder->rowset_writer(); _is_init = true; return Status::OK(); } +Status LoadStreamWriter::register_sink_upload_writer(const std::string& writer_id, + PCloudLoadWriteContext* context) { + // Control messages run synchronously in LoadStream::_dispatch, which attaches the load. + std::lock_guard lock(_lock); + const auto& ctx = _rowset_writer->context(); + if (!config::is_cloud_mode() || (ctx.tablet_schema->has_inverted_index() && + ctx.tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::V1)) { + return Status::NotSupported("sink upload requires cloud mode and no V1 inverted indexes"); + } + if (_pre_closed || writer_id.empty()) { + return Status::InvalidArgument("invalid sink writer registration for tablet {}", + _req.tablet_id); + } + if (!_sink_upload.exchange(true)) { + _max_segments_per_rowset = config::max_segment_num_per_rowset; + DORIS_CHECK_GT(_max_segments_per_rowset, 0); + } + auto it = _writer_segment_start_ids.find(writer_id); + if (it == _writer_segment_start_ids.end()) { + const int64_t start = + static_cast(_writer_segment_start_ids.size()) * _max_segments_per_rowset; + if (start + _max_segments_per_rowset > INT32_MAX) { + return Status::InvalidArgument("sink upload segment range overflow for tablet {}", + _req.tablet_id); + } + it = _writer_segment_start_ids.emplace(writer_id, static_cast(start)).first; + } + if (ctx.tablet->enable_unique_key_merge_on_write()) { + RETURN_IF_ERROR(static_cast(_rowset_builder.get()) + ->get_mow_snapshot_for_sink(context->mutable_mow_snapshot())); + } + context->set_writer_id(writer_id); + *context->mutable_rowset_meta() = _rowset_writer->rowset_meta()->get_rowset_pb(); + context->mutable_rowset_meta()->set_newest_write_timestamp(ctx.newest_write_timestamp); + context->set_segment_start_id(it->second); + context->set_segment_capacity(_max_segments_per_rowset); + context->set_encrypt_algorithm(ctx.encrypt_algorithm.value()); + context->set_file_cache_expiration_time(ctx.file_cache_expiration_time); + context->set_warm_up_file_cache(config::enable_file_cache && ctx.write_file_cache); + return Status::OK(); +} + +Status LoadStreamWriter::add_partial_rowset(const std::string& writer_id, const RowsetMetaPB& meta, + int64_t* num_added_segments, + const PCloudLoadMowResult* mow_result) { + // Control messages run synchronously in LoadStream::_dispatch, which attaches the load. + std::lock_guard lock(_lock); + *num_added_segments = 0; + auto it = _writer_segment_start_ids.find(writer_id); + if (it == _writer_segment_start_ids.end() || _pre_closed) { + return Status::InvalidArgument("unknown or closed sink writer {} for tablet {}", writer_id, + _req.tablet_id); + } + const bool is_mow = _rowset_builder->tablet()->enable_unique_key_merge_on_write(); + if (is_mow != (mow_result != nullptr)) { + return Status::InvalidArgument("sink upload MOW result does not match tablet {}", + _req.tablet_id); + } + auto previous = _partial_rowset_metas.find(it->second); + if (previous != _partial_rowset_metas.end()) { + if (!google::protobuf::util::MessageDifferencer::Equals(previous->second, meta)) { + return Status::InvalidArgument("conflicting sink writer result {}", writer_id); + } + if (is_mow && !google::protobuf::util::MessageDifferencer::Equals( + _mow_results.at(it->second), *mow_result)) { + return Status::InvalidArgument("conflicting sink MOW result {}", writer_id); + } + return Status::OK(); + } + RETURN_IF_ERROR(CloudRowsetBuilder::validate_partial_rowset_meta( + _rowset_writer->rowset_meta()->get_rowset_pb(), meta, it->second, + _max_segments_per_rowset)); + // Packed mappings name logical files in this rowset, never an uploader's absolute S3 URI. + if (!meta.packed_slice_locations().empty()) { + std::unordered_set paths; + for (auto id : meta.segment_ids()) { + auto path = _rowset_writer->context().segment_path(cast_set(id)); + paths.insert(path); + auto prefix = InvertedIndexDescriptor::get_index_file_path_prefix(path); + paths.insert(InvertedIndexDescriptor::get_index_file_path_v2(std::string(prefix))); + } + for (const auto& [path, location] : meta.packed_slice_locations()) { + if (!paths.contains(path) || location.offset() < 0 || location.size() < 0 || + location.packed_file_path().empty()) { + return Status::InvalidArgument("invalid sink upload packed slice {}", path); + } + } + } + if (is_mow) { + RETURN_IF_ERROR(static_cast(_rowset_builder.get()) + ->merge_sink_mow_bitmap(*mow_result)); + _mow_results.emplace(it->second, *mow_result); + } + _partial_rowset_metas.emplace(it->second, meta); + *num_added_segments = meta.num_segments(); + return Status::OK(); +} + Status LoadStreamWriter::append_data(uint32_t segid, uint64_t offset, butil::IOBuf buf, FileType file_type) { SCOPED_ATTACH_TASK(_resource_ctx); + DBUG_EXECUTE_IF("LoadStreamWriter.append_data.unexpected_transfer", { + return Status::InternalError("unexpected segment transfer in sink-upload load"); + }); io::FileWriter* file_writer = nullptr; auto& file_writers = file_type == FileType::SEGMENT_FILE ? _segment_file_writers : _inverted_file_writers; @@ -178,6 +293,17 @@ Status LoadStreamWriter::close_writer(uint32_t segid, FileType file_type) { return Status::Corruption("file {} closed with 0 bytes, file type is {}", file_writer->path().native(), file_type); } + if (config::is_cloud_mode()) { + // Streamed writers are owned here, outside the RowsetWriter's file collections. + // Use the logical path used at creation, not file_writer->path(), which may be an S3 URI. + auto file_path = _rowset_writer->context().segment_path(segid); + if (file_type == FileType::INVERTED_INDEX_FILE) { + auto prefix = InvertedIndexDescriptor::get_index_file_path_prefix(file_path); + file_path = InvertedIndexDescriptor::get_index_file_path_v2(std::string(prefix)); + } + return _rowset_writer->rowset_meta()->collect_packed_slice_location(*file_writer, + file_path); + } return Status::OK(); } @@ -252,7 +378,7 @@ Status LoadStreamWriter::_pre_close() { // in same partition has data loaded. // so we have to also init this LoadStreamWriter, so that it can create an empty rowset // for this tablet when being closed. - RETURN_IF_ERROR(init()); + RETURN_IF_ERROR(init(true)); } DCHECK(_is_init) @@ -305,7 +431,39 @@ Status LoadStreamWriter::_pre_close() { } } - RETURN_IF_ERROR(_rowset_builder->build_rowset()); + if (_sink_upload.load()) { + if (_writer_segment_start_ids.size() != _partial_rowset_metas.size()) { + return Status::Corruption("missing sink writer results for tablet {}", _req.tablet_id); + } + RowsetMetaPB meta; + RETURN_IF_ERROR(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + _rowset_writer->rowset_meta()->get_rowset_pb(), _partial_rowset_metas, + _max_segments_per_rowset, &meta)); + RETURN_IF_ERROR(static_cast(_rowset_builder.get()) + ->build_rowset_from_assembled_meta(meta)); + } else { + RETURN_IF_ERROR(_rowset_builder->build_rowset()); + if (config::is_cloud_mode()) { + // Forwarded files are owned here, outside CloudRowsetWriter's collections. + auto meta = _rowset_writer->rowset_meta(); + DORIS_CHECK(meta->segments_file_size().empty()); + DORIS_CHECK(meta->inverted_index_file_info().empty()); + std::vector sizes; + sizes.reserve(_segment_file_writers.size()); + for (const auto& writer : _segment_file_writers) { + sizes.push_back(writer->bytes_appended()); + } + meta->add_segments_file_size(sizes); + std::vector indexes(_inverted_file_writers.size()); + std::vector index_ptrs; + index_ptrs.reserve(indexes.size()); + for (size_t pos = 0; pos < indexes.size(); ++pos) { + indexes[pos].set_index_size(_inverted_file_writers[pos]->bytes_appended()); + index_ptrs.push_back(&indexes[pos]); + } + meta->add_inverted_index_files_info(index_ptrs); + } + } RETURN_IF_ERROR(_rowset_builder->submit_calc_delete_bitmap_task()); _pre_closed = true; return Status::OK(); @@ -317,9 +475,13 @@ Status LoadStreamWriter::close() { RETURN_IF_ERROR(_pre_close()); } RETURN_IF_ERROR(_rowset_builder->wait_calc_delete_bitmap()); - // FIXME(plat1ko): No `commit_txn` operation in cloud mode, need better abstractions - RETURN_IF_ERROR(static_cast(_rowset_builder.get())->commit_txn()); - + RETURN_IF_ERROR(_rowset_builder->commit_txn()); + if (_sink_upload.load() && config::enable_file_cache && _req.write_file_cache) { + auto& builder = static_cast(*_rowset_builder); + ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager().warm_up_rowset( + *builder.rowset_meta(), _rowset_builder->tablet_sptr()->table_id(), + /*sync_wait_timeout_ms=*/-1, /*warm_up_local=*/true); + } return Status::OK(); } diff --git a/be/src/load/channel/load_stream_writer.h b/be/src/load/channel/load_stream_writer.h index e82497ec02bfa1..2873c97b38f0c7 100644 --- a/be/src/load/channel/load_stream_writer.h +++ b/be/src/load/channel/load_stream_writer.h @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -51,7 +52,14 @@ class LoadStreamWriter { ~LoadStreamWriter(); - Status init(); + // is_empty is true for tablets first seen when closing the load. + Status init(bool is_empty = false); + Status register_sink_upload_writer(const std::string& writer_id, + PCloudLoadWriteContext* context); + Status add_partial_rowset(const std::string& writer_id, const RowsetMetaPB& meta, + int64_t* num_added_segments, + const PCloudLoadMowResult* mow_result = nullptr); + bool is_sink_upload() { return _sink_upload.load(); } Status append_data(uint32_t segid, uint64_t offset, butil::IOBuf buf, FileType file_type = FileType::SEGMENT_FILE); @@ -74,6 +82,12 @@ class LoadStreamWriter { // without lock Status _pre_close(); + std::atomic _sink_upload {false}; + int32_t _max_segments_per_rowset = 0; + std::unordered_map _writer_segment_start_ids; + // Partial results keyed by the writer's starting segment ID. + std::map _partial_rowset_metas; + std::map _mow_results; bool _is_init = false; bool _is_canceled = false; bool _pre_closed = false; diff --git a/be/src/load/delta_writer/delta_writer_context.h b/be/src/load/delta_writer/delta_writer_context.h index 06b4659c79bff5..8db1733f4ea9ff 100644 --- a/be/src/load/delta_writer/delta_writer_context.h +++ b/be/src/load/delta_writer/delta_writer_context.h @@ -56,6 +56,7 @@ struct WriteRequest { WriteRequestType write_req_type = WriteRequestType::DATA; std::string storage_vault_id; bool enable_table_memtable_backpressure = false; + bool cloud_sink_upload = false; }; struct TabletAddRowsPayload { diff --git a/be/src/load/delta_writer/delta_writer_v2.cpp b/be/src/load/delta_writer/delta_writer_v2.cpp index d5a40c43fcf8f8..5f70b1b86a7a11 100644 --- a/be/src/load/delta_writer/delta_writer_v2.cpp +++ b/be/src/load/delta_writer/delta_writer_v2.cpp @@ -28,6 +28,7 @@ #include #include +#include "cloud/cloud_storage_engine.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" #include "common/logging.h" @@ -36,20 +37,20 @@ #include "exec/sink/load_stream_stub.h" #include "io/fs/file_writer.h" // IWYU pragma: keep #include "load/memtable/memtable_memory_limiter.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/query_context.h" #include "service/backend_options.h" #include "storage/data_dir.h" #include "storage/index/inverted/inverted_index_desc.h" #include "storage/olap_define.h" -#include "storage/rowset/beta_rowset.h" #include "storage/rowset/beta_rowset_writer_v2.h" +#include "storage/rowset/rowset_factory.h" #include "storage/rowset/rowset_meta.h" #include "storage/rowset/rowset_writer.h" #include "storage/rowset/rowset_writer_context.h" #include "storage/schema.h" #include "storage/schema_change/schema_change.h" -#include "storage/segment/segment.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet_manager.h" #include "storage/tablet/tablet_schema.h" @@ -133,14 +134,20 @@ Status DeltaWriterV2::init() { context.memtable_on_sink_support_index_v2 = true; context.encrypt_algorithm = EncryptionAlgorithmPB::PLAINTEXT; - _rowset_writer = std::make_shared(_streams); - RETURN_IF_ERROR(_rowset_writer->init(context)); + if (_req.cloud_sink_upload) { + RETURN_IF_ERROR(_init_sink_upload_writer(context)); + } else { + _rowset_writer = std::make_shared(_streams); + RETURN_IF_ERROR(_rowset_writer->init(context)); + } RETURN_IF_ERROR(_memtable_writer->init(_rowset_writer, _tablet_schema, _partial_update_info, _workload_group, _streams[0]->enable_unique_mow(_req.index_id))); ExecEnv::GetInstance()->memtable_memory_limiter()->register_writer(_memtable_writer); _is_init = true; - _streams.clear(); + if (!_req.cloud_sink_upload) { + _streams.clear(); + } return Status::OK(); } @@ -198,6 +205,87 @@ Status DeltaWriterV2::close() { return _memtable_writer->close(); } +Status DeltaWriterV2::_init_sink_upload_writer(RowsetWriterContext& context) { + DORIS_CHECK(config::is_cloud_mode()); + DORIS_CHECK_EQ(_streams.size(), 1); + // The local id identifies this writer, including unshared writers on the same BE. + _sink_writer_id = context.rowset_id.to_string(); + PCloudLoadWriteContext target_write_context; + RETURN_IF_ERROR(_streams[0]->register_sink_upload_writer(_req.partition_id, _req.index_id, + _req.tablet_id, _sink_writer_id, + &target_write_context)); + if (context.enable_unique_key_merge_on_write) { + if (!target_write_context.has_mow_snapshot() || + !target_write_context.mow_snapshot().has_version() || + !target_write_context.mow_snapshot().has_delete_bitmap()) { + return Status::NotSupported("target BE does not support MOW sink upload"); + } + RETURN_IF_ERROR( + _init_mow_context_from_snapshot(context, target_write_context.mow_snapshot())); + } + const auto& meta = target_write_context.rowset_meta(); + context.rowset_id.init(meta.rowset_id_v2()); + context.tablet_schema_hash = meta.tablet_schema_hash(); + context.txn_expiration = meta.txn_expiration(); + context.newest_write_timestamp = meta.newest_write_timestamp(); + context.encrypt_algorithm = target_write_context.encrypt_algorithm(); + context.file_cache_expiration_time = target_write_context.file_cache_expiration_time(); + context.write_file_cache = _req.write_file_cache; + // Keep upload-time caching only when this sink is also the target BE. Older targets + // do not advertise warmup, so retain their existing sink-side cache behavior. + // Stream Load may leave the fragment's source backend id unset (-1). + context.disable_file_cache = + target_write_context.warm_up_file_cache() && + ExecEnv::GetInstance()->cluster_info()->backend_id != _streams[0]->dst_id(); + if (meta.has_inverted_index_storage_format()) { + context.inverted_index_storage_format = meta.inverted_index_storage_format(); + } + context.persist_inverted_index_storage_format = meta.has_inverted_index_storage_format(); + auto& engine = static_cast(ExecEnv::GetInstance()->storage_engine()); + context.storage_resource = engine.get_storage_resource(meta.resource_id()); + if (!context.storage_resource) { + return Status::InternalError("sink upload storage resource {} unavailable", + meta.resource_id()); + } + _segment_start_id = target_write_context.segment_start_id(); + context.is_partial_output_writer = true; + context.enable_segcompaction = false; + context.memtable_on_sink_support_index_v2 = false; + // Preserve whole-rowset first-segment packing, not first-segment-per-writer packing. + context.allow_packed_file = _segment_start_id == 0; + _rowset_writer = DORIS_TRY(RowsetFactory::create_rowset_writer(engine, context, false)); + _rowset_writer->set_segment_start_id(_segment_start_id, + target_write_context.segment_capacity()); + _req.txn_expiration = context.txn_expiration; + return Status::OK(); +} + +Status DeltaWriterV2::_init_mow_context_from_snapshot(RowsetWriterContext& context, + const PCloudLoadMowSnapshot& snapshot) { + auto& engine = static_cast(ExecEnv::GetInstance()->storage_engine()); + context.tablet = DORIS_TRY(engine.get_tablet(_req.tablet_id)); + auto ids = std::make_shared(); + std::vector rowsets; + rowsets.reserve(snapshot.rowsets_size()); + for (const auto& meta : snapshot.rowsets()) { + auto rowset_meta = std::make_shared(); + if (!rowset_meta->init_from_pb(meta)) { + return Status::InvalidArgument("invalid sink MOW snapshot rowset"); + } + RowsetSharedPtr rowset; + RETURN_IF_ERROR(RowsetFactory::create_rowset(rowset_meta->tablet_schema(), "", rowset_meta, + &rowset)); + ids->insert(rowset->rowset_id()); + rowsets.push_back(std::move(rowset)); + } + context.mow_context = std::make_shared( + snapshot.version(), _req.txn_id, std::move(ids), std::move(rowsets), + std::make_shared(_req.tablet_id)); + context.mow_context->snapshot_delete_bitmap = std::make_shared( + DeleteBitmap::from_pb(snapshot.delete_bitmap(), _req.tablet_id)); + return Status::OK(); +} + Status DeltaWriterV2::close_wait(int32_t& num_segments, RuntimeProfile* profile) { SCOPED_RAW_TIMER(&_close_wait_time); std::lock_guard l(_lock); @@ -208,12 +296,51 @@ Status DeltaWriterV2::close_wait(int32_t& num_segments, RuntimeProfile* profile) _update_profile(profile); } RETURN_IF_ERROR(_memtable_writer->close_wait(profile)); - num_segments = _rowset_writer->next_segment_id(); + num_segments = _rowset_writer->get_allocated_segment_id() - _segment_start_id; + if (_req.cloud_sink_upload) { + RETURN_IF_ERROR(_finish_sink_upload(profile)); + } _delta_written_success = true; return Status::OK(); } +Status DeltaWriterV2::_finish_sink_upload(RuntimeProfile* profile) { + RowsetSharedPtr partial; + RETURN_IF_ERROR(_rowset_writer->build(partial)); + if (UnixSeconds() >= _req.txn_expiration) { + return Status::TimedOut("sink upload transaction {} expired", _req.txn_id); + } + DBUG_EXECUTE_IF("DeltaWriterV2.sink_upload.after_upload_failure", + { return Status::InternalError("injected failure after sink upload"); }); + PCloudLoadMowResult mow_result; + const auto& mow_context = _rowset_writer->context().mow_context; + if (mow_context != nullptr) { + // CloudRowsetWriter::build has waited for all per-segment bitmap tasks. + mow_result.set_snapshot_version(mow_context->max_version); + *mow_result.mutable_delete_bitmap() = mow_context->delete_bitmap->to_pb(); + DBUG_EXECUTE_IF("DeltaWriterV2.sink_mow.after_bitmap_failure", { + return Status::InternalError("injected failure after sink MOW bitmap calculation"); + }); + if (profile != nullptr) { + profile->add_info_string("CloudMemtableMowBitmap", "true"); + } + } + const auto* result = mow_context != nullptr ? &mow_result : nullptr; + RETURN_IF_ERROR(_streams[0]->add_partial_rowset( + _req.partition_id, _req.index_id, _req.tablet_id, _sink_writer_id, + partial->rowset_meta()->get_rowset_pb(), result)); + DBUG_EXECUTE_IF("DeltaWriterV2.sink_upload.duplicate_result", { + RETURN_IF_ERROR(_streams[0]->add_partial_rowset( + _req.partition_id, _req.index_id, _req.tablet_id, _sink_writer_id, + partial->rowset_meta()->get_rowset_pb(), result)); + }); + if (profile != nullptr) { + profile->add_info_string("CloudMemtableSinkUpload", "true"); + } + return Status::OK(); +} + Status DeltaWriterV2::cancel() { return cancel_with_status(Status::Cancelled("already cancelled")); } diff --git a/be/src/load/delta_writer/delta_writer_v2.h b/be/src/load/delta_writer/delta_writer_v2.h index 337426515c2796..cf7a86cef6a8ef 100644 --- a/be/src/load/delta_writer/delta_writer_v2.h +++ b/be/src/load/delta_writer/delta_writer_v2.h @@ -50,7 +50,6 @@ class StorageEngine; class TupleDescriptor; class SlotDescriptor; class OlapTableSchemaParam; -class BetaRowsetWriterV2; class LoadStreamStub; class WorkloadGroup; @@ -88,6 +87,10 @@ class DeltaWriterV2 { const OlapTableSchemaParam* table_schema_param, const TabletSchema& ori_tablet_schema); int64_t _table_id() const; + Status _init_sink_upload_writer(RowsetWriterContext& context); + Status _finish_sink_upload(RuntimeProfile* profile); + Status _init_mow_context_from_snapshot(RowsetWriterContext& context, + const PCloudLoadMowSnapshot& snapshot); void _update_profile(RuntimeProfile* profile); @@ -95,7 +98,9 @@ class DeltaWriterV2 { bool _is_cancelled = false; WriteRequest _req; std::shared_ptr _workload_group; - std::shared_ptr _rowset_writer; + std::shared_ptr _rowset_writer; + std::string _sink_writer_id; + int32_t _segment_start_id = 0; TabletSchemaSPtr _tablet_schema; bool _delta_written_success = false; diff --git a/be/src/service/http/action/stream_load.cpp b/be/src/service/http/action/stream_load.cpp index 8f47edaba8aa31..8eb1b295220857 100644 --- a/be/src/service/http/action/stream_load.cpp +++ b/be/src/service/http/action/stream_load.cpp @@ -780,6 +780,10 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req, bool value = iequal(http_req->header(HTTP_MEMTABLE_ON_SINKNODE), "true"); request.__set_memtable_on_sink_node(value); } + if (!http_req->header(HTTP_CLOUD_MEMTABLE_SINK_UPLOAD).empty()) { + bool value = iequal(http_req->header(HTTP_CLOUD_MEMTABLE_SINK_UPLOAD), "true"); + request.__set_cloud_memtable_sink_upload(value); + } if (!http_req->header(HTTP_LOAD_STREAM_PER_NODE).empty()) { int stream_per_node = DORIS_TRY( safe_stoi(http_req->header(HTTP_LOAD_STREAM_PER_NODE), HTTP_LOAD_STREAM_PER_NODE)); diff --git a/be/src/service/http/http_common.h b/be/src/service/http/http_common.h index f250e22a278e55..ba54c7bf5baad2 100644 --- a/be/src/service/http/http_common.h +++ b/be/src/service/http/http_common.h @@ -66,6 +66,7 @@ static const std::string HTTP_TWO_PHASE_COMMIT = "two_phase_commit"; static const std::string HTTP_TXN_ID_KEY = "txn_id"; static const std::string HTTP_TXN_OPERATION_KEY = "txn_operation"; static const std::string HTTP_MEMTABLE_ON_SINKNODE = "memtable_on_sink_node"; +static const std::string HTTP_CLOUD_MEMTABLE_SINK_UPLOAD = "cloud_memtable_sink_upload"; static const std::string HTTP_LOAD_STREAM_PER_NODE = "load_stream_per_node"; static const std::string HTTP_WAL_ID_KY = "wal_id"; static const std::string HTTP_AUTH_CODE = "auth_code"; // deprecated diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 0b05f8fdeb4db8..d619068fae44ca 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -470,6 +470,9 @@ struct MowContext { std::shared_ptr rowset_ids; std::vector rowset_ptrs; std::shared_ptr delete_bitmap; + // Sink-upload sinks use rowset_ptrs and this immutable target snapshot instead of + // their local tablet's potentially newer rowsets/deletion state. + std::shared_ptr snapshot_delete_bitmap; }; // used for controll compaction diff --git a/be/src/storage/rowset/beta_rowset_writer.cpp b/be/src/storage/rowset/beta_rowset_writer.cpp index 265448a831472b..bbca749fcf2542 100644 --- a/be/src/storage/rowset/beta_rowset_writer.cpp +++ b/be/src/storage/rowset/beta_rowset_writer.cpp @@ -71,19 +71,6 @@ using namespace ErrorCode; namespace { -bool is_segment_overlapping(const std::vector& segments_encoded_key_bounds) { - std::string_view last; - for (auto&& segment_encode_key : segments_encoded_key_bounds) { - auto&& cur_min = segment_encode_key.min_key(); - auto&& cur_max = segment_encode_key.max_key(); - if (cur_min <= last) { - return true; - } - last = cur_max; - } - return false; -} - bool copy_key_bounds_with_truncation(const KeyBoundsPB& src, KeyBoundsPB* dst) { DCHECK(dst != nullptr); if (config::random_segments_key_bounds_truncation) { @@ -420,19 +407,28 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { return Status::OK(); } std::vector specified_rowsets; - { + if (_context.mow_context->snapshot_delete_bitmap != nullptr) { + specified_rowsets = _context.mow_context->rowset_ptrs; + } else { std::shared_lock meta_rlock(_context.tablet->get_header_lock()); specified_rowsets = _context.tablet->get_rowset_by_ids(_context.mow_context->rowset_ids.get()); } + RowsetSharedPtr streamed_rowset; + if (_seg_files.get(segment_id) == nullptr) { + // Streamed files are closed by LoadStreamWriter. Snapshot their packed mappings on + // its serial receive thread before later file closes mutate the shared rowset meta. + RETURN_IF_ERROR(_build_tmp(streamed_rowset, segment_id)); + } + // Submit the entire delete bitmap calculation process to thread pool for async execution // This avoids blocking memtable flush thread while waiting for file upload to complete // The process includes: file_writer->close(), _build_tmp, load_segments, and calc_delete_bitmap const auto submit_time_us = MonotonicMicros(); return _calc_delete_bitmap_token->submit_func([this, segment_id, specified_rowsets = std::move(specified_rowsets), - submit_time_us]() -> Status { + submit_time_us, streamed_rowset]() -> Status { const auto queue_time_us = MonotonicMicros() - submit_time_us; Status st = Status::OK(); // Step 1: Close file_writer (must be done before load_segments) @@ -457,10 +453,9 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { OlapStopWatch watch; // Step 2: Build tmp rowset (needs file_writer to be closed) - RowsetSharedPtr rowset_ptr; - st = _build_tmp(rowset_ptr); - if (!st.ok()) { - return st; + RowsetSharedPtr rowset_ptr = streamed_rowset; + if (rowset_ptr == nullptr) { + RETURN_IF_ERROR(_build_tmp(rowset_ptr, segment_id)); } // Step 3: Load segments (needs file_writer to be closed and rowset to be built) @@ -477,7 +472,7 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { st = BaseTablet::calc_delete_bitmap(_context.tablet, rowset_ptr, segments, specified_rowsets, _context.mow_context->delete_bitmap, _context.mow_context->max_version, nullptr, nullptr, - nullptr); + _context.mow_context->snapshot_delete_bitmap); if (!st.ok()) { return st; } @@ -961,9 +956,11 @@ Status BaseBetaRowsetWriter::_close_file_writers() { Status BetaRowsetWriter::_close_file_writers() { RETURN_IF_ERROR(BaseBetaRowsetWriter::_close_file_writers()); - // if _segment_start_id is not zero, that means it's a transient rowset writer for - // MoW partial update, don't need to do segment compaction. - if (_segment_start_id == 0) { + // Skip writer-local segment compaction for: + // 1. A distributed partial output writer, whose physical segment IDs are constrained to a + // coordinator-assigned slot that may start at zero. + // 2. A transient MoW partial-update writer, which appends segments from a nonzero segment ID. + if (!_context.is_partial_output_writer && _segment_start_id == 0) { if (_segcompaction_worker->cancel()) { std::lock_guard lk(_is_doing_segcompaction_lock); _is_doing_segcompaction = false; @@ -1057,7 +1054,8 @@ Status BaseBetaRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool ch std::vector segment_ids; std::optional segments_key_bounds_truncated; const bool record_segment_ids = - _context.write_type == DataWriteType::TYPE_COMPACTION && _segment_start_id != 0; + _context.is_partial_output_writer || + (_context.write_type == DataWriteType::TYPE_COMPACTION && _segment_start_id != 0); { std::lock_guard lock(_segid_statistics_map_mutex); if (record_segment_ids) { @@ -1117,7 +1115,9 @@ Status BaseBetaRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool ch rowset_meta->set_segments_overlap(OVERLAPPING); } - auto segment_num = _num_seg(); + // A temporary rowset contains only completed segments, which may finish out of order. + auto segment_num = + completed_segment_ids != nullptr ? completed_segment_ids->size() : _num_seg(); if (check_segment_num && config::check_segment_when_build_rowset_meta) { auto segments_encoded_key_bounds_size = segments_encoded_key_bounds.size(); if (segments_encoded_key_bounds_size != segment_num) { @@ -1148,15 +1148,18 @@ Status BaseBetaRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool ch rowset_meta->set_index_disk_size(total_index_size + _total_index_size); bool aggregate_key_bounds = config::enable_aggregate_non_mow_key_bounds && !_context.enable_unique_key_merge_on_write && - !_rowset_meta->is_row_binlog(); - rowset_meta->set_segments_key_bounds(segments_encoded_key_bounds, aggregate_key_bounds); + !_rowset_meta->is_row_binlog() && + !_context.is_partial_output_writer; + rowset_meta->set_segments_key_bounds( + segments_encoded_key_bounds, aggregate_key_bounds, + /*truncate_key_bounds=*/!_context.is_partial_output_writer); // TODO write zonemap to meta rowset_meta->set_empty((num_rows_written + _num_rows_written) == 0); rowset_meta->set_creation_time(time(nullptr)); return Status::OK(); } -Status BaseBetaRowsetWriter::_build_tmp(RowsetSharedPtr& rowset_ptr) { +Status BaseBetaRowsetWriter::_build_tmp(RowsetSharedPtr& rowset_ptr, int32_t segment_id) { Status status; std::shared_ptr tmp_rs_meta = std::make_shared(); tmp_rs_meta->init(_rowset_meta.get()); @@ -1169,6 +1172,15 @@ Status BaseBetaRowsetWriter::_build_tmp(RowsetSharedPtr& rowset_ptr) { } tmp_rs_meta->set_segment_ids(completed_segment_ids); + if (_context.packed_file_active) { + // Bitmap calculation reads only this segment's primary-key index. Other flushes + // may still be creating/closing files, so do not traverse their writer collections. + if (auto* writer = _seg_files.get(segment_id); writer != nullptr) { + RETURN_IF_ERROR(tmp_rs_meta->collect_packed_slice_location( + *writer, _context.segment_path(segment_id))); + } + } + status = RowsetFactory::create_rowset(_context.tablet_schema, _context.tablet_path, tmp_rs_meta, &rowset_ptr); DBUG_EXECUTE_IF("BaseBetaRowsetWriter::_build_tmp.create_rowset_failed", @@ -1304,8 +1316,11 @@ Status BetaRowsetWriter::_check_segment_number_limit(size_t segnum) { Status BaseBetaRowsetWriter::add_segment(uint32_t segment_id, const SegmentStatistics& segstat) { uint32_t segid_offset = segment_id - _segment_start_id; bool key_bounds_truncated = false; - SegmentStatistics stored_segstat = - copy_segment_statistics_with_truncated_key_bounds(segstat, key_bounds_truncated); + SegmentStatistics stored_segstat = segstat; + if (!_context.is_partial_output_writer) { + stored_segstat = + copy_segment_statistics_with_truncated_key_bounds(segstat, key_bounds_truncated); + } { std::lock_guard lock(_segid_statistics_map_mutex); CHECK_EQ(_segid_statistics_map.find(segment_id) == _segid_statistics_map.end(), true); diff --git a/be/src/storage/rowset/beta_rowset_writer.h b/be/src/storage/rowset/beta_rowset_writer.h index 49811ac7714512..1baf6cc00b5c78 100644 --- a/be/src/storage/rowset/beta_rowset_writer.h +++ b/be/src/storage/rowset/beta_rowset_writer.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include "common/status.h" @@ -46,6 +47,21 @@ namespace doris { class Block; +// NONOVERLAPPING requires key ranges to be ordered and disjoint. +template +bool is_segment_overlapping(const KeyBoundsRange& segments_encoded_key_bounds) { + std::string_view last; + for (auto&& segment_encode_key : segments_encoded_key_bounds) { + auto&& cur_min = segment_encode_key.min_key(); + auto&& cur_max = segment_encode_key.max_key(); + if (cur_min <= last) { + return true; + } + last = cur_max; + } + return false; +} + namespace segment_v2 { class VerticalSegmentWriter; } // namespace segment_v2 @@ -224,7 +240,7 @@ class BaseBetaRowsetWriter : public RowsetWriter { virtual Status _check_segment_number_limit(size_t segnum); virtual int64_t _num_seg() const; // build a tmp rowset for load segment to calc delete_bitmap for this segment - Status _build_tmp(RowsetSharedPtr& rowset_ptr); + Status _build_tmp(RowsetSharedPtr& rowset_ptr, int32_t segment_id); uint64_t get_rowset_num_rows() { std::lock_guard l(_segid_statistics_map_mutex); diff --git a/be/src/storage/rowset/beta_rowset_writer_v2.h b/be/src/storage/rowset/beta_rowset_writer_v2.h index 8aa40ce4cfd75d..263464384df056 100644 --- a/be/src/storage/rowset/beta_rowset_writer_v2.h +++ b/be/src/storage/rowset/beta_rowset_writer_v2.h @@ -120,8 +120,6 @@ class BetaRowsetWriterV2 : public RowsetWriter { return _segment_creator.get_allocated_segment_id(); }; - int32_t next_segment_id() { return _segment_creator.next_segment_id(); }; - int64_t delete_bitmap_ns() override { return _delete_bitmap_ns; } int64_t segment_writer_ns() override { return _segment_writer_ns; } diff --git a/be/src/storage/rowset/rowset_meta.cpp b/be/src/storage/rowset/rowset_meta.cpp index 05ca4be46b7b10..4ea5aef2db94df 100644 --- a/be/src/storage/rowset/rowset_meta.cpp +++ b/be/src/storage/rowset/rowset_meta.cpp @@ -37,6 +37,7 @@ #include "io/fs/local_file_system.h" #include "io/fs/packed_file_manager.h" #include "io/fs/packed_file_system.h" +#include "io/fs/packed_file_writer.h" #include "json2pb/json_to_pb.h" #include "json2pb/pb_to_json.h" #include "runtime/exec_env.h" @@ -56,6 +57,27 @@ RowsetMeta::~RowsetMeta() { } } +Status RowsetMeta::collect_packed_slice_location(const io::FileWriter& file_writer, + const std::string& file_path) { + if (file_writer.state() != io::FileWriter::State::CLOSED) { + return Status::OK(); + } + if (!file_writer.is_in_packed_file()) { + return Status::OK(); + } + + // Read the writer-owned location so collection cannot race with global index cleanup. + io::PackedSliceLocation location; + RETURN_IF_ERROR(static_cast(file_writer) + .get_packed_slice_location(&location)); + DORIS_CHECK(!location.packed_file_path.empty()); + add_packed_slice_location(file_path, location.packed_file_path, location.offset, location.size, + location.packed_file_size); + LOG(INFO) << "collect packed file index: " << file_path << " -> " << location.packed_file_path + << ", offset: " << location.offset << ", size: " << location.size; + return Status::OK(); +} + bool RowsetMeta::init(std::string_view pb_rowset_meta) { bool ret = _deserialize_from_pb(pb_rowset_meta); if (!ret) { @@ -381,7 +403,7 @@ int64_t RowsetMeta::segment_file_size_by_pos(size_t pos) const { } void RowsetMeta::set_segments_key_bounds(const std::vector& segments_key_bounds, - bool aggregate_into_single) { + bool aggregate_into_single, bool truncate_key_bounds) { _rowset_meta_pb.clear_segments_key_bounds(); bool did_aggregate = aggregate_into_single && !segments_key_bounds.empty(); if (did_aggregate) { @@ -406,8 +428,9 @@ void RowsetMeta::set_segments_key_bounds(const std::vector& segment } set_segments_key_bounds_aggregated(did_aggregate); - int32_t truncation_threshold = config::segments_key_bounds_truncation_threshold; - if (config::random_segments_key_bounds_truncation) { + int32_t truncation_threshold = + truncate_key_bounds ? config::segments_key_bounds_truncation_threshold : 0; + if (truncate_key_bounds && config::random_segments_key_bounds_truncation) { std::mt19937 generator(std::random_device {}()); std::uniform_int_distribution distribution(-10, 40); truncation_threshold = distribution(generator); diff --git a/be/src/storage/rowset/rowset_meta.h b/be/src/storage/rowset/rowset_meta.h index 9255cdb9a09d08..17594629a91695 100644 --- a/be/src/storage/rowset/rowset_meta.h +++ b/be/src/storage/rowset/rowset_meta.h @@ -467,9 +467,11 @@ class RowsetMeta : public MetadataAdder { } // If `aggregate_into_single` is true, collapse per-segment bounds into a single - // [rowset_min, rowset_max] entry and mark this rowset as aggregated. + // [rowset_min, rowset_max] entry and mark this rowset as aggregated. Partial distributed + // compaction outputs can disable truncation so the coordinator can validate adjacent bounds. void set_segments_key_bounds(const std::vector& segments_key_bounds, - bool aggregate_into_single = false); + bool aggregate_into_single = false, + bool truncate_key_bounds = true); void add_segment_key_bounds(KeyBoundsPB segments_key_bounds) { *_rowset_meta_pb.add_segments_key_bounds() = std::move(segments_key_bounds); @@ -535,6 +537,11 @@ class RowsetMeta : public MetadataAdder { RowsetMeta(const RowsetMeta&) = delete; RowsetMeta operator=(const RowsetMeta&) = delete; + // Collect only completed files; interim rowsets may still have open writers. + // file_path is the logical path used at creation, not the S3 writer's absolute path. + Status collect_packed_slice_location(const io::FileWriter& file_writer, + const std::string& file_path); + void add_packed_slice_location(const std::string& segment_path, const std::string& packed_file_path, int64_t offset, int64_t size, int64_t packed_file_size) { diff --git a/be/src/storage/rowset/rowset_writer_context.h b/be/src/storage/rowset/rowset_writer_context.h index 7c9d4add5ce7bc..a3ed5cca4f9c22 100644 --- a/be/src/storage/rowset/rowset_writer_context.h +++ b/be/src/storage/rowset/rowset_writer_context.h @@ -129,6 +129,8 @@ struct RowsetWriterContext { /// begin file cache opts bool write_file_cache = false; + // Disable all file-cache writes. + bool disable_file_cache = false; bool is_hot_data = false; // Absolute timestamp (seconds since epoch) after which the cache blocks written by // this rowset stop being TTL protected; 0 means no TTL. Always set it from @@ -147,6 +149,9 @@ struct RowsetWriterContext { bool is_transient_rowset_writer = false; + // Preserve assigned segment IDs and skip local segment compaction for a distributed shard. + bool is_partial_output_writer = false; + segment_v2::HistoricalRowRetrieverContext make_historical_row_retriever_context(); // Intent flag: caller can actively turn merge-file feature on/off for this rowset. @@ -284,6 +289,7 @@ struct RowsetWriterContext { append_info.rowset_id = rowset_id.to_string(); append_info.first_segment_id = first_segment_id; append_info.txn_id = txn_id; + append_info.write_file_cache = !disable_file_cache; append_info.expiration_time = file_cache_expiration_time; fs = std::make_shared(fs, append_info); } @@ -306,6 +312,13 @@ struct RowsetWriterContext { .file_cache_expiration_time = file_cache_expiration_time, .approximate_bytes_to_write = approximate_bytes_to_write}; + if (disable_file_cache) { + opts.write_file_cache = false; + opts.allow_adaptive_file_cache_write = false; + opts.approximate_bytes_to_write = 0; + return opts; + } + if (config::enable_file_cache_write_index_file_only) { opts.allow_adaptive_file_cache_write = false; opts.approximate_bytes_to_write = 0; diff --git a/be/src/storage/rowset/vertical_beta_rowset_writer.cpp b/be/src/storage/rowset/vertical_beta_rowset_writer.cpp index 719b90b62be1cb..17a250c0fbbc08 100644 --- a/be/src/storage/rowset/vertical_beta_rowset_writer.cpp +++ b/be/src/storage/rowset/vertical_beta_rowset_writer.cpp @@ -205,7 +205,7 @@ Status VerticalBetaRowsetWriter::build(RowsetSharedPtr& rowset) { const int32_t next_segment_id = T::get_allocated_segment_id(); const int32_t segment_num = this->_num_segment.load(std::memory_order_relaxed); DORIS_CHECK_EQ(next_segment_id - this->_segment_start_id, segment_num); - if (this->_segment_start_id != 0) { + if (this->_context.is_partial_output_writer || this->_segment_start_id != 0) { std::vector segment_ids; segment_ids.reserve(segment_num); for (int32_t segment_id = this->_segment_start_id; segment_id < next_segment_id; diff --git a/be/src/storage/segment/segment_index_file_cache_loader.cpp b/be/src/storage/segment/segment_index_file_cache_loader.cpp index 46a1ffa9525be3..a9c39873612dba 100644 --- a/be/src/storage/segment/segment_index_file_cache_loader.cpp +++ b/be/src/storage/segment/segment_index_file_cache_loader.cpp @@ -87,7 +87,8 @@ Status read_range_to_file_cache(io::FileReaderSPtr reader, uint64_t offset, uint Status SegmentIndexFileCacheLoader::preload_segment_index_to_file_cache( const RowsetWriterContext& context, uint32_t segment_id, const std::string& segment_path, const SegmentIndexFileCacheInfo& info) { - if (!enable_cloud_index_only_file_cache() || context.is_local_rowset()) { + if (!enable_cloud_index_only_file_cache() || context.is_local_rowset() || + context.disable_file_cache) { return Status::OK(); } diff --git a/be/test/cloud/cloud_load_stream_test.cpp b/be/test/cloud/cloud_load_stream_test.cpp new file mode 100644 index 00000000000000..884249f6f7d6cc --- /dev/null +++ b/be/test/cloud/cloud_load_stream_test.cpp @@ -0,0 +1,291 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cloud/cloud_committed_rs_mgr.h" +#include "cloud/cloud_rowset_builder.h" +#include "cloud/cloud_rowset_writer.h" +#include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet.h" +#include "cloud/cloud_txn_delete_bitmap_cache.h" +#include "cloud/config.h" +#include "load/channel/load_stream_mgr.h" +#include "load/channel/load_stream_writer.h" +#include "runtime/exec_env.h" +#include "storage/rowset/rowset.h" +#include "storage/rowset/rowset_factory.h" +#include "storage/rowset/rowset_meta.h" +#include "storage/tablet/tablet_meta.h" +#include "util/time.h" +#include "util/work_thread_pool.hpp" + +namespace doris { +namespace { + +class EmptyLoadRowsetBuilder : public CloudRowsetBuilder { +public: + using CloudRowsetBuilder::CloudRowsetBuilder; + + Status init() override { + skip_metadata_at_init = _skip_writing_rowset_metadata; + return Status::InternalError("stop before preparing rowset metadata"); + } + + Status commit_rowset(const std::string&, int64_t) override { + ++commit_calls; + return Status::InternalError("injected commit failure"); + } + + bool skip_metadata_at_init = false; + int commit_calls = 0; +}; + +class ClosingLoadRowsetBuilder : public BaseRowsetBuilder { +public: + ClosingLoadRowsetBuilder(CloudStorageEngine& engine, const WriteRequest& req, + std::atomic& prepared, std::function commit) + : BaseRowsetBuilder(req, nullptr), _prepared(prepared), _commit(std::move(commit)) { + _rowset_writer = std::make_shared(engine); + _rowset_writer->_rowset_meta = std::make_shared(); + } + + Status init() override { return Status::OK(); } + Status build_rowset() override { + ++_prepared; + return Status::OK(); + } + Status submit_calc_delete_bitmap_task() override { return Status::OK(); } + Status wait_calc_delete_bitmap() override { return Status::OK(); } + Status commit_txn() override { return _commit(); } + +private: + std::atomic& _prepared; + std::function _commit; +}; + +} // namespace + +class CloudLoadStreamTest : public testing::Test { +protected: + void SetUp() override { + _old_cloud_unique_id = config::cloud_unique_id; + _old_skip_empty = config::skip_writing_empty_rowset_metadata; + _old_make_visible = config::enable_cloud_make_rs_visible_on_be; + config::cloud_unique_id = "cloud_load_stream_test"; + config::enable_cloud_make_rs_visible_on_be = true; + _old_engine = std::move(ExecEnv::GetInstance()->_storage_engine); + auto engine = std::make_unique(EngineOptions {}); + _engine = engine.get(); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + _engine->_committed_rs_mgr = std::make_unique(); + _engine->_txn_delete_bitmap_cache = std::make_unique(1024); + ASSERT_TRUE(_engine->_txn_delete_bitmap_cache->init().ok()); + } + + void TearDown() override { + ExecEnv::GetInstance()->set_storage_engine(std::move(_old_engine)); + config::cloud_unique_id = _old_cloud_unique_id; + config::skip_writing_empty_rowset_metadata = _old_skip_empty; + config::enable_cloud_make_rs_visible_on_be = _old_make_visible; + } + + CloudTabletSPtr create_tablet(KeysType keys_type, bool mow) { + TabletMetaPB meta; + meta.set_tablet_id(10001); + meta.set_table_id(10002); + meta.set_enable_unique_key_merge_on_write(mow); + meta.mutable_schema()->set_keys_type(keys_type); + auto tablet_meta = std::make_shared(); + tablet_meta->init_from_pb(meta); + auto tablet = std::make_shared(*_engine, tablet_meta); + std::unique_lock lock(tablet->get_header_lock()); + tablet->reset_approximate_stats(0, 0, 0, 0); + return tablet; + } + + CloudStorageEngine* _engine = nullptr; + std::unique_ptr _old_engine; + std::string _old_cloud_unique_id; + bool _old_skip_empty = false; + bool _old_make_visible = false; +}; + +TEST_F(CloudLoadStreamTest, SetEmptyPolicyBeforePreparingMetadata) { + for (bool skip_empty : {false, true}) { + config::skip_writing_empty_rowset_metadata = skip_empty; + for (bool is_empty : {false, true}) { + WriteRequest req; + LoadStreamWriter writer(&req, nullptr); + auto builder = std::make_unique(*_engine, req, nullptr); + auto* observed_builder = builder.get(); + writer._rowset_builder = std::move(builder); + auto st = writer.init(is_empty); + ASSERT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("stop before preparing"), std::string::npos); + EXPECT_EQ(observed_builder->skip_metadata_at_init, skip_empty && is_empty); + } + } +} + +TEST_F(CloudLoadStreamTest, EmptyCommitSkipsRpcAndRegistersMarkers) { + for (auto keys_type : {DUP_KEYS, AGG_KEYS, UNIQUE_KEYS}) { + for (bool mow : {false, true}) { + if (mow && keys_type != UNIQUE_KEYS) { + continue; + } + WriteRequest req; + req.tablet_id = 10001; + req.txn_id = 100 + keys_type * 2 + mow; + req.txn_expiration = UnixSeconds() + 3600; + EmptyLoadRowsetBuilder builder(*_engine, req, nullptr); + auto tablet = create_tablet(keys_type, mow); + builder._tablet = tablet; + builder._tablet_schema = tablet->tablet_schema(); + auto meta = std::make_shared(); + meta->set_rowset_type(BETA_ROWSET); + meta->set_rowset_state(PREPARED); + meta->set_num_segments(0); + ASSERT_TRUE( + RowsetFactory::create_rowset(builder._tablet_schema, "", meta, &builder._rowset) + .ok()); + builder.set_skip_writing_rowset_metadata(true); + + auto st = builder.commit_txn(); + ASSERT_TRUE(st.ok()) << st.to_string(); + EXPECT_EQ(builder.commit_calls, 0); + EXPECT_TRUE(builder._is_committed); + EXPECT_EQ(tablet->fetch_add_approximate_num_rowsets(0), 1); + if (mow) { + EXPECT_TRUE(_engine->txn_delete_bitmap_cache().is_empty_rowset(req.txn_id, + req.tablet_id)); + } else { + auto marker = + _engine->committed_rs_mgr().get_committed_rowset(req.txn_id, req.tablet_id); + ASSERT_TRUE(marker.has_value()); + EXPECT_EQ(marker->first, nullptr); + EXPECT_GE(marker->second, req.txn_expiration); + } + } + } +} + +TEST_F(CloudLoadStreamTest, MetadataCommitFailureDoesNotRegisterMarker) { + WriteRequest req; + req.tablet_id = 10001; + req.txn_id = 200; + EmptyLoadRowsetBuilder builder(*_engine, req, nullptr); + auto tablet = create_tablet(DUP_KEYS, false); + builder._tablet = tablet; + builder.set_skip_writing_rowset_metadata(false); + + auto st = builder.commit_txn(); + ASSERT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("injected commit failure"), std::string::npos); + EXPECT_EQ(builder.commit_calls, 1); + EXPECT_FALSE(builder._is_committed); + EXPECT_EQ(tablet->fetch_add_approximate_num_rowsets(0), 0); + EXPECT_FALSE(_engine->committed_rs_mgr() + .get_committed_rowset(req.txn_id, req.tablet_id) + .has_value()); +} + +TEST_F(CloudLoadStreamTest, CloseTabletsConcurrentlyAndCollectAllResults) { + constexpr int tablet_count = 25; + FifoThreadPool pool(16, 64, "CloudLoadStreamCloseTest"); + LoadStreamMgr manager(1); + manager.set_heavy_work_pool(&pool); + RuntimeProfile profile("CloudLoadStreamCloseTest"); + PUniqueId load_id; + IndexStream index(load_id, 1, 1, nullptr, &manager, &profile, UnixSeconds() + 3600, "", false); + + std::atomic prepared {0}; + std::mutex mutex; + std::condition_variable cv; + int started = 0; + int active = 0; + int peak_active = 0; + bool timed_out = false; + for (int64_t tablet_id = 1; tablet_id <= tablet_count; ++tablet_id) { + auto tablet = std::make_shared(load_id, tablet_id, 1, &manager, &profile, + UnixSeconds() + 3600, "", false); + WriteRequest req; + req.tablet_id = tablet_id; + auto writer = std::make_shared(&req, nullptr); + writer->_rowset_builder = + std::make_unique(*_engine, req, prepared, [&, tablet_id] { + EXPECT_EQ(prepared.load(), tablet_count); + std::unique_lock lock(mutex); + ++started; + ++active; + peak_active = std::max(peak_active, active); + cv.notify_all(); + // Hold the first wave until ten tablets are closing concurrently. + // A serial implementation times out once instead of hanging the test. + if (!cv.wait_for(lock, std::chrono::seconds(5), + [&] { return started >= 10 || timed_out; })) { + timed_out = true; + cv.notify_all(); + } + --active; + return tablet_id % 2 == 0 + ? Status::OK() + : Status::InternalError("injected tablet close failure"); + }); + writer->_rowset_writer = writer->_rowset_builder->rowset_writer(); + writer->_is_init = true; + tablet->_load_stream_writer = std::move(writer); + index._tablet_streams_map.emplace(tablet_id, std::move(tablet)); + } + + std::vector success_tablets; + FailedTablets failed_tablets; + index.close({}, &success_tablets, &failed_tablets); + + EXPECT_FALSE(timed_out); + EXPECT_EQ(peak_active, 10); + EXPECT_EQ(started, tablet_count); + std::sort(success_tablets.begin(), success_tablets.end()); + std::vector expected_success; + for (int64_t id = 2; id <= tablet_count; id += 2) { + expected_success.push_back(id); + } + EXPECT_EQ(success_tablets, expected_success); + std::vector failed_ids; + for (const auto& [id, st] : failed_tablets) { + failed_ids.push_back(id); + EXPECT_NE(st.to_string().find("injected tablet close failure"), std::string::npos); + } + std::sort(failed_ids.begin(), failed_ids.end()); + std::vector expected_failed; + for (int64_t id = 1; id <= tablet_count; id += 2) { + expected_failed.push_back(id); + } + EXPECT_EQ(failed_ids, expected_failed); +} + +} // namespace doris diff --git a/be/test/cloud/cloud_warm_up_manager_test.cpp b/be/test/cloud/cloud_warm_up_manager_test.cpp index 4ac284b990bf01..90234a8d107105 100644 --- a/be/test/cloud/cloud_warm_up_manager_test.cpp +++ b/be/test/cloud/cloud_warm_up_manager_test.cpp @@ -30,9 +30,14 @@ #include "cloud/cloud_storage_engine.h" #include "cloud/config.h" +#include "common/config.h" #include "cpp/sync_point.h" +#include "runtime/cluster_info.h" +#include "runtime/exec_env.h" +#include "service/backend_options.h" #include "storage/olap_common.h" #include "storage/rowset/rowset_meta.h" +#include "util/defer_op.h" namespace doris { @@ -83,7 +88,13 @@ class CloudWarmUpManagerTest : public testing::Test { int32_t _origin_thread_pool_size = 0; }; -TEST_F(CloudWarmUpManagerTest, NonPositiveTimeoutQueuesBackgroundCopyAndReturns) { +TEST_F(CloudWarmUpManagerTest, LocalWarmupQueuesBackgroundCopyWithoutEventJobs) { + ClusterInfo cluster_info; + cluster_info.backend_id = 123; + auto* original_cluster_info = ExecEnv::GetInstance()->cluster_info(); + ExecEnv::GetInstance()->set_cluster_info(&cluster_info); + Defer restore_cluster_info { + [&] { ExecEnv::GetInstance()->set_cluster_info(original_cluster_info); }}; config::warm_up_manager_thread_pool_size = 1; CloudWarmUpManager manager(_engine); @@ -118,10 +129,20 @@ TEST_F(CloudWarmUpManagerTest, NonPositiveTimeoutQueuesBackgroundCopyAndReturns) SyncPoint::CallbackGuard warmup_enter_guard; SyncPoint::get_instance()->set_call_back( - "CloudWarmUpManager::_warm_up_rowset.enter", + "CloudWarmUpManager::_do_warm_up_rowset", [&](std::vector&& args) { auto* rs_meta = try_any_cast(args[0]); - auto* timeout_ms = try_any_cast(args[1]); + auto* replicas = try_any_cast*>(args[1]); + auto* timeout_ms = try_any_cast(args[2]); + auto* skip_existence_check = try_any_cast(args[3]); + auto* result = try_any_cast*>(args.back()); + result->second = true; + ASSERT_EQ(1, replicas->size()); + EXPECT_EQ(0, replicas->at(0).job_id); + EXPECT_EQ(123, replicas->at(0).replica.backend_id); + EXPECT_EQ(BackendOptions::get_localhost(), replicas->at(0).replica.host); + EXPECT_EQ(config::brpc_port, replicas->at(0).replica.brpc_port); + EXPECT_FALSE(*skip_existence_check); { std::lock_guard lock(observed_mtx); observed_tablet_id = rs_meta->tablet_id(); @@ -138,7 +159,8 @@ TEST_F(CloudWarmUpManagerTest, NonPositiveTimeoutQueuesBackgroundCopyAndReturns) std::atomic returned = false; std::thread caller([&] { - manager.warm_up_rowset(*rs_meta, /*table_id=*/0, /*sync_wait_timeout_ms=*/-1); + manager.warm_up_rowset(*rs_meta, /*table_id=*/0, /*sync_wait_timeout_ms=*/-1, + /*warm_up_local=*/true); returned = true; }); diff --git a/be/test/load/channel/load_stream_stub_map_test.cpp b/be/test/load/channel/load_stream_stub_map_test.cpp index 430d4f71a0cf75..080d82a705775d 100644 --- a/be/test/load/channel/load_stream_stub_map_test.cpp +++ b/be/test/load/channel/load_stream_stub_map_test.cpp @@ -16,6 +16,8 @@ // under the License. #include +#include + #include "exec/sink/load_stream_map_pool.h" #include "exec/sink/load_stream_stub.h" @@ -56,4 +58,15 @@ TEST_F(LoadStreamMapPoolTest, test) { EXPECT_EQ(0, pool.size()); } +TEST_F(LoadStreamMapPoolTest, AccumulateIndependentWriterSegmentCounts) { + LoadStreamMap map(UniqueId(1, 2), 10, 1, 2, nullptr); + std::thread first([&] { map.save_segments_for_tablet({{100, 2}, {200, 1}}); }); + std::thread second([&] { map.save_segments_for_tablet({{100, 3}, {300, 4}}); }); + first.join(); + second.join(); + EXPECT_EQ(5, map._segments_for_tablet.at(100)); + EXPECT_EQ(1, map._segments_for_tablet.at(200)); + EXPECT_EQ(4, map._segments_for_tablet.at(300)); +} + } // namespace doris diff --git a/be/test/runtime/load_stream_test.cpp b/be/test/runtime/load_stream_test.cpp index ce218d1ceff14b..6e1c46c6052a4f 100644 --- a/be/test/runtime/load_stream_test.cpp +++ b/be/test/runtime/load_stream_test.cpp @@ -33,6 +33,7 @@ #include #include +#include "cloud/cloud_rowset_builder.h" #include "common/config.h" #include "common/status.h" #include "gtest/gtest_pred_impl.h" @@ -1420,4 +1421,250 @@ TEST_F(LoadStreamMgrTest, incremental_close_race_orhpans_streams) { config::enable_debug_points = saved_debug_points; } +TEST_F(LoadStreamMgrTest, SinkUploadDuplicateAndMissingWriterResults) { + WriteRequest req; + req.tablet_id = NORMAL_TABLET_ID; + RuntimeProfile profile("sink-upload-test"); + LoadStreamWriter writer(&req, &profile); + writer._rowset_builder->_tablet = engine_ref->tablet_manager()->get_tablet(NORMAL_TABLET_ID); + writer._writer_segment_start_ids.emplace("writer", 100); + RowsetMetaPB meta; + meta.set_num_segments(1); + meta.add_segment_ids(100); + // Seed an already accepted result to exercise retries without uploading a file. + writer._partial_rowset_metas.emplace(100, meta); + int64_t added = -1; + ASSERT_TRUE(writer.add_partial_rowset("writer", meta, &added).ok()); + EXPECT_EQ(0, added); + EXPECT_FALSE(writer.add_partial_rowset("unknown", meta, &added).ok()); + meta.set_num_rows(7); + EXPECT_FALSE(writer.add_partial_rowset("writer", meta, &added).ok()); + writer._is_init = true; + writer._sink_upload = true; + writer._partial_rowset_metas.clear(); + EXPECT_FALSE(writer.pre_close().ok()); + writer._pre_closed = true; + EXPECT_FALSE(writer.add_partial_rowset("writer", meta, &added).ok()); +} + +class CloudSinkUploadMetaTest : public testing::Test { +protected: + RowsetMetaPB base_meta() { + RowsetMetaPB meta; + meta.set_rowset_id(0); + meta.set_rowset_id_v2("000000000000000000000000000000000000000000000001"); + meta.set_tablet_id(10); + meta.set_txn_id(20); + meta.set_resource_id("vault"); + meta.set_index_id(30); + meta.set_partition_id(40); + meta.mutable_load_id()->set_hi(50); + meta.mutable_load_id()->set_lo(60); + meta.mutable_tablet_schema()->set_keys_type(DUP_KEYS); + meta.mutable_tablet_schema()->set_num_short_key_columns(0); + meta.mutable_tablet_schema()->set_num_rows_per_row_block(1024); + meta.mutable_tablet_schema()->set_compress_kind(COMPRESS_LZ4); + return meta; + } + + RowsetMetaPB partial(int id) { + auto meta = base_meta(); + meta.set_num_segments(1); + meta.add_segment_ids(id); + meta.add_num_segment_rows(id + 1); + meta.set_num_rows(id + 1); + meta.add_segments_file_size(100 + id); + meta.set_data_disk_size(100 + id); + meta.set_index_disk_size(10); + meta.set_total_disk_size(110 + id); + auto* bounds = meta.add_segments_key_bounds(); + bounds->set_min_key("a"); + bounds->set_max_key("z"); + meta.add_inverted_index_file_info()->set_index_size(10); + return meta; + } +}; + +TEST_F(CloudSinkUploadMetaTest, AssembleSparseSegmentsAndPackedLocations) { + auto first = partial(0); + auto* location = &(*first.mutable_packed_slice_locations())["data/10/rowset_0.dat"]; + location->set_packed_file_path("packed/object"); + location->set_offset(64); + location->set_size(100); + RowsetMetaPB merged; + // A writer can finish first even though its allocated range is last. + std::map partials; + partials.emplace(100, partial(102)); + partials.emplace(0, first); + ASSERT_TRUE(CloudRowsetBuilder::assemble_rowset_meta_from_partials(base_meta(), partials, 100, + &merged) + .ok()); + EXPECT_EQ(2, merged.num_segments()); + EXPECT_EQ(0, merged.segment_ids(0)); + EXPECT_EQ(102, merged.segment_ids(1)); + EXPECT_EQ(1, merged.num_segment_rows(0)); + EXPECT_EQ(103, merged.num_segment_rows(1)); + EXPECT_EQ(202, merged.segments_file_size(1)); + EXPECT_EQ(104, merged.num_rows()); + EXPECT_EQ(322, merged.total_disk_size()); + EXPECT_EQ(2, merged.inverted_index_file_info_size()); + EXPECT_EQ(COMMITTED, merged.rowset_state()); + RowsetMetaPB restored; + ASSERT_TRUE(restored.ParseFromString(merged.SerializeAsString())); + EXPECT_EQ(64, restored.packed_slice_locations().at("data/10/rowset_0.dat").offset()); + EXPECT_EQ(102, restored.segment_ids(1)); +} + +TEST_F(CloudSinkUploadMetaTest, RejectInvalidPartialResults) { + RowsetMetaPB merged; + auto check = [&](const RowsetMetaPB& meta) { + EXPECT_FALSE( + CloudRowsetBuilder::validate_partial_rowset_meta(base_meta(), meta, 100, 100).ok()); + }; + check(partial(99)); // Outside this writer's range. + check(partial(200)); + auto meta = partial(100); + meta.set_rowset_id_v2("different-rowset"); + check(meta); + meta = partial(100); + meta.clear_segments_file_size(); + check(meta); + meta = partial(100); + meta.set_num_rows(1); + check(meta); + meta = partial(100); + meta.set_total_disk_size(1); + check(meta); + meta = partial(100); + meta.set_segments_key_bounds_aggregated(true); + check(meta); + meta = partial(100); + meta.mutable_tablet_schema()->add_index()->set_index_type(INVERTED); + meta.clear_inverted_index_file_info(); + check(meta); + EXPECT_FALSE(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + base_meta(), {{0, partial(0)}, {1, partial(1)}}, 1, &merged) + .ok()); +} + +TEST_F(CloudSinkUploadMetaTest, EmptyWriter) { + RowsetMetaPB merged; + ASSERT_TRUE(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + base_meta(), {{0, base_meta()}}, 100, &merged) + .ok()); + EXPECT_EQ(0, merged.num_segments()); + EXPECT_TRUE(merged.empty()); +} + +TEST_F(CloudSinkUploadMetaTest, AssembleVariantSchemaWithoutDuplicateFields) { + auto base = base_meta(); + auto* schema = base.mutable_tablet_schema(); + auto* key = schema->add_column(); + key->set_unique_id(1); + key->set_name("k"); + key->set_type("INT"); + key->set_is_key(true); + key->set_is_nullable(false); + auto* variant = schema->add_column(); + variant->set_unique_id(2); + variant->set_name("v"); + variant->set_type("VARIANT"); + variant->set_is_key(false); + variant->set_is_nullable(true); + auto* index = schema->add_index(); + index->set_index_id(3); + index->set_index_name("idx_k"); + index->set_index_type(INVERTED); + index->add_col_unique_id(1); + schema->add_cluster_key_uids(1); + + auto first = partial(0); + auto second = partial(100); + *first.mutable_tablet_schema() = *schema; + *second.mutable_tablet_schema() = *schema; + RowsetMetaPB merged; + ASSERT_TRUE(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + base, {{0, first}, {100, second}}, 100, &merged) + .ok()); + const auto& merged_schema = merged.tablet_schema(); + ASSERT_EQ(2, merged_schema.column_size()); + EXPECT_EQ(1, merged_schema.column(0).unique_id()); + EXPECT_EQ(2, merged_schema.column(1).unique_id()); + EXPECT_EQ("VARIANT", merged_schema.column(1).type()); + ASSERT_EQ(1, merged_schema.index_size()); + EXPECT_EQ(3, merged_schema.index(0).index_id()); + ASSERT_EQ(1, merged_schema.cluster_key_uids_size()); + EXPECT_EQ(1, merged_schema.cluster_key_uids(0)); +} + +TEST_F(CloudSinkUploadMetaTest, PreserveAggregateAndMorOverlappingLayout) { + for (auto type : {AGG_KEYS, UNIQUE_KEYS}) { + auto base = base_meta(); + base.mutable_tablet_schema()->set_keys_type(type); + auto first = partial(0); + auto second = partial(100); + first.mutable_tablet_schema()->set_keys_type(type); + second.mutable_tablet_schema()->set_keys_type(type); + RowsetMetaPB merged; + ASSERT_TRUE(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + base, {{0, first}, {100, second}}, 100, &merged) + .ok()); + EXPECT_EQ(type, merged.tablet_schema().keys_type()); + EXPECT_EQ(OVERLAPPING, merged.segments_overlap_pb()); + EXPECT_EQ(100, merged.segment_ids(1)); + EXPECT_EQ(102, merged.num_rows()); + } +} + +TEST_F(CloudSinkUploadMetaTest, PreserveDisjointKeyRanges) { + auto first = partial(0); + auto second = partial(100); + first.mutable_segments_key_bounds(0)->set_max_key("b"); + second.mutable_segments_key_bounds(0)->set_min_key("c"); + RowsetMetaPB merged; + auto base = base_meta(); + ASSERT_TRUE(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + base, {{0, first}, {100, second}}, 100, &merged) + .ok()); + EXPECT_EQ(NONOVERLAPPING, merged.segments_overlap_pb()); + + // Primary-key bounds cannot establish the ordering of cluster keys. + base.mutable_tablet_schema()->add_cluster_key_uids(1); + ASSERT_TRUE(CloudRowsetBuilder::assemble_rowset_meta_from_partials( + base, {{0, first}, {100, second}}, 100, &merged) + .ok()); + EXPECT_EQ(OVERLAPPING, merged.segments_overlap_pb()); +} + +TEST(CloudSinkMowTest, RequireMatchingSnapshotAndCompleteBitmap) { + PCloudLoadMowResult result; + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + result.set_snapshot_version(5); + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + auto* bitmap = result.mutable_delete_bitmap(); + EXPECT_TRUE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 6).ok()); + bitmap->add_rowset_ids("020000000000000100000000000000020000000000000003"); + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + bitmap->add_segment_ids(1000); + bitmap->add_versions(0); + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + roaring::Roaring rows; + rows.add(7); + std::string bytes(rows.getSizeInBytes(), '\0'); + rows.write(bytes.data()); + bitmap->add_segment_delete_bitmaps(bytes); + ASSERT_TRUE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + auto decoded = DeleteBitmap::from_pb(*bitmap, 1); + RowsetId rowset_id; + rowset_id.init(bitmap->rowset_ids(0)); + EXPECT_TRUE(decoded.contains({rowset_id, 1000, 0}, 7)); + EXPECT_FALSE(decoded.contains({rowset_id, 0, 0}, 7)); + bitmap->set_versions(0, 5); + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); + bitmap->set_versions(0, 0); + bitmap->set_segment_delete_bitmaps(0, "invalid"); + EXPECT_FALSE(CloudRowsetBuilder::validate_sink_mow_result(result, 5).ok()); +} + } // namespace doris diff --git a/be/test/service/http/stream_load_test.cpp b/be/test/service/http/stream_load_test.cpp index 93dd9d044c7c8b..d3a88e79ce98b6 100644 --- a/be/test/service/http/stream_load_test.cpp +++ b/be/test/service/http/stream_load_test.cpp @@ -27,6 +27,7 @@ #include "event2/http.h" #include "event2/http_struct.h" #include "evhttp.h" +#include "gen_cpp/FrontendService_types.h" #include "load/group_commit/wal/wal_manager.h" #include "load/stream_load/stream_load_context.h" #include "runtime/exec_env.h" @@ -43,6 +44,16 @@ namespace doris { +TEST(StreamLoadRequestTest, SinkUploadPreservesOptionalPresence) { + TStreamLoadPutRequest request; + EXPECT_FALSE(request.__isset.cloud_memtable_sink_upload); + for (bool enabled : {false, true}) { + request.__set_cloud_memtable_sink_upload(enabled); + EXPECT_TRUE(request.__isset.cloud_memtable_sink_upload); + EXPECT_EQ(request.cloud_memtable_sink_upload, enabled); + } +} + class StreamLoadTest : public testing::Test { public: StreamLoadTest() = default; diff --git a/be/test/storage/cloud_file_cache_write_index_only_test.cpp b/be/test/storage/cloud_file_cache_write_index_only_test.cpp index 445768e7d63a73..b6e9ed042ee3dd 100644 --- a/be/test/storage/cloud_file_cache_write_index_only_test.cpp +++ b/be/test/storage/cloud_file_cache_write_index_only_test.cpp @@ -603,6 +603,35 @@ TEST_F(CloudFileCacheWriteIndexOnlyConfigTest, EXPECT_EQ(open_file_count, 0); } +TEST_F(CloudFileCacheWriteIndexOnlyTest, RemoteSinkSkipsIndexPreload) { + auto context = create_context(create_schema(true)); + context.disable_file_cache = true; + int load_count = 0; + SyncPoint::CallbackGuard guard; + SyncPoint::get_instance()->set_call_back( + "SegmentIndexFileCacheLoader::load_segment_index_to_file_cache", + [&](auto&& args) { + ++load_count; + auto* ret = try_any_cast_ret(args); + ret->first = Status::OK(); + ret->second = true; + }, + &guard); + SyncPoint::get_instance()->enable_processing(); + segment_v2::SegmentIndexFileCacheInfo info; + info.segment_file_size = 2; + info.index_ranges.push_back({.offset = 1, .size = 1}); + ASSERT_TRUE(segment_v2::SegmentIndexFileCacheLoader::preload_segment_index_to_file_cache( + context, 0, "remote_sink.dat", info) + .ok()); + EXPECT_EQ(0, load_count); + context.disable_file_cache = false; + ASSERT_TRUE(segment_v2::SegmentIndexFileCacheLoader::preload_segment_index_to_file_cache( + context, 0, "remote_sink.dat", info) + .ok()); + EXPECT_EQ(1, load_count); +} + TEST_F(CloudFileCacheWriteIndexOnlyTest, LoadUsesTheWholeBlockWritePathAndPreloadsAfterAllSegmentFilesClosed) { auto tablet_schema = create_schema(true); diff --git a/be/test/storage/compaction/compaction_file_cache_test.cpp b/be/test/storage/compaction/compaction_file_cache_test.cpp index 765df31bf1c7b3..592c25390894f5 100644 --- a/be/test/storage/compaction/compaction_file_cache_test.cpp +++ b/be/test/storage/compaction/compaction_file_cache_test.cpp @@ -347,6 +347,26 @@ TEST_F(CompactionFileCacheTest, GlobalIndexFileOnlyTakesPrecedenceOverCompaction EXPECT_EQ(index_opts.approximate_bytes_to_write, 0); } +TEST_F(CompactionFileCacheTest, DisableFileCacheOverridesAllWritePolicies) { + config::enable_file_cache_write_index_file_only = true; + + RowsetWriterContext ctx; + ctx.write_file_cache = true; + ctx.disable_file_cache = true; + ctx.compaction_output_write_index_only = true; + ctx.approximate_bytes_to_write = 12345; + + auto segment_opts = ctx.get_file_writer_options(FileType::SEGMENT_FILE); + EXPECT_FALSE(segment_opts.write_file_cache); + EXPECT_FALSE(segment_opts.allow_adaptive_file_cache_write); + EXPECT_EQ(segment_opts.approximate_bytes_to_write, 0); + + auto index_opts = ctx.get_file_writer_options(FileType::INVERTED_INDEX_FILE); + EXPECT_FALSE(index_opts.write_file_cache); + EXPECT_FALSE(index_opts.allow_adaptive_file_cache_write); + EXPECT_EQ(index_opts.approximate_bytes_to_write, 0); +} + // ============================================================================ // Tests for should_enable_compaction_cache_index_only function // ============================================================================ diff --git a/be/test/storage/rowset/beta_rowset_test.cpp b/be/test/storage/rowset/beta_rowset_test.cpp index c7337ff4f9bfc6..8b5ad55be446c8 100644 --- a/be/test/storage/rowset/beta_rowset_test.cpp +++ b/be/test/storage/rowset/beta_rowset_test.cpp @@ -60,6 +60,8 @@ #include "storage/segment/segment.h" #include "storage/storage_engine.h" #include "storage/storage_policy.h" +#include "storage/tablet/tablet.h" +#include "storage/tablet/tablet_meta.h" #include "storage/tablet/tablet_schema.h" #include "storage/utils.h" #include "util/s3_util.h" @@ -239,9 +241,44 @@ class BetaRowsetWriterForTest : public BetaRowsetWriter { public: explicit BetaRowsetWriterForTest(StorageEngine& engine) : BetaRowsetWriter(engine) {} - Status build_tmp(RowsetSharedPtr& rowset) { return _build_tmp(rowset); } + Status build_tmp(RowsetSharedPtr& rowset, int32_t segment_id) { + return _build_tmp(rowset, segment_id); + } }; +TEST_F(BetaRowsetTest, StreamedBitmapSnapshotFailurePreventsQueueing) { + class SnapshotFailureWriter : public BetaRowsetWriter { + public: + explicit SnapshotFailureWriter(StorageEngine& engine) : BetaRowsetWriter(engine) {} + + protected: + Status _build_rowset_meta(RowsetMeta*, bool, std::vector*) override { + return Status::InternalError("streamed metadata snapshot failed"); + } + }; + + EngineOptions options; + StorageEngine engine(options); + auto tablet_meta = std::make_shared(); + tablet_meta->set_enable_unique_key_merge_on_write(true); + SnapshotFailureWriter writer(engine); + auto tablet_schema = std::make_shared(); + create_tablet_schema(tablet_schema); + RowsetWriterContext writer_context; + create_rowset_writer_context(tablet_schema, &writer_context); + ASSERT_TRUE(writer.init(writer_context).ok()); + writer._context.tablet = std::make_shared(engine, tablet_meta, nullptr); + writer._context.mow_context = std::make_shared( + 0, 1, std::make_shared(), std::vector {}, + std::make_shared(0)); + // No owned file writer: LoadStreamWriter has already closed the streamed file. + // Snapshot errors must be returned before touching the asynchronous executor. + ASSERT_EQ(writer._calc_delete_bitmap_token, nullptr); + auto st = writer._generate_delete_bitmap(0); + EXPECT_FALSE(st.ok()); + EXPECT_NE(st.to_string().find("streamed metadata snapshot failed"), std::string::npos); +} + class S3ClientMock : public Aws::S3::S3Client { S3ClientMock() {} S3ClientMock(const Aws::Auth::AWSCredentials& credentials, @@ -522,6 +559,8 @@ TEST_F(BetaRowsetTest, TmpRowsetUsesCompletedSegmentIds) { RowsetWriterContext writer_context; create_rowset_writer_context(tablet_schema, &writer_context); + writer_context.is_partial_output_writer = true; + EngineOptions options; StorageEngine engine(options); BetaRowsetWriterForTest writer(engine); @@ -533,7 +572,7 @@ TEST_F(BetaRowsetTest, TmpRowsetUsesCompletedSegmentIds) { ASSERT_TRUE(writer.add_segment(2, segment_statistics).ok()); RowsetSharedPtr tmp_rowset; - ASSERT_TRUE(writer.build_tmp(tmp_rowset).ok()); + ASSERT_TRUE(writer.build_tmp(tmp_rowset, 6).ok()); ASSERT_NE(tmp_rowset, nullptr); EXPECT_EQ(tmp_rowset->num_segments(), 2); EXPECT_EQ(tmp_rowset->rowset_meta()->position_of(2), 0); diff --git a/be/test/storage/rowset/rowset_meta_test.cpp b/be/test/storage/rowset/rowset_meta_test.cpp index 633c4f4999abed..b8a415e574a0bc 100644 --- a/be/test/storage/rowset/rowset_meta_test.cpp +++ b/be/test/storage/rowset/rowset_meta_test.cpp @@ -25,15 +25,19 @@ #include #include +#include #include #include +#include "common/exception.h" #include "common/status.h" #include "cpp/sync_point.h" #include "gtest/gtest_pred_impl.h" #include "io/fs/file_reader.h" #include "io/fs/file_system.h" +#include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" +#include "io/fs/packed_file_writer.h" #include "storage/olap_common.h" #include "storage/olap_meta.h" #include "storage/tablet/tablet_schema.h" @@ -47,6 +51,98 @@ using std::string; namespace doris { +namespace { + +class PackedLocationWriter : public io::FileWriter { +public: + explicit PackedLocationWriter(std::string path) : _path(std::move(path)) {} + + Status close(bool non_block = false) override { + _state = non_block ? State::ASYNC_CLOSING : State::CLOSED; + return Status::OK(); + } + Status appendv(const Slice*, size_t) override { return Status::NotSupported("test writer"); } + const io::Path& path() const override { return _path; } + size_t bytes_appended() const override { return 10; } + State state() const override { return _state; } + +private: + io::Path _path; + State _state = State::OPENED; +}; + +} // namespace + +TEST(RowsetMetaPackedFileTest, CollectCompletedSegmentAndIndex) { + RowsetMeta meta; + RowsetId rowset_id; + rowset_id.init(1); + meta.set_rowset_id(rowset_id); + // Use destination paths with a nonzero segment ID, including the V2 index file. + for (const auto* suffix : {".dat", ".idx"}) { + std::string path = std::string("data/123/packed_location_test_7") + suffix; + io::PackedFileWriter writer( + std::make_unique("s3://bucket/prefix/" + path), path); + io::PackedSliceLocation location; + location.packed_file_path = "data/packed_file/test.bin"; + location.offset = 20; + location.size = 10; + location.packed_file_size = 100; + // The writer's handle remains sufficient after the manager's index is gone. + writer._packed_slice_handle = std::make_shared(location); + writer._packed_slice_handle->set_upload_result(io::PackedSliceUploadState::UPLOADED, + location.packed_file_size); + + ASSERT_TRUE(meta.collect_packed_slice_location(writer, path).ok()); + EXPECT_EQ(meta.get_rowset_pb().packed_slice_locations().count(path), 0); + writer._state = io::FileWriter::State::ASYNC_CLOSING; + ASSERT_TRUE(meta.collect_packed_slice_location(writer, path).ok()); + EXPECT_EQ(meta.get_rowset_pb().packed_slice_locations().count(path), 0); + writer._state = io::FileWriter::State::CLOSED; + ASSERT_TRUE(meta.collect_packed_slice_location(writer, path).ok()); + // Repeated collection must not duplicate the mapping. + ASSERT_TRUE(meta.collect_packed_slice_location(writer, path).ok()); + auto pb = meta.get_rowset_pb(); + const auto& stored = pb.packed_slice_locations().at(path); + EXPECT_EQ(stored.packed_file_path(), location.packed_file_path); + EXPECT_EQ(stored.offset(), location.offset); + EXPECT_EQ(stored.size(), location.size); + EXPECT_EQ(stored.packed_file_size(), location.packed_file_size); + } + RowsetMeta restored; + std::string serialized; + ASSERT_TRUE(meta.serialize(&serialized)); + ASSERT_TRUE(restored.init(serialized)); + EXPECT_EQ(restored.get_rowset_pb().packed_slice_locations_size(), 2); +} + +TEST(RowsetMetaPackedFileTest, DirectFileNeedsNoPackedLocation) { + RowsetMeta meta; + PackedLocationWriter writer("direct_file.dat"); + ASSERT_TRUE(writer.close().ok()); + EXPECT_TRUE(meta.collect_packed_slice_location(writer, "direct_file.dat").ok()); + EXPECT_EQ(meta.get_rowset_pb().packed_slice_locations_size(), 0); +} + +TEST(RowsetMetaPackedFileTest, MissingPackedLocationFails) { + RowsetMeta meta; + io::PackedFileWriter writer( + std::make_unique("missing_packed_location.dat"), + "missing_packed_location.dat"); + writer._state = io::FileWriter::State::CLOSED; +#ifndef NDEBUG + GTEST_FLAG_SET(death_test_style, "threadsafe"); + EXPECT_DEATH(static_cast( + meta.collect_packed_slice_location(writer, "missing_packed_location.dat")), + "location.packed_file_path.empty"); +#else + EXPECT_THROW(static_cast( + meta.collect_packed_slice_location(writer, "missing_packed_location.dat")), + Exception); +#endif + EXPECT_EQ(meta.get_rowset_pb().packed_slice_locations_size(), 0); +} + const std::string rowset_meta_path = "./be/test/storage/test_data/rowset.json"; class RowsetMetaTest : public testing::Test { diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 40a1b8dfcfb1c6..8e7583ab87c7e2 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -622,6 +622,11 @@ public class Config extends ConfigBase { + "in stream load") public static boolean stream_load_default_memtable_on_sink_node = false; + @ConfField(mutable = true, masterOnly = true, description = "Upload cloud memtable segments from sink BEs " + + "to object storage in HTTP Stream, Routine Load, and Group Commit internal loads. " + + "Ordinary Stream Load uses this default unless the cloud_memtable_sink_upload header is set.") + public static boolean cloud_stream_load_default_memtable_sink_upload = true; + @ConfField(mutable = true, masterOnly = true, description = "Whether to enable forwarding group commit stream " + "load to follower nodes. If true, stream load with " + "group commit mode will be forwarded to a follower FE " + "round robin.") diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index 3c43e26f971048..743248cdf2b300 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -379,13 +379,15 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme } } } - if (isPartialUpdate() || hasInvertedIndexV1 || Config.isCloudMode()) { + if (isPartialUpdate() || hasInvertedIndexV1) { isEnableMemtableOnSinkNode = false; } // Generate loading task and init the plan of task LoadLoadingTask task = createTask(db, table, brokerFileGroups, isEnableMemtableOnSinkNode, batchSize, aggKey, attachment); + task.setCloudMemtableSinkUpload(Boolean.parseBoolean(sessionVariables.getOrDefault( + SessionVariable.ENABLE_CLOUD_MEMTABLE_SINK_UPLOAD, "true"))); idToTasks.put(task.getSignature(), task); // idToTasks contains previous LoadPendingTasks, so idToTasks is just used to save all tasks. // use newLoadingTasks to save new created loading tasks and submit them later. diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java index 05fe3f637e3201..0246533a05d96b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java @@ -110,6 +110,8 @@ public BulkLoadJob(EtlJobType jobType, long dbId, String label, if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); sessionVariables.put(SessionVariable.SQL_MODE, Long.toString(var.getSqlMode())); + sessionVariables.put(SessionVariable.ENABLE_CLOUD_MEMTABLE_SINK_UPLOAD, + Boolean.toString(var.enableCloudMemtableSinkUpload)); sessionVariables.put(SessionVariable.AUTO_PROFILE_THRESHOLD_MS, Long.toString(var.getAutoProfileThresholdMs())); sessionVariables.put(SessionVariable.PROFILE_LEVEL, Long.toString(var.getProfileLevel())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java index 3993ab2dd498b5..b91087d2d727ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java @@ -84,6 +84,7 @@ public class LoadLoadingTask extends LoadTask { private final boolean singleTabletLoadPerSink; private final boolean enableMemTableOnSinkNode; + private boolean cloudMemtableSinkUpload; private final int batchSize; private NereidsLoadingTaskPlanner planner; @@ -147,6 +148,10 @@ public void setLoadBackendSelectionHint(BackendSelection.SelectionHint hint) { loadBackendSelectionHint = hint; } + public void setCloudMemtableSinkUpload(boolean enabled) { + cloudMemtableSinkUpload = enabled; + } + public TUniqueId getLoadId() { return loadId; } @@ -190,6 +195,7 @@ protected void executeOnce() throws Exception { curCoordinator.setExecMemoryLimit(execMemLimit); curCoordinator.setMemTableOnSinkNode(enableMemTableOnSinkNode); + curCoordinator.getQueryOptions().setEnableCloudMemtableSinkUpload(cloudMemtableSinkUpload); if (enableMemTableOnSinkNode) { curCoordinator.setBatchSize(batchSize); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadTaskInfo.java index 2ece54c823ecfb..8b847837ff2f9a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadTaskInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadTaskInfo.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.Separator; import org.apache.doris.catalog.info.PartitionNamesInfo; +import org.apache.doris.common.Config; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.thrift.TFileCompressType; @@ -147,6 +148,10 @@ default boolean isMemtableOnSinkNode() { return false; } + default boolean isCloudMemtableSinkUpload() { + return Config.cloud_stream_load_default_memtable_sink_upload; + } + default int getStreamPerNode() { return 2; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java index 78896eae5aa6a2..d3082d5264470d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java @@ -320,6 +320,7 @@ public TPipelineFragmentParams plan(TUniqueId loadId, int fragmentInstanceIdInde ? taskInfo.isMemtableOnSinkNode() : false; queryOptions.setEnableMemtableOnSinkNode(enableMemtableOnSinkNode); + queryOptions.setEnableCloudMemtableSinkUpload(taskInfo.isCloudMemtableSinkUpload()); queryOptions.setNewVersionUnixTimestamp(true); queryOptions.setNewVersionPercentile(true); queryOptions.setNewVersionBitmapOpCount(true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadTask.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadTask.java index f5ddca41f19a29..dbd89058849686 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadTask.java @@ -90,6 +90,7 @@ public class NereidsStreamLoadTask implements NereidsLoadTaskInfo { private boolean enableProfile = false; private boolean memtableOnSinkNode = false; + private boolean cloudMemtableSinkUpload = true; private int streamPerNode = 2; private byte enclose = 0; @@ -327,6 +328,11 @@ public boolean isMemtableOnSinkNode() { return memtableOnSinkNode; } + @Override + public boolean isCloudMemtableSinkUpload() { + return cloudMemtableSinkUpload; + } + public void setMemtableOnSinkNode(boolean memtableOnSinkNode) { this.memtableOnSinkNode = memtableOnSinkNode; } @@ -368,6 +374,7 @@ public static NereidsStreamLoadTask fromTStreamLoadPutRequest(TStreamLoadPutRequ * setMultiTableBaseTaskInfo */ public void setMultiTableBaseTaskInfo(LoadTaskInfo task) throws UserException { + this.cloudMemtableSinkUpload = Config.cloud_stream_load_default_memtable_sink_upload; this.mergeType = task.getMergeType(); this.columnSeparator = task.getColumnSeparator(); this.whereExpr = task.getWhereExpr() != null ? parseWhereExpr( @@ -516,6 +523,11 @@ private void setOptionalFromTSLPutRequest(TStreamLoadPutRequest request) throws } else { this.memtableOnSinkNode = Config.stream_load_default_memtable_on_sink_node; } + if (request.isSetCloudMemtableSinkUpload()) { + this.cloudMemtableSinkUpload = request.isCloudMemtableSinkUpload(); + } else { + this.cloudMemtableSinkUpload = Config.cloud_stream_load_default_memtable_sink_upload; + } if (request.isSetStreamPerNode()) { this.streamPerNode = request.getStreamPerNode(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 9e40aa4869c539..fa4cb4e619113e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -651,6 +651,8 @@ public String toString() { public static final String ENABLE_MEMTABLE_ON_SINK_NODE = "enable_memtable_on_sink_node"; + public static final String ENABLE_CLOUD_MEMTABLE_SINK_UPLOAD = "enable_cloud_memtable_sink_upload"; + public static final String LOAD_STREAM_PER_NODE = "load_stream_per_node"; public static final String ENABLE_UNIQUE_KEY_PARTIAL_UPDATE = "enable_unique_key_partial_update"; @@ -2694,6 +2696,9 @@ public Map getForceEagerAggHintMap() { @VarAttrDef.VarAttr(name = ENABLE_MEMTABLE_ON_SINK_NODE, needForward = true) public boolean enableMemtableOnSinkNode = true; + @VarAttrDef.VarAttr(name = ENABLE_CLOUD_MEMTABLE_SINK_UPLOAD, needForward = true) + public boolean enableCloudMemtableSinkUpload = true; + @VarAttrDef.VarAttr(name = LOAD_STREAM_PER_NODE) public int loadStreamPerNode = 2; @@ -5554,6 +5559,7 @@ public TQueryOptions toThrift() { tResult.setTruncateCharOrVarcharColumns(truncateCharOrVarcharColumns); tResult.setEnableMemtableOnSinkNode(enableMemtableOnSinkNode); + tResult.setEnableCloudMemtableSinkUpload(enableCloudMemtableSinkUpload); tResult.setInvertedIndexConjunctionOptThreshold(invertedIndexConjunctionOptThreshold); tResult.setInvertedIndexMaxExpansions(invertedIndexMaxExpansions); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 2f98052fd9a321..70e929a5b71bcc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -3205,6 +3205,7 @@ private void httpStreamPutImpl(TStreamLoadPutRequest request, TStreamLoadPutResu } else { ctx.getSessionVariable().enableMemtableOnSinkNode = Config.stream_load_default_memtable_on_sink_node; } + ctx.getSessionVariable().enableCloudMemtableSinkUpload = Config.cloud_stream_load_default_memtable_sink_upload; ctx.getSessionVariable().groupCommit = request.getGroupCommitMode(); ctx.getSessionVariable().setEnableInsertStrict(false); if (request.isSetPartialUpdate() && !request.isPartialUpdate()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java index 9c488169ab4c70..ccfa6fc5aa38d6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java @@ -58,6 +58,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -206,8 +208,9 @@ public void testPendingTaskOnFinishedWithDuplicated() { Assertions.assertEquals(0, idToTasks.size()); } - @Test - public void testPendingTaskOnFinished() throws Exception { + @ParameterizedTest + @CsvSource({"false", "true"}) + public void testPendingTaskOnFinished(boolean sinkUpload) throws Exception { BrokerPendingTaskAttachment attachment = Mockito.mock(BrokerPendingTaskAttachment.class); Env env = Mockito.mock(Env.class); InternalCatalog catalog = Mockito.mock(InternalCatalog.class); @@ -240,6 +243,7 @@ public void testPendingTaskOnFinished() throws Exception { BrokerLoadJob brokerLoadJob = new BrokerLoadJob(); Deencapsulation.setField(brokerLoadJob, "state", JobState.LOADING); + brokerLoadJob.sessionVariables.put("enable_cloud_memtable_sink_upload", Boolean.toString(sinkUpload)); BrokerDesc brokerDesc = Mockito.mock(BrokerDesc.class); Deencapsulation.setField(brokerLoadJob, "brokerDesc", brokerDesc); long taskId = 1L; @@ -306,6 +310,10 @@ public void testPendingTaskOnFinished() throws Exception { Assertions.assertEquals(true, finishedTaskIds.contains(taskId)); Map idToTasks = Deencapsulation.getField(brokerLoadJob, "idToTasks"); Assertions.assertEquals(3, idToTasks.size()); + for (LoadTask task : idToTasks.values()) { + boolean actualSinkUpload = Deencapsulation.getField(task, "cloudMemtableSinkUpload"); + Assertions.assertEquals(sinkUpload, actualSinkUpload); + } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java index 39cff42be39064..a63d1694323ce9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadPlannerTest.java @@ -17,12 +17,18 @@ package org.apache.doris.planner; +import org.apache.doris.common.Config; import org.apache.doris.common.IdGenerator; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.load.NereidsLoadUtils; +import org.apache.doris.nereids.load.NereidsStreamLoadTask; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileType; +import org.apache.doris.thrift.TStreamLoadPutRequest; +import org.apache.doris.thrift.TUniqueId; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -37,6 +43,30 @@ public void testParseStmt() throws Exception { Assertions.assertEquals(4, expressions.size()); } + @Test + public void testStreamLoadSinkUploadRequestOverridesDefaultConfig() throws Exception { + boolean original = Config.cloud_stream_load_default_memtable_sink_upload; + try { + for (boolean configured : new boolean[] {false, true}) { + Config.cloud_stream_load_default_memtable_sink_upload = configured; + TStreamLoadPutRequest request = new TStreamLoadPutRequest(); + request.setLoadId(new TUniqueId(1, 2)); + request.setTxnId(3); + request.setFileType(TFileType.FILE_STREAM); + request.setFormatType(TFileFormatType.FORMAT_CSV_PLAIN); + Assertions.assertEquals(configured, NereidsStreamLoadTask.fromTStreamLoadPutRequest(request) + .isCloudMemtableSinkUpload()); + for (boolean requested : new boolean[] {false, true}) { + request.setCloudMemtableSinkUpload(requested); + Assertions.assertEquals(requested, NereidsStreamLoadTask.fromTStreamLoadPutRequest(request) + .isCloudMemtableSinkUpload()); + } + } + } finally { + Config.cloud_stream_load_default_memtable_sink_upload = original; + } + } + @Test public void testExprIdGenerator() { IdGenerator exprIdGenerator1 = StatementScopeIdGenerator.getExprIdGenerator(); diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 55c64d302a03a5..b17b90680c8dff 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -1020,6 +1020,9 @@ message POpenLoadStreamRequest { optional bool enable_profile = 6 [default = false]; optional int64 total_streams = 7; optional int64 idle_timeout_ms = 8; + optional int64 txn_expiration = 9; + optional string storage_vault_id = 10; + optional bool write_file_cache = 11; } message PTabletSchemaWithIndex { @@ -1028,6 +1031,27 @@ message PTabletSchemaWithIndex { optional bool enable_unique_key_merge_on_write = 3; } +// All writers of one MOW load compare against the same rowsets and deletion state. +message PCloudLoadMowSnapshot { + optional int64 version = 1; + repeated RowsetMetaPB rowsets = 2; + optional DeleteBitmapPB delete_bitmap = 3; +} + +// A writer owns one stable segment range for the lifetime of a load attempt. +message PCloudLoadWriteContext { + optional string writer_id = 1; + optional RowsetMetaPB rowset_meta = 2; + optional int32 segment_start_id = 3; + optional int32 segment_capacity = 4; + optional EncryptionAlgorithmPB encrypt_algorithm = 5; + // Absolute cache expiration timestamp in seconds; 0 means no TTL. + optional int64 file_cache_expiration_time = 6; + // The target can asynchronously cache completed sink-upload rowsets. + optional bool warm_up_file_cache = 7 [default = false]; + optional PCloudLoadMowSnapshot mow_snapshot = 8; +} + message POpenLoadStreamResponse { optional PStatus status = 1; repeated PTabletSchemaWithIndex tablet_schemas = 2; @@ -1047,6 +1071,12 @@ message PLoadStreamResponse { repeated PTabletSchemaWithIndex tablet_schemas = 5; optional bool eos = 6; repeated PTabletLoadRowsetInfo tablet_load_rowset_num_infos = 7; + optional PCloudLoadWriteContext write_context = 8; +} + +message PCloudLoadMowResult { + optional int64 snapshot_version = 1; + optional DeleteBitmapPB delete_bitmap = 2; } message PStreamHeader { @@ -1055,6 +1085,8 @@ message PStreamHeader { CLOSE_LOAD = 2; ADD_SEGMENT = 3; GET_SCHEMA = 4; + GET_WRITE_CONTEXT = 5; + ADD_PARTIAL_ROWSET = 6; } optional PUniqueId load_id = 1; optional int64 partition_id = 2; @@ -1070,6 +1102,9 @@ message PStreamHeader { optional uint64 offset = 12; optional FileType file_type = 13; optional int64 num_incremental_streams = 14; + optional string writer_id = 15; + optional RowsetMetaPB partial_rowset_meta = 16; + optional PCloudLoadMowResult mow_result = 17; } message PGetWalQueueSizeRequest{ diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index fb3f4c3860b8ba..5b207f1062200a 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -632,6 +632,7 @@ struct TStreamLoadPutRequest { // For cloud 1000: optional string cloud_cluster 1001: optional i64 table_id + 1002: optional bool cloud_memtable_sink_upload } struct TStreamLoadPutResult { diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 3f80d4ae5a6557..eb6d24f20ec3a3 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -540,6 +540,7 @@ struct TQueryOptions { // reads the data instead. // Defaults to false because an old FE never sends this field, and BE checked both cases before. 1006: optional bool force_pushdown_zonemap_minmax = false + 1007: optional bool enable_cloud_memtable_sink_upload = true; } diff --git a/regression-test/data/cloud_p0/test_cloud_memtable_group_commit.out b/regression-test/data/cloud_p0/test_cloud_memtable_group_commit.out new file mode 100644 index 00000000000000..14525f0ad631ca --- /dev/null +++ b/regression-test/data/cloud_p0/test_cloud_memtable_group_commit.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !response -- +Success true async_mode true 2 2 0 + +-- !pending_rows -- +1 10 +2 20 + +-- !rows -- +1 100 +2 20 +3 30 + diff --git a/regression-test/data/cloud_p0/test_cloud_memtable_mow_forward.out b/regression-test/data/cloud_p0/test_cloud_memtable_mow_forward.out new file mode 100644 index 00000000000000..6f0e9ba4e9beb3 --- /dev/null +++ b/regression-test/data/cloud_p0/test_cloud_memtable_mow_forward.out @@ -0,0 +1,109 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !initial_false -- +0 119960 11997 +1 119970 11998 +2 119980 11999 +3 119990 12000 + +-- !index_false -- +1 119970 11998 +3 119990 12000 + +-- !layout_false -- +true false true true true true + +-- !deleted_false -- +0 119960 11997 +1 119970 11998 +3 119990 12000 + +-- !stream_status_false -- +Success + +-- !stream_false -- +0 150000 15000 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !group_status_false -- +Success true + +-- !group_false -- +0 150000 15000 +1 160000 16000 +2 140000 14000 +3 119990 12000 + +-- !compacted_false -- +0 150000 15000 +1 160000 16000 +2 140000 14000 +3 119990 12000 + +-- !initial_true -- +0 119960 11997 +1 119970 11998 +2 119980 11999 +3 119990 12000 + +-- !index_true -- +1 119970 11998 +3 119990 12000 + +-- !layout_true -- +true true true true true true + +-- !deleted_true -- +0 119960 11997 +1 119970 11998 +3 119990 12000 + +-- !stream_status_true -- +Success + +-- !stream_true -- +0 150000 15000 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !group_status_true -- +Success true + +-- !group_true -- +0 150000 15000 +1 160000 16000 +2 140000 14000 +3 119990 12000 + +-- !compacted_true -- +0 150000 15000 +1 160000 16000 +2 140000 14000 +3 119990 12000 + +-- !mow_fallback -- +1 20 + +-- !broker_rows -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 100 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + diff --git a/regression-test/data/cloud_p0/test_cloud_memtable_on_sink.out b/regression-test/data/cloud_p0/test_cloud_memtable_on_sink.out new file mode 100644 index 00000000000000..ce3cc76caba9fc --- /dev/null +++ b/regression-test/data/cloud_p0/test_cloud_memtable_on_sink.out @@ -0,0 +1,278 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !s3_rows_false -- +20 1133 2266 + +-- !s3_index_false -- +31 62 +50 100 +50 100 +57 114 + +-- !s3_packed_meta_false -- +6 20 2 1 1 0 0,1,2,3,4,5 + +-- !s3_rows_true -- +20 1133 2266 + +-- !s3_index_true -- +31 62 +50 100 +50 100 +57 114 + +-- !s3_packed_meta_true -- +6 20 2 1 1 4 0,1,1000,1001,2000,2001 + +-- !stream_rows -- +22 1334 2668 + +-- !failed_upload_rows -- +22 1334 2668 + +-- !agg_false -- +0 17994000 0 11996 3000 1 10 20 +1 17997000 1 11997 3000 1 11 21 +2 18000000 2 11998 3000 1 12 22 +3 18003000 3 11999 3000 1 13 23 + +-- !mor_false -- +0 0 +1 10 +2 20 +3 30 + +-- !seq_false -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !seq_index_false -- +1 119970 11997 +3 119990 11999 + +-- !seq_lower_false -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !mor_new_version_false -- +0 99 +1 10 +2 20 +3 30 + +-- !agg_new_version_false -- +0 17994010 -1 13000 3000 1 99 20 +1 17997000 1 11997 3000 1 11 21 +2 18000000 2 11998 3000 1 12 22 +3 18003000 3 11999 3000 1 13 23 + +-- !seq_empty_false -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !agg_compacted_false -- +0 17994010 -1 13000 3000 1 99 20 +1 17997000 1 11997 3000 1 11 21 +2 18000000 2 11998 3000 1 12 22 +3 18003000 3 11999 3000 1 13 23 + +-- !mor_compacted_false -- +0 99 +1 10 +2 20 +3 30 + +-- !seq_compacted_false -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !agg_broker_false -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 200 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + +-- !mor_broker_false -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 100 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + +-- !layout_agg -- +true true true true true + +-- !layout_mor -- +true true true true true + +-- !layout_mor_seq -- +true true true true true + +-- !agg_true -- +0 17994000 0 11996 3000 1 10 20 +1 17997000 1 11997 3000 1 11 21 +2 18000000 2 11998 3000 1 12 22 +3 18003000 3 11999 3000 1 13 23 + +-- !mor_true -- +0 0 +1 10 +2 20 +3 30 + +-- !seq_true -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !seq_index_true -- +1 119970 11997 +3 119990 11999 + +-- !seq_lower_true -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !mor_new_version_true -- +0 99 +1 10 +2 20 +3 30 + +-- !agg_new_version_true -- +0 17994010 -1 13000 3000 1 99 20 +1 17997000 1 11997 3000 1 11 21 +2 18000000 2 11998 3000 1 12 22 +3 18003000 3 11999 3000 1 13 23 + +-- !seq_empty_true -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !agg_compacted_true -- +0 17994010 -1 13000 3000 1 99 20 +1 17997000 1 11997 3000 1 11 21 +2 18000000 2 11998 3000 1 12 22 +3 18003000 3 11999 3000 1 13 23 + +-- !mor_compacted_true -- +0 99 +1 10 +2 20 +3 30 + +-- !seq_compacted_true -- +0 119960 11996 +1 119970 11997 +2 119980 11998 +3 119990 11999 + +-- !agg_broker_true -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 200 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + +-- !mor_broker_true -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 100 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + +-- !agg_stream -- +50 203 + +-- !mor_stream -- +0 119960 11996 +1 130010 13001 +2 119980 11998 +3 119990 11999 + +-- !mor_delete -- +0 119960 11996 +1 130010 13001 +3 119990 11999 + +-- !mor_reinsert -- +0 119960 11996 +1 130010 13001 +2 150000 15000 +3 119990 11999 + +-- !mor_final_compacted -- +0 119960 11996 +1 130010 13001 +2 150000 15000 +3 119990 11999 + diff --git a/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_cache.out b/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_cache.out new file mode 100644 index 00000000000000..3fd2f58bdbb155 --- /dev/null +++ b/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_cache.out @@ -0,0 +1,41 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !load_plain_remote -- +Success 2 0 + +-- !layout_plain_remote -- +1 false + +-- !cache_plain_remote -- +true true + +-- !load_plain_local -- +Success 2 0 + +-- !layout_plain_local -- +1 false + +-- !cache_plain_local -- +true true + +-- !load_packed_remote -- +Success 2 0 + +-- !layout_packed_remote -- +1 true + +-- !cache_packed_remote -- +true true + +-- !load_packed_local -- +Success 2 0 + +-- !layout_packed_local -- +1 true + +-- !cache_packed_local -- +true true + +-- !rows -- +1 40 4 +2 80 4 + diff --git a/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_mow.out b/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_mow.out new file mode 100644 index 00000000000000..f52daf0cf0ab28 --- /dev/null +++ b/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_mow.out @@ -0,0 +1,147 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !initial -- +0 0 +1 10 +2 20 +3 30 + +-- !initial_seq -- +0 119960 11997 +1 119970 11998 +2 119980 11999 +3 119990 12000 + +-- !index -- +1 119970 11998 +3 119990 12000 + +-- !layout_cloud_mow_direct -- +true true true true + +-- !layout_cloud_mow_direct_seq -- +true true true true + +-- !lower_seq -- +0 119960 11997 +1 119970 11998 +2 119980 11999 +3 119990 12000 + +-- !new_version -- +0 99 +1 10 +2 20 +3 30 + +-- !empty -- +0 119960 11997 +1 119970 11998 +2 119980 11999 +3 119990 12000 + +-- !delete -- +0 119960 11997 +1 119970 11998 +3 119990 12000 + +-- !lower_after_delete -- +0 119960 11997 +1 119970 11998 +3 119990 12000 + +-- !reinsert -- +0 119960 11997 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !compacted -- +0 99 +1 10 +2 20 +3 30 + +-- !compacted_seq -- +0 119960 11997 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !stream -- +0 150000 15000 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !concurrent_compaction -- +0 210000 21000 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !failed_invisible -- +0 210000 21000 +1 119970 11998 +2 140000 14000 +3 119990 12000 + +-- !cluster -- +0 -11996 11997 +1 -11997 11998 +2 -11998 11999 +3 -11999 12000 + +-- !cluster_update -- +0 999 20000 +1 -11997 11998 +2 -11998 11999 +3 -11999 12000 + +-- !cluster_compacted -- +0 999 20000 +1 -11997 11998 +2 -11998 11999 +3 -11999 12000 + +-- !broker_true -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 100 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + +-- !broker_false -- +20 40 +21 42 +31 62 +41 82 +49 98 +50 100 +57 114 +58 116 +60 120 +62 124 +65 130 +66 132 +68 136 +8 16 +80 160 +81 162 +85 170 +90 180 +91 182 + diff --git a/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_unshared.out b/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_unshared.out new file mode 100644 index 00000000000000..7829dc48776f74 --- /dev/null +++ b/regression-test/data/cloud_p0/test_cloud_memtable_sink_upload_unshared.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !rows -- +100000 4999950000 9999900000 + +-- !rowset_layout -- +true true true 100000 + +-- !empty_input -- +100000 4999950000 9999900000 + +-- !second_load -- +100010 4999950045 9999900090 + diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_group_commit.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_group_commit.groovy new file mode 100644 index 00000000000000..c45db6eed22a66 --- /dev/null +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_group_commit.groovy @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.suite.client.BackendClientImpl +import org.apache.doris.thrift.TGetRealtimeExecStatusRequest +import org.apache.doris.thrift.TNetworkAddress +import org.apache.doris.thrift.TStatusCode +import org.apache.doris.thrift.TUniqueId + +suite("test_cloud_memtable_group_commit", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(3) + options.enableDebugPoints() + options.feConfigs += ['stream_load_default_memtable_on_sink_node=true', + 'cloud_stream_load_default_memtable_sink_upload=true'] + docker(options) { + sql "DROP TABLE IF EXISTS cloud_memtable_group_commit" + sql """ + CREATE TABLE cloud_memtable_group_commit (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "group_commit_interval_ms"="200", "disable_auto_compaction"="true") + """ + sql "INSERT INTO cloud_memtable_group_commit VALUES (1,10),(2,20)" + // Internal Group Commit plans use a new session, not this SQL connection's variables. + sql "SET GLOBAL enable_profile=true" + sql "SET GLOBAL profile_level=2" + def tablet = sql_return_maparray("SHOW TABLETS FROM cloud_memtable_group_commit")[0] + def backends = sql_return_maparray("SHOW BACKENDS") + def block = "VTabletWriterV2.close.profile_ready" + try { + GetDebugPoint().enableDebugPointForAllBEs(block, [timeout: "120"]) + def label = "" + streamLoad { + table "cloud_memtable_group_commit" + set "column_separator", "," + set "group_commit", "async_mode" + // The internal load must use the FE defaults rather than this outer request's flag. + set "memtable_on_sink_node", "false" + set "cloud_memtable_sink_upload", "false" + unset "label" + inputText "1,100\n3,30\n" + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + logger.info("Cloud memtable Group Commit response: {}", result) + def json = parseJson(result) + label = json.Label + order_qt_response """ + SELECT '${json.Status}', ${json.GroupCommit == true}, '${json.GroupCommitMode}', + ${label.startsWith('group_commit_')}, ${json.NumberTotalRows}, + ${json.NumberLoadedRows}, ${json.NumberFilteredRows} + """ + } + } + def idParts = label.substring('group_commit_'.length()).split('_') + def queryId = new TUniqueId(Long.parseUnsignedLong(idParts[0], 16), + Long.parseUnsignedLong(idParts[1], 16)) + def request = new TGetRealtimeExecStatusRequest().setId(queryId).setReqType("profile") + // Probe the BEs for this internal query; its writer remains alive at the debug point. + awaitUntil(60) { + backends.any { be -> + def backend = new BackendClientImpl( + new TNetworkAddress(be.Host, be.BePort as int), be.HttpPort as int) + try { + backend.client.inputProtocol.transport.setTimeout(5000) + def response = backend.client.getRealtimeExecStatus(request) + if (response.status.statusCode == TStatusCode.NOT_FOUND) { + return false + } + if (response.status.statusCode != TStatusCode.OK) { + throw new IllegalStateException("Get BE profile failed: ${response.status}") + } + def profile = response.reportExecStatusParams.queryProfile + def nodes = profile.fragmentIdToProfile.values().flatten() + .collectMany { it.profile.nodes } + return nodes.any { it.name == "DeltaWriterV2 ${tablet.TabletId}" } && + nodes.any { + it.infoStrings['CloudMemtableSinkUpload'] == 'true' && + it.infoStrings['CloudMemtableMowBitmap'] == 'true' + } + } finally { + backend.close() + } + } + } + order_qt_pending_rows "SELECT * FROM cloud_memtable_group_commit" + } finally { + GetDebugPoint().disableDebugPointForAllBEs(block) + } + // Publication can only finish after releasing the writer. + awaitUntil(60) { + (sql "SELECT COUNT(*) FROM cloud_memtable_group_commit")[0][0] == 3 + } + order_qt_rows "SELECT * FROM cloud_memtable_group_commit" + } +} diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_mow_forward.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_mow_forward.groovy new file mode 100644 index 00000000000000..1fc69b1b97754d --- /dev/null +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_mow_forward.groovy @@ -0,0 +1,267 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.suite.client.BackendClientImpl +import org.apache.doris.thrift.TGetRealtimeExecStatusRequest +import org.apache.doris.thrift.TNetworkAddress +import org.apache.doris.thrift.TStatusCode +import org.apache.doris.thrift.TUniqueId + +suite("test_cloud_memtable_mow_forward", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(3) + options.enableDebugPoints() + options.beConfigs += ['small_file_threshold_bytes=1048576', + 'enable_merge_on_write_correctness_check=true'] + options.feConfigs += ['stream_load_default_memtable_on_sink_node=true', + 'cloud_stream_load_default_memtable_sink_upload=false'] + docker(options) { + def backends = sql_return_maparray("SHOW BACKENDS") + def awaitGroupProfile = { label, tabletId -> + def idParts = label.substring('group_commit_'.length()).split('_') + def queryId = new TUniqueId(Long.parseUnsignedLong(idParts[0], 16), + Long.parseUnsignedLong(idParts[1], 16)) + def request = new TGetRealtimeExecStatusRequest().setId(queryId).setReqType("profile") + awaitUntil(60) { + backends.any { be -> + def backend = new BackendClientImpl( + new TNetworkAddress(be.Host, be.BePort as int), be.HttpPort as int) + try { + backend.client.inputProtocol.transport.setTimeout(5000) + def response = backend.client.getRealtimeExecStatus(request) + if (response.status.statusCode == TStatusCode.NOT_FOUND) { + return false + } + if (response.status.statusCode != TStatusCode.OK) { + throw new IllegalStateException("Get BE profile failed: ${response.status}") + } + def nodes = response.reportExecStatusParams.queryProfile.fragmentIdToProfile + .values().flatten().collectMany { it.profile.nodes } + return nodes.any { it.name == "DeltaWriterV2 ${tabletId}" } + } finally { + backend.close() + } + } + } + } + def checkUniqueKeys = { + sql """ + SELECT assert_true(COUNT(*) = 0, 'cloud_mow_forward contains duplicate keys') FROM ( + SELECT k, COUNT(*) AS a FROM cloud_mow_forward GROUP BY k HAVING a > 1 + ) duplicates + """ + } + + sql "DROP TABLE IF EXISTS cloud_mow_forward_source" + sql """ + CREATE TABLE cloud_mow_forward_source (n BIGINT NOT NULL) + DUPLICATE KEY(n) DISTRIBUTED BY HASH(n) BUCKETS 12 + PROPERTIES ("replication_num"="1") + """ + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_mow_forward_source SELECT number FROM numbers('number'='12000')" + sql "SET enable_cloud_memtable_sink_upload=false" + sql "SET parallel_pipeline_task_num=4" + sql "SET profile_level=2" + sql "SET enable_sql_cache=false" + sql "SET enable_file_cache=false" + // Any accidental switch to sink upload must fail before its result is committed. + GetDebugPoint().enableDebugPointForAllBEs("DeltaWriterV2.sink_upload.after_upload_failure") + try { + [false, true].each { packed -> + setBeConfigTemporary(['enable_packed_file': packed.toString()]) { + sql "DROP TABLE IF EXISTS cloud_mow_forward" + sql """ + CREATE TABLE cloud_mow_forward ( + k BIGINT NOT NULL, v BIGINT, seq BIGINT NOT NULL, + INDEX idx_k(k) USING INVERTED + ) UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "function_column.sequence_col"="seq", "inverted_index_storage_format"="V2", + "disable_auto_compaction"="true", "group_commit_interval_ms"="200") + """ + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_mow_forward VALUES (0,-1,-1),(1,-1,-1),(2,-1,-1),(3,-1,-1)" + sql "INSERT INTO cloud_mow_forward VALUES (0,0,0),(1,0,0),(2,0,0),(3,0,0)" + sql "SET enable_memtable_on_sink_node=true" + sql "SET enable_profile=true" + sql """ + /* cloud_mow_forward_${packed} */ + INSERT INTO cloud_mow_forward SELECT n%4,n*10,n+1 FROM cloud_mow_forward_source + """ + new ProfileAction(context).getProfileBySql("cloud_mow_forward_${packed}", ["DeltaWriterV2"]) + sql "SET enable_profile=false" + quickTest("initial_${packed}", "SELECT * FROM cloud_mow_forward", true) + checkUniqueKeys() + quickTest("index_${packed}", "SELECT * FROM cloud_mow_forward WHERE k IN (1,3)", true) + def tablet = sql_return_maparray("SHOW TABLETS FROM cloud_mow_forward")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM cloud_mow_forward")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + code, body -> + assertEquals(200, code) + def meta = parseJson(body) + quickTest("layout_${packed}", """ + SELECT ${meta.num_segments as int} > 1, + ${(meta.packed_slice_locations ?: [:]).size() > 0}, + ${meta.segments_file_size.size() == (meta.num_segments as int)}, + ${meta.segments_file_size.every { (it as long) > 0 }}, + ${meta.inverted_index_file_info.size() == (meta.num_segments as int)}, + ${meta.inverted_index_file_info.every { (it.index_size as long) > 0 }} + """, true) + } + sql "INSERT INTO cloud_mow_forward VALUES (0,-1,1)" + sql "INSERT INTO cloud_mow_forward (k,v,seq,__DORIS_DELETE_SIGN__) VALUES (2,130000,13000,1)" + sql "INSERT INTO cloud_mow_forward VALUES (2,-1,1)" + quickTest("deleted_${packed}", "SELECT * FROM cloud_mow_forward", true) + sql "INSERT INTO cloud_mow_forward VALUES (2,140000,14000)" + streamLoad { + table "cloud_mow_forward" + set "column_separator", "," + set "memtable_on_sink_node", "true" + set "group_commit", "off_mode" + inputText "0,150000,15000\n3,-1,1\n" + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + sql "SELECT assert_true('${response.Status}' = 'Success', 'forwarded Stream Load failed')" + quickTest("stream_status_${packed}", "SELECT '${response.Status}'", true) + } + } + quickTest("stream_${packed}", "SELECT * FROM cloud_mow_forward", true) + def originalEnableProfile = sql("SHOW GLOBAL VARIABLES LIKE 'enable_profile'")[0][1] + def originalProfileLevel = sql("SHOW GLOBAL VARIABLES LIKE 'profile_level'")[0][1] + def profileBlock = "VTabletWriterV2.close.profile_ready" + def groupLabel = "" + try { + sql "SET GLOBAL enable_profile=true" + sql "SET GLOBAL profile_level=2" + GetDebugPoint().enableDebugPointForAllBEs(profileBlock, [timeout: "120"]) + streamLoad { + table "cloud_mow_forward" + set "column_separator", "," + set "group_commit", "async_mode" + set "memtable_on_sink_node", "false" + set "cloud_memtable_sink_upload", "true" + unset "label" + inputText "1,160000,16000\n" + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + sql "SELECT assert_true('${response.Status}' = 'Success', 'Group Commit Stream Load failed')" + groupLabel = response.Label + quickTest("group_status_${packed}", "SELECT '${response.Status}', '${response.GroupCommit}'", true) + } + } + awaitGroupProfile(groupLabel, tablet.TabletId) + } finally { + GetDebugPoint().disableDebugPointForAllBEs(profileBlock) + sql "SET GLOBAL enable_profile=${originalEnableProfile}" + sql "SET GLOBAL profile_level=${originalProfileLevel}" + } + awaitUntil(60) { + (sql "SELECT v FROM cloud_mow_forward WHERE k=1")[0][0] == 160000 + } + quickTest("group_${packed}", "SELECT * FROM cloud_mow_forward", true) + checkUniqueKeys() + trigger_and_wait_compaction("cloud_mow_forward", "full") + quickTest("compacted_${packed}", "SELECT * FROM cloud_mow_forward", true) + checkUniqueKeys() + } + } + // MOW without sink upload forwards files and calculates bitmaps on the target BE. + sql "DROP TABLE IF EXISTS cloud_memtable_mow_fallback" + sql """ + CREATE TABLE cloud_memtable_mow_fallback (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true") + """ + sql "INSERT INTO cloud_memtable_mow_fallback VALUES (1,10)" + sql "INSERT INTO cloud_memtable_mow_fallback VALUES (1,20)" + order_qt_mow_fallback "SELECT * FROM cloud_memtable_mow_fallback" + + sql "DROP TABLE IF EXISTS cloud_mow_forward_broker" + sql """ + CREATE TABLE cloud_mow_forward_broker (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "disable_auto_compaction"="true") + """ + sql "INSERT INTO cloud_mow_forward_broker VALUES (50,-1)" + def brokerLabel = "mow_forward_" + UUID.randomUUID().toString().replace('-', '_') + sql "SET enable_profile=true" + try { + sql """ + LOAD LABEL ${brokerLabel} ( + DATA INFILE("s3://${getS3BucketName()}/regression/load/data/basic_data.csv") + INTO TABLE cloud_mow_forward_broker COLUMNS TERMINATED BY "|" FORMAT AS "CSV" + (k, c01, c02, c03, c04, c05, c06, c07, c08, c09, + c10, c11, c12, c13, c14, c15, c16, c17, c18) SET (v=k*2) + ) WITH S3 ( + "AWS_ACCESS_KEY"="${getS3AK()}", "AWS_SECRET_KEY"="${getS3SK()}", + "AWS_ENDPOINT"="${getS3Endpoint()}", "AWS_REGION"="${getS3Region()}", + "provider"="${getS3Provider()}" + ) PROPERTIES ("load_parallelism"="1") + """ + waitForBrokerLoadDone(brokerLabel) + def load = sql_return_maparray("SHOW LOAD WHERE LABEL = '${brokerLabel}'")[0] + new ProfileAction(context).getProfile(load.JobId.toString(), ["DeltaWriterV2"]) + } finally { + sql "SET enable_profile=false" + } + sql """ + SELECT assert_true(COUNT(*) = 0, 'cloud_mow_forward_broker contains duplicate keys') FROM ( + SELECT k, COUNT(*) AS a FROM cloud_mow_forward_broker GROUP BY k HAVING a > 1 + ) duplicates + """ + order_qt_broker_rows "SELECT k,v FROM cloud_mow_forward_broker" + + sql "DROP TABLE IF EXISTS cloud_memtable_partial_update" + sql """ + CREATE TABLE cloud_memtable_partial_update (k INT NOT NULL, v INT, untouched INT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true") + """ + sql "INSERT INTO cloud_memtable_partial_update VALUES (1,10,100)" + streamLoad { + table "cloud_memtable_partial_update" + set "column_separator", "," + set "columns", "k,v" + set "partial_columns", "true" + set "memtable_on_sink_node", "true" + set "cloud_memtable_sink_upload", "false" + set "group_commit", "off_mode" + inputText "1,20\n" + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + sql "SELECT assert_true('${response.Status}' = 'Success', 'partial-update Stream Load failed')" + } + } + sql """ + SELECT assert_true(COUNT(*) = 1 AND SUM(v) = 20 AND SUM(untouched) = 100, 'partial update result mismatch') + FROM cloud_memtable_partial_update WHERE k = 1 + """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("DeltaWriterV2.sink_upload.after_upload_failure") + } + } +} diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_on_sink.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_on_sink.groovy new file mode 100644 index 00000000000000..aee48339813798 --- /dev/null +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_on_sink.groovy @@ -0,0 +1,553 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction +import org.apache.doris.regression.suite.ClusterOptions + +suite("test_cloud_memtable_on_sink", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(3) + options.enableDebugPoints() + // Split the small S3 CSV across all three BEs with load_parallelism = 1. + options.feConfigs += ['min_bytes_per_broker_scanner = 100'] + // The segment-count checks require the configured four-row CSV batches. + options.beConfigs += ['enable_packed_file=true', 'small_file_threshold_bytes=1048576', + 'enable_adaptive_batch_size=false'] + + // Ordinary Stream Load inherits the FE default unless its header overrides it. + options.feConfigs += ['cloud_stream_load_default_memtable_sink_upload = true'] + docker(options) { + def loadS3 = { table, label -> + sql """ + LOAD LABEL ${label} ( + DATA INFILE("s3://${getS3BucketName()}/regression/load/data/basic_data.csv") + INTO TABLE ${table} COLUMNS TERMINATED BY "|" FORMAT AS "CSV" + (k, c01, c02, c03, c04, c05, c06, c07, c08, c09, + c10, c11, c12, c13, c14, c15, c16, c17, c18) + SET (v = k * 2) + ) WITH S3 ( + "AWS_ACCESS_KEY"="${getS3AK()}", "AWS_SECRET_KEY"="${getS3SK()}", + "AWS_ENDPOINT"="${getS3Endpoint()}", "AWS_REGION"="${getS3Region()}", + "provider"="${getS3Provider()}" + ) PROPERTIES ("load_parallelism"="1") + """ + waitForBrokerLoadDone(label) + return sql_return_maparray("SHOW LOAD WHERE LABEL = '${label}'")[0] + } + + def testDuplicate = { + [false, true].each { sinkUpload -> + sql "SET enable_cloud_memtable_sink_upload = ${sinkUpload}" + if (sinkUpload) { + GetDebugPoint().enableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + GetDebugPoint().enableDebugPointForAllBEs("DeltaWriterV2.sink_upload.duplicate_result") + } + + sql "SET enable_sql_cache = false" + sql "DROP TABLE IF EXISTS test_cloud_duplicate_memtable_on_sink_source" + sql "DROP TABLE IF EXISTS test_cloud_duplicate_memtable_on_sink" + + sql """ + CREATE TABLE test_cloud_duplicate_memtable_on_sink_source ( + k BIGINT NOT NULL, + v BIGINT NOT NULL + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 3 + PROPERTIES ("replication_num" = "1") + """ + sql """ + CREATE TABLE test_cloud_duplicate_memtable_on_sink ( + k BIGINT NOT NULL, + v BIGINT NOT NULL + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + + sql "SET enable_memtable_on_sink_node = false" + sql """ + INSERT INTO test_cloud_duplicate_memtable_on_sink_source + SELECT number, number * 2 FROM numbers("number" = "100000") + """ + + try { + sql "SET enable_memtable_on_sink_node = true" + sql "SET profile_level = 2" + sql "SET enable_profile = true" + sql """ + /* cloud_duplicate_memtable_on_sink_profile_${sinkUpload} */ + INSERT INTO test_cloud_duplicate_memtable_on_sink + SELECT k, v FROM test_cloud_duplicate_memtable_on_sink_source + """ + def required = ["DeltaWriterV2"] + if (sinkUpload) { + required += "CloudMemtableSinkUpload: true" + } + def profileString = new ProfileAction(context).getProfileBySql( + "cloud_duplicate_memtable_on_sink_profile_${sinkUpload}", required) + logger.info("memtable-on-sink profile:\n{}", profileString) + } finally { + sql "SET enable_profile = false" + sql "SET enable_memtable_on_sink_node = false" + } + + sql """ + SELECT assert_true( + COUNT(*) = 100000 + AND SUM(k) = 4999950000 + AND SUM(v) = 9999900000, + 'cloud duplicate memtable-on-sink result mismatch') + FROM test_cloud_duplicate_memtable_on_sink + """ + + sql "DROP TABLE IF EXISTS test_cloud_duplicate_memtable_on_sink_s3" + sql """ + CREATE TABLE test_cloud_duplicate_memtable_on_sink_s3 ( + k BIGINT NOT NULL, + v BIGINT NOT NULL, + INDEX idx_v (v) USING INVERTED + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2", + "disable_auto_compaction" = "true" + ) + """ + + def label = "cloud_duplicate_memtable_on_sink_s3_" + UUID.randomUUID().toString().replace('-', '_') + try { + sql "SET enable_memtable_on_sink_node = true" + sql "SET enable_profile = true" + // The three scanners read 6, 6, and 8 rows. Flush after the first four rows on + // each BE, then flush the remaining rows at close to produce two segments each. + sql "SET broker_load_batch_size = 4" + GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush", [execute: 1]) + def load = loadS3("test_cloud_duplicate_memtable_on_sink_s3", label) + assertEquals("FINISHED", load.State, "S3 load did not finish: ${load}") + + def profileString = new ProfileAction(context).getProfile( + load.JobId.toString(), ["DeltaWriterV2", "NumScanners"]) + logger.info("S3 memtable-on-sink profile:\n{}", profileString) + + // Only inspect per-BE pipelines, excluding the merged profile's duplicate counters. + def pipelines = profileString.split(/(?m)(?=^[ \t]*(?:Pipeline \d+|FragmentLevelProfile:)\(host=)/) + .findAll { it.trim().startsWith("Pipeline ") } + def backends = sql_return_maparray("SHOW BACKENDS") + backends.each { backend -> + def pipeline = pipelines.find { + it.readLines()[0].contains("hostname:${backend.Host},") && it.contains("DeltaWriterV2") + } + assertNotNull(pipeline, "Missing S3 sink writer on BE ${backend.Host}") + assertTrue(pipeline.contains("FILE_SCAN_OPERATOR"), "Missing S3 scanner on BE ${backend.Host}") + assertTrue((pipeline =~ /(?m)^\s*- NumScanners: 1\s*$/).find(), + "Expected one S3 scanner on BE ${backend.Host}") + assertTrue((pipeline =~ /(?m)^\s*- SegmentNum: 2\s*$/).find(), + "Expected two flushed segments on BE ${backend.Host}") + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs("MemTable.need_flush") + sql "SET enable_profile = false" + sql "SET enable_memtable_on_sink_node = false" + } + + // Read both segment data and V2 indexes without the uploader's file cache. + sql "SET enable_file_cache = false" + quickTest("s3_rows_${sinkUpload}", """ + SELECT COUNT(*), SUM(k), SUM(v) + FROM test_cloud_duplicate_memtable_on_sink_s3 + """, true) + quickTest("s3_index_${sinkUpload}", """ + SELECT k, v FROM test_cloud_duplicate_memtable_on_sink_s3 WHERE v IN (62, 100, 114) + """, true) + def tablet = sql_return_maparray("SHOW TABLETS FROM test_cloud_duplicate_memtable_on_sink_s3")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM test_cloud_duplicate_memtable_on_sink_s3")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + responseCode, body -> + assertEquals(200, responseCode) + logger.info("S3 memtable-on-sink rowset meta: {}", body) + def rowsetMeta = parseJson(body) + def locations = rowsetMeta.packed_slice_locations + def segmentIds = rowsetMeta.segment_ids ?: (0..<(rowsetMeta.num_segments as int)).toList() + logger.info("S3 rowset layout (sinkUpload={}): {}", sinkUpload, [ + tablet_id: tablet.TabletId, version: partition.VisibleVersion, + rowset_id: rowsetMeta.rowset_id_v2, segment_ids: segmentIds, + num_segment_rows: rowsetMeta.num_segment_rows, + segments_file_size: rowsetMeta.segments_file_size, + packed_files: locations.keySet().sort() + ]) + // Only the destination rowset's first segment and its V2 index are packed. + // The other five segments retain independent files. + quickTest("s3_packed_meta_${sinkUpload}", """ + SELECT ${rowsetMeta.num_segments as int}, ${rowsetMeta.num_rows as long}, + ${locations.size()}, + ${locations.keySet().count { it.endsWith('_0.dat') }}, + ${locations.keySet().count { it.endsWith('_0.idx') }}, + ${segmentIds.count { (it as long) >= 1000 }}, '${segmentIds.join(",")}' + """, true) + } + } + + GetDebugPoint().disableDebugPointForAllBEs("DeltaWriterV2.sink_upload.duplicate_result") + + // Stream Load inherits the FE sink-upload default independently of SQL session options. + // This hook is only reached by the memtable-on-sink receiving writer. + // A successful fallback would fail this check instead of silently passing. + def failClose = "LoadStreamWriter.close.cancelled" + try { + GetDebugPoint().enableDebugPointForAllBEs(failClose) + streamLoad { + table "test_cloud_duplicate_memtable_on_sink_s3" + set "column_separator", "," + set "memtable_on_sink_node", "true" + set "group_commit", "off_mode" + inputText "100,200\n101,202\n" + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + sql "SELECT assert_true('${response.Status}' = 'Fail', 'Stream Load should fail at receiver close')" + } + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs(failClose) + } + sql """ + SELECT assert_true(COUNT(*) = 20 AND SUM(k) = 1133 AND SUM(v) = 2266, 'failed Stream Load changed visible rows') + FROM test_cloud_duplicate_memtable_on_sink_s3 + """ + + def failUpload = "DeltaWriterV2.sink_upload.after_upload_failure" + try { + GetDebugPoint().enableDebugPointForAllBEs(failUpload) + [null, "true"].each { sinkUploadHeader -> + streamLoad { + table "test_cloud_duplicate_memtable_on_sink_s3" + set "column_separator", "," + set "memtable_on_sink_node", "true" + if (sinkUploadHeader != null) { + set "cloud_memtable_sink_upload", sinkUploadHeader + } + set "group_commit", "off_mode" + inputText "100,200\n101,202\n" + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + sql "SELECT assert_true('${response.Status}' = 'Fail', 'Stream Load should reach sink upload')" + assertTrue(response.Message.contains("injected failure after sink upload")) + } + } + } + // Allow file bytes while keeping the sink-upload failure hook enabled. + GetDebugPoint().disableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + streamLoad { + table "test_cloud_duplicate_memtable_on_sink_s3" + set "column_separator", "," + set "memtable_on_sink_node", "true" + set "cloud_memtable_sink_upload", "false" + set "group_commit", "off_mode" + inputStream new ByteArrayInputStream("100,200\n101,202\n".getBytes()) + time 30000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + assertEquals("Success", parseJson(result).Status) + } + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs(failUpload) + GetDebugPoint().enableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + } + order_qt_stream_rows """ + SELECT COUNT(*), SUM(k), SUM(v) FROM test_cloud_duplicate_memtable_on_sink_s3 + """ + + sql "SET enable_memtable_on_sink_node = true" + sql "SET enable_cloud_memtable_sink_upload = true" + // An uploaded partial rowset must remain invisible if its result cannot be sent. + try { + GetDebugPoint().enableDebugPointForAllBEs("DeltaWriterV2.sink_upload.after_upload_failure") + test { + sql "INSERT INTO test_cloud_duplicate_memtable_on_sink_s3 VALUES (999, 1998)" + exception "injected failure after sink upload" + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs("DeltaWriterV2.sink_upload.after_upload_failure") + } + order_qt_failed_upload_rows """ + SELECT COUNT(*), SUM(k), SUM(v) FROM test_cloud_duplicate_memtable_on_sink_s3 + """ + } + + def testAggregateAndMor = { + sql "DROP TABLE IF EXISTS cloud_memtable_models_source" + sql """ + CREATE TABLE cloud_memtable_models_source (n BIGINT NOT NULL) + DUPLICATE KEY(n) DISTRIBUTED BY HASH(n) BUCKETS 12 + PROPERTIES ("replication_num"="1") + """ + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_memtable_models_source SELECT number FROM numbers('number'='12000')" + sql "SET enable_memtable_on_sink_node=true" + sql "SET parallel_pipeline_task_num=4" + sql "SET profile_level=2" + sql "SET enable_file_cache=false" + + def aggQuery = """ + SELECT k, s, lo, hi, bitmap_count(b), hll_cardinality(h), r, rn + FROM cloud_memtable_agg + """ + def morQuery = "SELECT k, v FROM cloud_memtable_mor" + def seqQuery = "SELECT k, v, seq FROM cloud_memtable_mor_seq" + def checkLayout = { table, tag -> + def tablet = sql_return_maparray("SHOW TABLETS FROM ${table}")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM ${table}")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + code, body -> + assertEquals(200, code) + def meta = parseJson(body) + logger.info("{} rowset layout: {}", table, meta) + def ids = meta.segment_ids ?: [] + quickTest(tag, """ + SELECT ${meta.segments_overlap_pb == 'OVERLAPPING'}, + ${ids.size() == (meta.num_segments as int)}, + ${ids.toSet().size() == ids.size()}, + ${ids.any { (it as long) >= 1000 }}, + ${(meta.packed_slice_locations ?: [:]).size() > 0} + """, true) + } + } + def loadModelS3 = { table, sinkUpload -> + def label = "agg_mor_" + UUID.randomUUID().toString().replace('-', '_') + sql "SET enable_profile=true" + def load = loadS3(table, label) + def required = sinkUpload ? ["DeltaWriterV2", "CloudMemtableSinkUpload: true"] : ["DeltaWriterV2"] + new ProfileAction(context).getProfile(load.JobId.toString(), required) + sql "SET enable_profile=false" + } + + [false, true].each { sinkUpload -> + sql "SET enable_cloud_memtable_sink_upload=${sinkUpload}" + if (sinkUpload) { + GetDebugPoint().enableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + } + sql "DROP TABLE IF EXISTS cloud_memtable_agg" + sql "DROP TABLE IF EXISTS cloud_memtable_mor" + sql "DROP TABLE IF EXISTS cloud_memtable_mor_seq" + sql "DROP TABLE IF EXISTS cloud_memtable_agg_broker" + sql "DROP TABLE IF EXISTS cloud_memtable_mor_broker" + sql """ + CREATE TABLE cloud_memtable_agg ( + k BIGINT NOT NULL, s BIGINT SUM, lo BIGINT MIN, hi BIGINT MAX, + b BITMAP BITMAP_UNION, h HLL HLL_UNION, + r BIGINT REPLACE, rn BIGINT REPLACE_IF_NOT_NULL + ) AGGREGATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true") + """ + sql """ + CREATE TABLE cloud_memtable_mor (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="false", + "disable_auto_compaction"="true") + """ + sql """ + CREATE TABLE cloud_memtable_mor_seq ( + k BIGINT NOT NULL, v BIGINT, seq BIGINT NOT NULL, + INDEX idx_k (k) USING INVERTED + ) UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="false", + "function_column.sequence_col"="seq", "inverted_index_storage_format"="V2", + "disable_auto_compaction"="true") + """ + sql "SET enable_profile=true" + sql """ + /* cloud_memtable_agg_${sinkUpload} */ + INSERT INTO cloud_memtable_agg + SELECT n % 4, n, n, n, to_bitmap(n), hll_hash(CAST(n % 4 AS STRING)), + n % 4 + 10, IF(n % 8 < 4, n % 4 + 20, NULL) + FROM cloud_memtable_models_source + """ + sql """ + /* cloud_memtable_mor_${sinkUpload} */ + INSERT INTO cloud_memtable_mor SELECT n % 4, n % 4 * 10 FROM cloud_memtable_models_source + """ + sql """ + /* cloud_memtable_mor_seq_${sinkUpload} */ + INSERT INTO cloud_memtable_mor_seq SELECT n % 4, n * 10, n FROM cloud_memtable_models_source + """ + ['agg', 'mor', 'mor_seq'].each { model -> + def required = sinkUpload ? ["DeltaWriterV2", "CloudMemtableSinkUpload: true"] : ["DeltaWriterV2"] + new ProfileAction(context).getProfileBySql("cloud_memtable_${model}_${sinkUpload}", required) + if (sinkUpload) { + checkLayout("cloud_memtable_${model}", "layout_${model}") + } + } + sql "SET enable_profile=false" + quickTest("agg_${sinkUpload}", aggQuery, true) + quickTest("mor_${sinkUpload}", morQuery, true) + quickTest("seq_${sinkUpload}", seqQuery, true) + quickTest("seq_index_${sinkUpload}", seqQuery + " WHERE k IN (1,3)", true) + + // A newer transaction with a lower Sequence must not replace the business-newer row. + sql "INSERT INTO cloud_memtable_mor_seq VALUES (0,-1,1)" + quickTest("seq_lower_${sinkUpload}", seqQuery, true) + sql "INSERT INTO cloud_memtable_mor VALUES (0,99)" + quickTest("mor_new_version_${sinkUpload}", morQuery, true) + sql """ + INSERT INTO cloud_memtable_agg + VALUES (0,10,-1,13000,bitmap_empty(),hll_empty(),99,NULL) + """ + quickTest("agg_new_version_${sinkUpload}", aggQuery, true) + sql "INSERT INTO cloud_memtable_mor_seq SELECT n % 4,n*10,n FROM cloud_memtable_models_source WHERE n<0" + quickTest("seq_empty_${sinkUpload}", seqQuery, true) + ['agg', 'mor', 'mor_seq'].each { model -> + trigger_and_wait_compaction("cloud_memtable_${model}", "full") + } + quickTest("agg_compacted_${sinkUpload}", aggQuery, true) + quickTest("mor_compacted_${sinkUpload}", morQuery, true) + quickTest("seq_compacted_${sinkUpload}", seqQuery, true) + + sql """ + CREATE TABLE cloud_memtable_agg_broker (k BIGINT NOT NULL, v BIGINT SUM) + AGGREGATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true") + """ + sql """ + CREATE TABLE cloud_memtable_mor_broker (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="false", + "disable_auto_compaction"="true") + """ + loadModelS3("cloud_memtable_agg_broker", sinkUpload) + loadModelS3("cloud_memtable_mor_broker", sinkUpload) + quickTest("agg_broker_${sinkUpload}", "SELECT k,v FROM cloud_memtable_agg_broker", true) + quickTest("mor_broker_${sinkUpload}", "SELECT k,v FROM cloud_memtable_mor_broker", true) + } + + streamLoad { + table "cloud_memtable_agg_broker" + set "column_separator", "," + set "memtable_on_sink_node", "true" + inputStream new ByteArrayInputStream("50,1\n50,2\n".getBytes()) + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + assertEquals("Success", parseJson(result).Status) + } + } + order_qt_agg_stream "SELECT k,v FROM cloud_memtable_agg_broker WHERE k=50" + streamLoad { + table "cloud_memtable_mor_seq" + set "column_separator", "," + set "memtable_on_sink_node", "true" + inputStream new ByteArrayInputStream("0,1,1\n1,130010,13001\n".getBytes()) + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + assertEquals("Success", parseJson(result).Status) + } + } + order_qt_mor_stream seqQuery + sql "INSERT INTO cloud_memtable_mor_seq (k,v,seq,__DORIS_DELETE_SIGN__) VALUES (2,140000,14000,1)" + order_qt_mor_delete seqQuery + sql "INSERT INTO cloud_memtable_mor_seq VALUES (2,150000,15000)" + order_qt_mor_reinsert seqQuery + trigger_and_wait_compaction("cloud_memtable_mor_seq", "full") + order_qt_mor_final_compacted seqQuery + } + + def testVariant = { + sql "DROP TABLE IF EXISTS cloud_memtable_variant_reference" + sql "DROP TABLE IF EXISTS cloud_memtable_variant" + sql """ + CREATE TABLE cloud_memtable_variant_reference (k BIGINT NOT NULL, v VARIANT) + DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true") + """ + sql "CREATE TABLE cloud_memtable_variant LIKE cloud_memtable_variant_reference" + def input = """ + SELECT n, parse_to_variant(IF(n % 2 = 0, + CONCAT('{"a":', n, ',"nested":{"b":', n * 2, '}}'), + CONCAT('{"a":', n, ',"extra":"', n, '"}'))) + FROM cloud_memtable_models_source WHERE n < 1200 + """ + def newPath = """SELECT 1200, parse_to_variant('{"a":1200,"later":2400}')""" + def projection = """ + SELECT k, CAST(v['a'] AS BIGINT), CAST(v['nested']['b'] AS BIGINT), + CAST(v['extra'] AS STRING), CAST(v['later'] AS BIGINT) + """ + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_memtable_variant_reference ${input}" + sql "INSERT INTO cloud_memtable_variant_reference ${newPath}" + def expected = sql "${projection} FROM cloud_memtable_variant_reference ORDER BY k" + sql "SET enable_memtable_on_sink_node=true" + sql "SET enable_cloud_memtable_sink_upload=true" + sql "SET enable_profile=true" + GetDebugPoint().enableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + try { + sql """ + /* cloud_memtable_variant_profile */ + INSERT INTO cloud_memtable_variant ${input} + """ + new ProfileAction(context).getProfileBySql("cloud_memtable_variant_profile", + ["DeltaWriterV2", "CloudMemtableSinkUpload: true"]) + sql "SET enable_profile=false" + def tablet = sql_return_maparray("SHOW TABLETS FROM cloud_memtable_variant")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM cloud_memtable_variant")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + code, body -> + assertEquals(200, code) + def ids = parseJson(body).segment_ids + assertTrue(ids.any { (it as long) >= 1000 }, + "VARIANT schema assembly must include multiple sink writers: ${ids}") + } + // A later rowset introduces another extracted path before compaction. + sql "INSERT INTO cloud_memtable_variant ${newPath}" + assertEquals(expected, sql("${projection} FROM cloud_memtable_variant ORDER BY k")) + trigger_and_wait_compaction("cloud_memtable_variant", "full") + assertEquals(expected, sql("${projection} FROM cloud_memtable_variant ORDER BY k")) + } finally { + sql "SET enable_profile=false" + GetDebugPoint().disableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + } + } + + def originalBatchSize = sql("SHOW VARIABLES LIKE 'broker_load_batch_size'")[0][1] + try { + testDuplicate() + } finally { + sql "SET broker_load_batch_size=${originalBatchSize}" + sql "SET enable_profile=false" + GetDebugPoint().disableDebugPointForAllBEs("MemTable.need_flush") + GetDebugPoint().disableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + GetDebugPoint().disableDebugPointForAllBEs("DeltaWriterV2.sink_upload.duplicate_result") + } + try { + testAggregateAndMor() + } finally { + sql "SET enable_profile=false" + GetDebugPoint().disableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + } + testVariant() + } +} diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_cache.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_cache.groovy new file mode 100644 index 00000000000000..02dd58cd435a8d --- /dev/null +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_cache.groovy @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.Http + +suite("test_cloud_memtable_sink_upload_cache", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(3) + options.enableDebugPoints() + options.feConfigs += ['cloud_stream_load_default_memtable_sink_upload=true'] + options.beConfigs += ['enable_file_cache=true', 'small_file_threshold_bytes=1048576'] + docker(options) { + sql "DROP TABLE IF EXISTS cloud_memtable_sink_upload_cache" + sql """ + CREATE TABLE cloud_memtable_sink_upload_cache ( + k BIGINT NOT NULL, v BIGINT, INDEX idx_k(k) USING INVERTED + ) DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true", + "inverted_index_storage_format"="V2") + """ + def tablet = sql_return_maparray("SHOW TABLETS FROM cloud_memtable_sink_upload_cache")[0] + def backends = sql_return_maparray("SHOW BACKENDS") + def target = backends.find { it.BackendId == tablet.BackendId } + def remote = backends.find { it.BackendId != tablet.BackendId } + def ms = cluster.getAllMetaservices()[0] + def cached = { be, file -> + !Http.GET("http://${be.Host}:${be.HttpPort}/api/file_cache?op=list_cache&value=${file}", true).isEmpty() + } + def warmupStats = { + Http.GET("http://${target.Host}:${target.HttpPort}/api/warmup_event_driven_stats", true) + .data.find { it.job_id == 0 } + } + def skipWarmup = "FileCacheBlockDownloader::download_segment_file.skip_warmup" + GetDebugPoint().enableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + try { + [false, true].each { packed -> + setBeConfigTemporary(['enable_packed_file': packed.toString()]) { + [false, true].each { local -> + def sink = local ? target : remote + def tag = "${packed ? 'packed' : 'plain'}_${local ? 'local' : 'remote'}" + def before = warmupStats() + long finishedSegments = before?.finish?.seg?.num?.get('1h') ?: 0 + long finishedIndexes = before?.finish?.idx?.num?.get('1h') ?: 0 + // For the same-BE case, cache must come from uploading, even without warmup. + if (local) { + GetDebugPoint().enableDebugPointForAllBEs(skipWarmup) + } + try { + streamLoad { + table "cloud_memtable_sink_upload_cache" + directToBe sink.Host, sink.HttpPort as int + set "column_separator", "," + set "memtable_on_sink_node", "true" + set "group_commit", "off_mode" + inputStream new ByteArrayInputStream("1,10\n2,20\n".getBytes()) + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + def response = parseJson(result) + quickTest("load_${tag}", """ + SELECT '${response.Status}', ${response.NumberLoadedRows}, + ${response.NumberFilteredRows} + """, true) + } + } + def partition = sql_return_maparray( + "SHOW PARTITIONS FROM cloud_memtable_sink_upload_cache")[0] + def meta + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, + partition.VisibleVersion) { code, body -> + assertEquals(200, code) + meta = parseJson(body) + } + def ids = meta.segment_ids + quickTest("layout_${tag}", """ + SELECT ${meta.num_segments as int}, + ${(meta.packed_slice_locations ?: [:]).size() > 0} + """, true) + def files = ids.collectMany { id -> + ["${meta.rowset_id_v2}_${id}.dat", "${meta.rowset_id_v2}_${id}.idx"] + } + // Wait for download callbacks, then inspect cache keys before querying data. + // list_cache alone also lists blocks whose downloads have not completed. + awaitUntil(60) { + def stats = warmupStats() + stats != null && + stats.finish.seg.num['1h'] >= finishedSegments + ids.size() && + stats.finish.idx.num['1h'] >= finishedIndexes + ids.size() && + files.every { cached(target, it) } + } + if (packed) { + // Drain sink-side async cache writes before asserting their absence. + awaitUntil(60) { + // BRPC serves HTML to Java's user agent, and plain text to curl. + def (code, out, err) = curl("GET", + "http://${sink.Host}:${sink.BrpcPort}/vars/packed_file_cache_async_write_count") + assertEquals(0, code, err) + out.trim().split(':')[-1].trim().toLong() == 0 + } + } + quickTest("cache_${tag}", """ + SELECT ${files.every { cached(target, it) }}, + ${files.every { !cached(remote, it) }} + """, true) + } finally { + if (local) { + GetDebugPoint().disableDebugPointForAllBEs(skipWarmup) + } + } + } + } + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + } + order_qt_rows "SELECT k, SUM(v), COUNT(*) FROM cloud_memtable_sink_upload_cache GROUP BY k" + } +} diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_mow.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_mow.groovy new file mode 100644 index 00000000000000..5992161726f235 --- /dev/null +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_mow.groovy @@ -0,0 +1,282 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction +import org.apache.doris.regression.suite.ClusterOptions +import java.util.concurrent.TimeUnit + +suite("test_cloud_memtable_sink_upload_mow", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(3) + options.enableDebugPoints() + options.beConfigs += ['share_delta_writers=false', 'enable_packed_file=true', + 'small_file_threshold_bytes=1048576', + 'enable_merge_on_write_correctness_check=true'] + options.feConfigs += ['cloud_stream_load_default_memtable_sink_upload=true', + 'min_bytes_per_broker_scanner=100'] + docker(options) { + sql "SET enable_sql_cache=false" + def checkUniqueKeys = { table -> + sql """ + SELECT assert_true(COUNT(*) = 0, '${table} contains duplicate keys') FROM ( + SELECT k, COUNT(*) AS a FROM ${table} GROUP BY k HAVING a > 1 + ) duplicates + """ + } + sql "DROP TABLE IF EXISTS cloud_mow_source" + sql "DROP TABLE IF EXISTS cloud_mow_direct" + sql "DROP TABLE IF EXISTS cloud_mow_direct_seq" + sql "DROP TABLE IF EXISTS cloud_mow_broker" + sql """ + CREATE TABLE cloud_mow_source (n BIGINT NOT NULL) + DUPLICATE KEY(n) DISTRIBUTED BY HASH(n) BUCKETS 12 + PROPERTIES ("replication_num"="1") + """ + sql """ + CREATE TABLE cloud_mow_direct (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "disable_auto_compaction"="true") + """ + sql """ + CREATE TABLE cloud_mow_direct_seq ( + k BIGINT NOT NULL, v BIGINT, seq BIGINT NOT NULL, INDEX idx_k(k) USING INVERTED + ) UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "function_column.sequence_col"="seq", "inverted_index_storage_format"="V2", + "disable_auto_compaction"="true") + """ + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_mow_source SELECT number FROM numbers('number'='12000')" + // Two versions leave deleted old rows in the snapshot. + sql "INSERT INTO cloud_mow_direct VALUES (0,-1),(1,-1),(2,-1),(3,-1)" + sql "INSERT INTO cloud_mow_direct VALUES (0,0),(1,0),(2,0),(3,0)" + sql "INSERT INTO cloud_mow_direct_seq VALUES (0,-1,-1),(1,-1,-1),(2,-1,-1),(3,-1,-1)" + sql "INSERT INTO cloud_mow_direct_seq VALUES (0,0,0),(1,0,0),(2,0,0),(3,0,0)" + sql "SET enable_memtable_on_sink_node=true" + sql "SET enable_cloud_memtable_sink_upload=true" + sql "SET parallel_pipeline_task_num=4" + sql "SET enable_file_cache=false" + sql "SET profile_level=2" + sql "SET enable_profile=true" + GetDebugPoint().enableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + GetDebugPoint().enableDebugPointForAllBEs("DeltaWriterV2.sink_upload.duplicate_result") + sql """ + /* cloud_mow_direct_profile */ + INSERT INTO cloud_mow_direct SELECT n%4,n%4*10 FROM cloud_mow_source + """ + sql """ + /* cloud_mow_direct_seq_profile */ + INSERT INTO cloud_mow_direct_seq SELECT n%4,n*10,n+1 FROM cloud_mow_source + """ + ['cloud_mow_direct_profile', 'cloud_mow_direct_seq_profile'].each { tag -> + new ProfileAction(context).getProfileBySql(tag, + ["CloudMemtableSinkUpload: true", "CloudMemtableMowBitmap: true"]) + } + sql "SET enable_profile=false" + GetDebugPoint().disableDebugPointForAllBEs("DeltaWriterV2.sink_upload.duplicate_result") + order_qt_initial "SELECT * FROM cloud_mow_direct" + order_qt_initial_seq "SELECT * FROM cloud_mow_direct_seq" + order_qt_index "SELECT * FROM cloud_mow_direct_seq WHERE k IN (1,3)" + ['cloud_mow_direct', 'cloud_mow_direct_seq'].each { table -> + checkUniqueKeys(table) + def tablet = sql_return_maparray("SHOW TABLETS FROM ${table}")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM ${table}")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + code, body -> + assertEquals(200, code) + def meta = parseJson(body) + logger.info("MOW rowset layout: {}", meta) + def ids = meta.segment_ids ?: [] + quickTest("layout_${table}", """ + SELECT ${ids.size() == (meta.num_segments as int)}, + ${ids.toSet().size() == ids.size()}, + ${ids.any { (it as long) >= 1000 }}, + ${(meta.packed_slice_locations ?: [:]).size() > 0} + """, true) + } + } + sql "INSERT INTO cloud_mow_direct_seq VALUES (0,-1,1)" + order_qt_lower_seq "SELECT * FROM cloud_mow_direct_seq" + sql "INSERT INTO cloud_mow_direct VALUES (0,99)" + order_qt_new_version "SELECT * FROM cloud_mow_direct" + sql "INSERT INTO cloud_mow_direct_seq SELECT n%4,n*10,n FROM cloud_mow_source WHERE n<0" + order_qt_empty "SELECT * FROM cloud_mow_direct_seq" + sql "INSERT INTO cloud_mow_direct_seq (k,v,seq,__DORIS_DELETE_SIGN__) VALUES (2,130000,13000,1)" + order_qt_delete "SELECT * FROM cloud_mow_direct_seq" + sql "INSERT INTO cloud_mow_direct_seq VALUES (2,-1,1)" + order_qt_lower_after_delete "SELECT * FROM cloud_mow_direct_seq" + sql "INSERT INTO cloud_mow_direct_seq VALUES (2,140000,14000)" + order_qt_reinsert "SELECT * FROM cloud_mow_direct_seq" + trigger_and_wait_compaction("cloud_mow_direct", "full") + trigger_and_wait_compaction("cloud_mow_direct_seq", "full") + order_qt_compacted "SELECT * FROM cloud_mow_direct" + order_qt_compacted_seq "SELECT * FROM cloud_mow_direct_seq" + checkUniqueKeys("cloud_mow_direct") + checkUniqueKeys("cloud_mow_direct_seq") + + streamLoad { + table "cloud_mow_direct_seq" + set "column_separator", "," + set "memtable_on_sink_node", "true" + inputStream new ByteArrayInputStream("0,150000,15000\n3,-1,1\n".getBytes()) + check { result, exception, startTime, endTime -> + if (exception != null) { throw exception } + assertEquals("Success", parseJson(result).Status) + } + } + order_qt_stream "SELECT * FROM cloud_mow_direct_seq" + + // Freeze the snapshot, then replace its rowsets with a concurrent load and compaction. + def block = "CloudRowsetBuilder.sink_mow.snapshot_ready" + def backends = sql_return_maparray("SHOW BACKENDS") + GetDebugPoint().enableDebugPointForAllBEs(block, [timeout: "120"]) + def pending = thread { + sql "SET enable_memtable_on_sink_node=true" + sql "SET enable_cloud_memtable_sink_upload=true" + sql "INSERT INTO cloud_mow_direct_seq VALUES (0,200000,20000)" + } + try { + awaitUntil(60) { + backends.any { be -> + def (code, out, err) = curl("GET", + "http://${be.Host}:${be.BrpcPort}/vars/cloud_memtable_mow_snapshot_waiters") + assertEquals(0, code, err) + def ready = out =~ /cloud_memtable_mow_snapshot_waiters\s*:\s*(\d+)/ + ready.find() && ready.group(1).toLong() > 0 + } + } + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_mow_direct_seq VALUES (0,210000,21000)" + trigger_and_wait_compaction("cloud_mow_direct_seq", "full") + } finally { + GetDebugPoint().disableDebugPointForAllBEs(block) + sql "SET enable_memtable_on_sink_node=true" + } + pending.get(120, TimeUnit.SECONDS) + order_qt_concurrent_compaction "SELECT * FROM cloud_mow_direct_seq" + checkUniqueKeys("cloud_mow_direct_seq") + + GetDebugPoint().enableDebugPointForAllBEs("DeltaWriterV2.sink_mow.after_bitmap_failure") + try { + test { + sql "INSERT INTO cloud_mow_direct_seq VALUES (99,990000,99000)" + exception "injected failure after sink MOW bitmap calculation" + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs("DeltaWriterV2.sink_mow.after_bitmap_failure") + } + order_qt_failed_invisible "SELECT * FROM cloud_mow_direct_seq" + + // Cluster keys encode a physical row ID in the primary-key index. + sql "DROP TABLE IF EXISTS cloud_mow_cluster" + sql """ + CREATE TABLE cloud_mow_cluster (k BIGINT NOT NULL, v BIGINT NOT NULL, seq BIGINT NOT NULL) + UNIQUE KEY(k) ORDER BY(v) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true", + "function_column.sequence_col"="seq", "disable_auto_compaction"="true") + """ + sql "INSERT INTO cloud_mow_cluster SELECT n%4,-n,n+1 FROM cloud_mow_source" + order_qt_cluster "SELECT * FROM cloud_mow_cluster" + sql "INSERT INTO cloud_mow_cluster VALUES (0,999,20000),(1,999,1)" + order_qt_cluster_update "SELECT * FROM cloud_mow_cluster" + trigger_and_wait_compaction("cloud_mow_cluster", "full") + order_qt_cluster_compacted "SELECT * FROM cloud_mow_cluster" + checkUniqueKeys("cloud_mow_cluster") + + setBeConfigTemporary(['enable_packed_file': 'false']) { + sql "DROP TABLE IF EXISTS cloud_mow_plain_reference" + sql "DROP TABLE IF EXISTS cloud_mow_plain" + sql "CREATE TABLE cloud_mow_plain_reference LIKE cloud_mow_direct_seq" + sql "CREATE TABLE cloud_mow_plain LIKE cloud_mow_direct_seq" + try { + sql "SET enable_memtable_on_sink_node=false" + sql "INSERT INTO cloud_mow_plain_reference VALUES (0,-1,-1),(1,-1,-1)" + sql "INSERT INTO cloud_mow_plain VALUES (0,-1,-1),(1,-1,-1)" + sql "INSERT INTO cloud_mow_plain_reference SELECT n%4,n*10,n+1 FROM cloud_mow_source" + def expected = sql "SELECT k,v,seq FROM cloud_mow_plain_reference ORDER BY k" + sql "SET enable_memtable_on_sink_node=true" + sql "SET enable_profile=true" + sql """ + /* cloud_mow_plain_profile */ + INSERT INTO cloud_mow_plain SELECT n%4,n*10,n+1 FROM cloud_mow_source + """ + new ProfileAction(context).getProfileBySql("cloud_mow_plain_profile", + ["CloudMemtableSinkUpload: true", "CloudMemtableMowBitmap: true"]) + sql "SET enable_profile=false" + checkUniqueKeys("cloud_mow_plain") + assertEquals(expected, sql("SELECT k,v,seq FROM cloud_mow_plain ORDER BY k")) + def tablet = sql_return_maparray("SHOW TABLETS FROM cloud_mow_plain")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM cloud_mow_plain")[0] + def ms = cluster.getAllMetaservices()[0] + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + code, body -> + assertEquals(200, code) + def meta = parseJson(body) + assertTrue((meta.packed_slice_locations ?: [:]).isEmpty(), + "Unpacked MOW rowset contains packed mappings: ${meta}") + } + trigger_and_wait_compaction("cloud_mow_plain", "full") + checkUniqueKeys("cloud_mow_plain") + assertEquals(expected, sql("SELECT k,v,seq FROM cloud_mow_plain ORDER BY k")) + } finally { + sql "SET enable_profile=false" + sql "SET enable_memtable_on_sink_node=true" + } + } + + sql """ + CREATE TABLE cloud_mow_broker (k BIGINT NOT NULL, v BIGINT) + UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "enable_unique_key_merge_on_write"="true") + """ + // Switch the same table from sink upload to forwarding and overwrite a sink-uploaded key. + [true, false].each { sinkUpload -> + if (!sinkUpload) { + sql "INSERT INTO cloud_mow_broker VALUES (50,-1)" + GetDebugPoint().disableDebugPointForAllBEs("LoadStreamWriter.append_data.unexpected_transfer") + } + sql "SET enable_cloud_memtable_sink_upload=${sinkUpload}" + def label = "mow_broker_" + UUID.randomUUID().toString().replace('-', '_') + sql "SET enable_profile=true" + sql """ + LOAD LABEL ${label} ( + DATA INFILE("s3://${getS3BucketName()}/regression/load/data/basic_data.csv") + INTO TABLE cloud_mow_broker COLUMNS TERMINATED BY "|" FORMAT AS "CSV" + (k, c01, c02, c03, c04, c05, c06, c07, c08, c09, + c10, c11, c12, c13, c14, c15, c16, c17, c18) SET (v=k*2) + ) WITH S3 ( + "AWS_ACCESS_KEY"="${getS3AK()}", "AWS_SECRET_KEY"="${getS3SK()}", + "AWS_ENDPOINT"="${getS3Endpoint()}", "AWS_REGION"="${getS3Region()}", + "provider"="${getS3Provider()}" + ) PROPERTIES ("load_parallelism"="1") + """ + waitForBrokerLoadDone(label) + def load = sql_return_maparray("SHOW LOAD WHERE LABEL = '${label}'")[0] + assertEquals("FINISHED", load.State, "Broker load did not finish: ${load}") + new ProfileAction(context).getProfile(load.JobId.toString(), + sinkUpload ? ["CloudMemtableSinkUpload: true", "CloudMemtableMowBitmap: true"] + : ["DeltaWriterV2"]) + sql "SET enable_profile=false" + quickTest("broker_${sinkUpload}", "SELECT * FROM cloud_mow_broker", true) + checkUniqueKeys("cloud_mow_broker") + } + } +} diff --git a/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_unshared.groovy b/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_unshared.groovy new file mode 100644 index 00000000000000..b147958b1f7a05 --- /dev/null +++ b/regression-test/suites/cloud_p0/test_cloud_memtable_sink_upload_unshared.groovy @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction +import org.apache.doris.regression.suite.ClusterOptions + +suite("test_cloud_memtable_sink_upload_unshared", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(3) + options.beConfigs += ['share_delta_writers=false', 'enable_packed_file=false', + 'max_segment_num_per_rowset=1000'] + docker(options) { + sql "DROP TABLE IF EXISTS test_cloud_sink_upload_source" + sql "DROP TABLE IF EXISTS test_cloud_sink_upload_unshared" + sql """ + CREATE TABLE test_cloud_sink_upload_source (k BIGINT, v BIGINT) + DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 12 + PROPERTIES ("replication_num"="1") + """ + sql """ + CREATE TABLE test_cloud_sink_upload_unshared (k BIGINT, v BIGINT) + DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ("replication_num"="1", "disable_auto_compaction"="true") + """ + sql "SET enable_sql_cache=false" + sql "SET enable_memtable_on_sink_node=false" + sql """ + INSERT INTO test_cloud_sink_upload_source + SELECT number, number * 2 FROM numbers("number"="100000") + """ + sql "SET enable_memtable_on_sink_node=true" + sql "SET enable_cloud_memtable_sink_upload=true" + sql "SET parallel_pipeline_task_num=4" + sql "SET profile_level=2" + sql "SET enable_profile=true" + sql """ + /* cloud_sink_upload_unshared_profile */ + INSERT INTO test_cloud_sink_upload_unshared + SELECT * FROM test_cloud_sink_upload_source + """ + def profileString = new ProfileAction(context).getProfileBySql( + "cloud_sink_upload_unshared_profile", ["CloudMemtableSinkUpload: true"]) + logger.info("unshared sink-upload profile:\n{}", profileString) + sql "SET enable_profile=false" + sql "SET enable_file_cache=false" + order_qt_rows """ + SELECT COUNT(*), SUM(k), SUM(v) FROM test_cloud_sink_upload_unshared + """ + def tablet = sql_return_maparray("SHOW TABLETS FROM test_cloud_sink_upload_unshared")[0] + def partition = sql_return_maparray("SHOW PARTITIONS FROM test_cloud_sink_upload_unshared")[0] + def ms = cluster.getAllMetaservices()[0] + def backendCount = sql_return_maparray("SHOW BACKENDS").size() + getSegmentFilesFromMs("${ms.host}:${ms.httpPort}", tablet.TabletId, partition.VisibleVersion) { + responseCode, body -> + assertEquals(200, responseCode) + def rowsetMeta = parseJson(body) + def segmentIds = rowsetMeta.segment_ids ?: [] + logger.info("Unshared sink-upload rowset layout: {}", [ + tablet_id: tablet.TabletId, version: partition.VisibleVersion, + rowset_id: rowsetMeta.rowset_id_v2, segment_ids: segmentIds, + num_segment_rows: rowsetMeta.num_segment_rows, + segments_file_size: rowsetMeta.segments_file_size + ]) + // One shared writer per BE cannot produce more nonempty ranges than BEs. + def writerRanges = segmentIds.collect { (it as long).intdiv(1000) }.toSet() + assertTrue(writerRanges.size() > backendCount, + "Expected multiple writers on one source BE for tablet ${tablet.TabletId}: ${segmentIds}") + order_qt_rowset_layout """ + SELECT ${segmentIds.size() == (rowsetMeta.num_segments as int)}, + ${segmentIds.toSet().size() == segmentIds.size()}, + ${segmentIds.any { (it as long) >= 1000 }}, + ${rowsetMeta.num_rows as long} + """ + } + sql """ + INSERT INTO test_cloud_sink_upload_unshared + SELECT * FROM test_cloud_sink_upload_source WHERE k < 0 + """ + order_qt_empty_input """ + SELECT COUNT(*), SUM(k), SUM(v) FROM test_cloud_sink_upload_unshared + """ + // Verify that another transaction can coexist with the sparse-id rowset. + sql """ + INSERT INTO test_cloud_sink_upload_unshared + SELECT * FROM test_cloud_sink_upload_source WHERE k < 10 + """ + order_qt_second_load """ + SELECT COUNT(*), SUM(k), SUM(v) FROM test_cloud_sink_upload_unshared + """ + } +}