[opt](build) 1/4: Speed up BE full build ~22% by cutting hot-header include edges

.gitignore

@@ -99,6 +99,7 @@ fe/fe-core/src/main/java/org/apache/parquet
 be/build*/
 be/cmake-build*/
 be/ut_build*/
+be/compile-bench-results/
 be/src/gen_cpp/*.[cc, cpp, h]
 be/src/gen_cpp/opcode
 be/tags

be/src/agent/agent_server.cpp

@@ -37,6 +37,7 @@
 #include "common/config.h"
 #include "common/logging.h"
 #include "common/status.h"
+#include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "storage/olap_define.h"
 #include "storage/options.h"

be/src/agent/task_worker_pool.cpp

@@ -65,6 +65,7 @@
 #include "io/fs/path.h"
 #include "io/fs/remote_file_system.h"
 #include "io/fs/s3_file_system.h"
+#include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "runtime/fragment_mgr.h"
 #include "runtime/index_policy/index_policy_mgr.h"

be/src/cloud/cloud_compaction_stop_token.cpp

@@ -22,6 +22,7 @@
 #include "cloud/cloud_meta_mgr.h"
 #include "cloud/config.h"
 #include "common/logging.h"
+#include "service/backend_options.h"
 
 namespace doris {
 

be/src/cloud/cloud_index_change_compaction.cpp

@@ -21,6 +21,7 @@
 #include "cloud/config.h"
 #include "common/status.h"
 #include "cpp/sync_point.h"
+#include "service/backend_options.h"
 
 namespace doris {
 

be/src/cloud/cloud_meta_mgr.cpp

@@ -57,7 +57,9 @@
 #include "cpp/sync_point.h"
 #include "io/fs/obj_storage_client.h"
 #include "load/stream_load/stream_load_context.h"
+#include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
+#include "service/backend_options.h"
 #include "storage/olap_common.h"
 #include "storage/rowset/rowset.h"
 #include "storage/rowset/rowset_factory.h"

be/src/cloud/cloud_storage_engine.cpp

@@ -59,6 +59,7 @@
 #include "load/memtable/memtable_flush_executor.h"
 #include "runtime/exec_env.h"
 #include "runtime/memory/cache_manager.h"
+#include "service/backend_options.h"
 #include "storage/compaction/cumulative_compaction_policy.h"
 #include "storage/compaction/cumulative_compaction_time_series_policy.h"
 #include "storage/compaction_task_tracker.h"

be/src/cloud/cloud_warm_up_manager.cpp

@@ -43,6 +43,7 @@
 #include "common/logging.h"
 #include "cpp/sync_point.h"
 #include "io/cache/block_file_cache_downloader.h"
+#include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "service/backend_options.h"
 #include "storage/index/inverted/inverted_index_desc.h"

be/src/common/signal_handler.h

@@ -33,6 +33,7 @@
 
 #pragma once
 
+#include <butil/macros.h> // ARRAYSIZE_UNSAFE
 #include <glog/logging.h>
 
 #include <boost/stacktrace.hpp>

be/src/core/block/block.cpp

@@ -20,6 +20,7 @@
 
 #include "core/block/block.h"
 
+#include <concurrentqueue.h>
 #include <fmt/format.h>
 #include <gen_cpp/data.pb.h>
 #include <glog/logging.h>

be/src/core/data_type/data_type_timestamptz.cpp

@@ -20,9 +20,16 @@
 
 #include "core/data_type/data_type_timestamptz.h"
 
+#include <gen_cpp/data.pb.h>
+
 #include "exprs/function/cast/cast_to_timestamptz.h"
 
 namespace doris {
+void DataTypeTimeStampTz::to_pb_column_meta(PColumnMeta* col_meta) const {
+    DataTypeNumberBase<PrimitiveType::TYPE_TIMESTAMPTZ>::to_pb_column_meta(col_meta);
+    col_meta->mutable_decimal_param()->set_scale(_scale);
+}
+
 Field DataTypeTimeStampTz::get_field(const TExprNode& node) const {
     TimestampTzValue res;
     CastParameters params {.status = Status::OK(), .is_strict = true};

be/src/core/data_type/data_type_timestamptz.h

@@ -60,10 +60,7 @@ class DataTypeTimeStampTz final : public DataTypeNumberBase<PrimitiveType::TYPE_
         scalar_type->set_scale(_scale);
     }
 
-    void to_pb_column_meta(PColumnMeta* col_meta) const override {
-        DataTypeNumberBase<PrimitiveType::TYPE_TIMESTAMPTZ>::to_pb_column_meta(col_meta);
-        col_meta->mutable_decimal_param()->set_scale(_scale);
-    }
+    void to_pb_column_meta(PColumnMeta* col_meta) const override;
 
     UInt32 get_scale() const override { return _scale; }
 

be/src/exec/exchange/vdata_stream_recvr.h

@@ -48,12 +48,12 @@
 #include "runtime/runtime_profile.h"
 #include "runtime/task_execution_context.h"
 #include "runtime/thread_context.h"
-#include "runtime/workload_group/workload_group.h"
 #include "util/stopwatch.hpp"
 
 namespace doris {
 class MemTracker;
 class PBlock;
+class PTransmitDataParams;
 class MemTrackerLimiter;
 class RuntimeState;
 

be/src/exec/operator/materialization_opertor.cpp

@@ -33,6 +33,7 @@
 #include "exec/operator/operator.h"
 #include "exec/rowid_fetcher.h"
 #include "exec/scan/file_scanner.h"
+#include "runtime/workload_group/workload_group.h"
 #include "util/brpc_client_cache.h"
 #include "util/brpc_closure.h"
 #include "util/pretty_printer.h"

be/src/exec/pipeline/dependency.h

@@ -28,6 +28,7 @@
 #include <sqltypes.h>
 
 #include <atomic>
+#include <condition_variable>
 #include <functional>
 #include <memory>
 #include <mutex>

be/src/exec/pipeline/pipeline_fragment_context.cpp

@@ -123,6 +123,7 @@
 #include "exec/spill/spill_file.h"
 #include "io/fs/stream_load_pipe.h"
 #include "load/stream_load/new_load_stream_mgr.h"
+#include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "runtime/fragment_mgr.h"
 #include "runtime/result_buffer_mgr.h"

be/src/exec/pipeline/task_scheduler.h

@@ -31,7 +31,6 @@
 #include "exec/pipeline/pipeline_task.h"
 #include "exec/pipeline/task_queue.h"
 #include "runtime/query_context.h"
-#include "runtime/workload_group/workload_group.h"
 #include "util/thread.h"
 #include "util/uid_util.h"
 

be/src/exec/rowid_fetcher.h

@@ -22,7 +22,9 @@
 #include <gen_cpp/DataSinks_types.h>
 #include <gen_cpp/internal_service.pb.h>
 
+#include <condition_variable>
 #include <memory>
+#include <mutex>
 #include <semaphore>
 #include <utility>
 #include <vector>

be/src/exec/scan/file_scanner.cpp

@@ -88,6 +88,7 @@
 #include "runtime/descriptors.h"
 #include "runtime/runtime_profile.h"
 #include "runtime/runtime_state.h"
+#include "service/backend_options.h"
 
 namespace cctz {
 class time_zone;

be/src/exec/scan/meta_scanner.cpp

@@ -38,6 +38,7 @@
 #include "core/data_type/define_primitive_type.h"
 #include "core/types.h"
 #include "format/table/parquet_metadata_reader.h"
+#include "runtime/cluster_info.h"
 #include "runtime/descriptors.h"
 #include "runtime/exec_env.h"
 #include "runtime/runtime_state.h"

be/src/exec/scan/scanner_context.cpp

@@ -45,6 +45,8 @@
 #include "runtime/exec_env.h"
 #include "runtime/runtime_profile.h"
 #include "runtime/runtime_state.h"
+#include "runtime/thread_context.h"
+#include "runtime/workload_management/resource_context.h"
 #include "storage/tablet/tablet.h"
 #include "util/time.h"
 #include "util/uid_util.h"
@@ -53,6 +55,18 @@ namespace doris {
 
 using namespace std::chrono_literals;
 
+// ==================== ScanTask ====================
+ScanTask::ScanTask(std::weak_ptr<ScannerDelegate> delegate_scanner) : scanner(delegate_scanner) {
+    _resource_ctx = thread_context()->resource_ctx();
+    DorisMetrics::instance()->scanner_task_cnt->increment(1);
+}
+
+ScanTask::~ScanTask() {
+    SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_resource_ctx->memory_context()->mem_tracker());
+    DorisMetrics::instance()->scanner_task_cnt->increment(-1);
+    cached_block.reset();
+}
+
 // ==================== ScannerContext ====================
 ScannerContext::ScannerContext(RuntimeState* state, ScanLocalStateBase* local_state,
                                const TupleDescriptor* output_tuple_desc,

be/src/exec/scan/scanner_context.h

@@ -43,6 +43,7 @@
 
 namespace doris {
 
+class ResourceContext;
 class RuntimeState;
 class TupleDescriptor;
 class WorkloadGroup;
@@ -96,16 +97,9 @@ class ScanTask {
         COMPLETED, // finished with result or error, waiting to be collected by scan node
         EOS,       // finished and no more data, waiting to be collected by scan node
     };
-    ScanTask(std::weak_ptr<ScannerDelegate> delegate_scanner) : scanner(delegate_scanner) {
-        _resource_ctx = thread_context()->resource_ctx();
-        DorisMetrics::instance()->scanner_task_cnt->increment(1);
-    }
+    ScanTask(std::weak_ptr<ScannerDelegate> delegate_scanner);
 
-    ~ScanTask() {
-        SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_resource_ctx->memory_context()->mem_tracker());
-        DorisMetrics::instance()->scanner_task_cnt->increment(-1);
-        cached_block.reset();
-    }
+    ~ScanTask();
 
 private:
     // whether current scanner is finished

be/src/exec/scan/task_executor/tools/simulator/simulation_split.h

@@ -19,6 +19,7 @@
 
 #include <atomic>
 #include <chrono>
+#include <condition_variable>
 #include <future>
 #include <memory>
 #include <mutex>

be/src/exec/sink/autoinc_buffer.cpp

@@ -24,6 +24,7 @@
 
 #include "common/logging.h"
 #include "common/status.h"
+#include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "runtime/runtime_profile.h"
 #include "util/client_cache.h"

be/src/exec/sink/vrow_distribution.cpp

@@ -35,6 +35,7 @@
 #include "core/column/column_vector.h"
 #include "core/data_type/data_type.h"
 #include "exec/sink/writer/vtablet_writer.h"
+#include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "runtime/query_context.h"
 #include "runtime/runtime_state.h"

be/src/exec/sink/writer/async_result_writer.cpp

@@ -25,6 +25,7 @@
 #include "runtime/exec_env.h"
 #include "runtime/fragment_mgr.h"
 #include "runtime/runtime_state.h"
+#include "runtime/workload_group/workload_group.h"
 
 namespace doris {
 class ObjectPool;

be/src/exec/sink/writer/maxcompute/vmc_table_writer.cpp

@@ -22,6 +22,7 @@
 #include "exprs/vexpr.h"
 #include "exprs/vexpr_context.h"
 #include "format/transformer/vjni_format_transformer.h"
+#include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "runtime/runtime_state.h"
 #include "util/uid_util.h"

be/src/exec/sink/writer/vtablet_writer.cpp

@@ -65,13 +65,15 @@
 #include "exec/sink/vtablet_finder.h"
 #include "exprs/vexpr.h"
 #include "exprs/vexpr_fwd.h"
+#include "load/memtable/memtable_memory_limiter.h"
 #include "runtime/descriptors.h"
 #include "runtime/exec_env.h"
 #include "runtime/memory/memory_reclamation.h"
 #include "runtime/query_context.h"
 #include "runtime/runtime_profile.h"
 #include "runtime/runtime_state.h"
 #include "runtime/thread_context.h"
+#include "runtime/workload_group/workload_group.h"
 #include "service/backend_options.h"
 #include "storage/binlog.h"
 #include "storage/tablet_info.h"

be/src/exec/sink/writer/vtablet_writer_v2.cpp

@@ -54,6 +54,7 @@
 #include "exec/sink/load_stream_stub.h" // IWYU pragma: keep
 #include "exec/sink/vtablet_block_convertor.h"
 #include "exec/sink/vtablet_finder.h"
+#include "load/memtable/memtable_memory_limiter.h"
 
 namespace doris {
 

be/src/exec/sink/writer/vwal_writer.cpp

@@ -23,6 +23,7 @@
 #include <sstream>
 
 #include "io/fs/encrypted_fs_factory.h"
+#include "io/fs/local_file_system.h"
 #include "util/debug_points.h"
 
 namespace doris {

be/src/exprs/aggregate/aggregate_function.h

@@ -45,6 +45,7 @@ namespace doris {
 class Arena;
 class IColumn;
 class IDataType;
+class QueryContext;
 
 struct AggregateFunctionAttr {
     bool is_window_function {false};

be/src/exprs/function/function_java_udf.cpp

@@ -19,6 +19,7 @@
 
 #include <bthread/bthread.h>
 
+#include <future>
 #include <memory>
 #include <string>
 
@@ -29,6 +30,7 @@
 #include "runtime/exec_env.h"
 #include "runtime/user_function_cache.h"
 #include "util/jni-util.h"
+#include "util/threadpool.h"
 
 const char* EXECUTOR_CLASS = "org/apache/doris/udf/UdfExecutor";
 const char* EXECUTOR_CTOR_SIGNATURE = "([B)V";

be/src/exprs/vectorized_agg_fn.h

@@ -23,6 +23,7 @@
 #include <vector>
 
 #include "common/be_mock_util.h"
+#include "common/factory_creator.h"
 #include "common/status.h"
 #include "core/data_type/data_type.h"
 #include "exec/sort/sort_description.h"

be/src/format/parquet/parquet_predicate.h

@@ -32,6 +32,7 @@
 #include "format/parquet/parquet_column_convert.h"
 #include "format/parquet/parquet_common.h"
 #include "format/parquet/schema_desc.h"
+#include "io/fs/file_reader.h"
 #include "storage/olap_scan_common.h"
 #include "storage/segment/row_ranges.h"
 #include "util/timezone_utils.h"

be/src/format/table/paimon_jni_reader.cpp

@@ -26,6 +26,7 @@
 #include "runtime/descriptors.h"
 #include "runtime/exec_env.h"
 #include "runtime/runtime_state.h"
+#include "storage/options.h"
 #include "util/string_util.h"
 #include "util/uid_util.h"
 

be/src/information_schema/schema_active_queries_scanner.cpp

@@ -20,6 +20,7 @@
 #include "core/block/block.h"
 #include "core/data_type/data_type_factory.hpp"
 #include "core/string_ref.h"
+#include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "runtime/runtime_state.h"
 #include "util/client_cache.h"

be/src/information_schema/schema_authentication_integrations_scanner.cpp

@@ -23,6 +23,7 @@
 #include "core/data_type/data_type_factory.hpp"
 #include "core/string_ref.h"
 #include "information_schema/schema_helper.h"
+#include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "runtime/runtime_state.h"
 #include "util/client_cache.h"

be/src/information_schema/schema_backend_configuration_scanner.cpp

@@ -24,6 +24,7 @@
 #include "core/block/block.h"
 #include "core/data_type/define_primitive_type.h"
 #include "core/string_ref.h"
+#include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "runtime/runtime_state.h"
 

be/src/information_schema/schema_backend_ms_rpc_table_throttlers_scanner.cpp

@@ -26,6 +26,7 @@
 #include "core/block/block.h"
 #include "core/data_type/define_primitive_type.h"
 #include "core/string_ref.h"
+#include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "runtime/runtime_state.h"
 

be/src/information_schema/schema_database_properties_scanner.cpp

@@ -21,6 +21,7 @@
 #include "core/data_type/data_type_factory.hpp"
 #include "core/string_ref.h"
 #include "information_schema/schema_helper.h"
+#include "runtime/cluster_info.h"
 #include "runtime/exec_env.h"
 #include "runtime/runtime_state.h"
 #include "util/client_cache.h"