diff --git a/.gitignore b/.gitignore index b792323544c640..9115120067d9e9 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,7 @@ fe/fe-core/src/main/java/org/apache/parquet be/build*/ be/cmake-build*/ be/ut_build*/ +be/compile-bench-results/ be/src/gen_cpp/*.[cc, cpp, h] be/src/gen_cpp/opcode be/tags diff --git a/be/src/agent/agent_server.cpp b/be/src/agent/agent_server.cpp index bfc8c9d278294e..0ea7351dff7a16 100644 --- a/be/src/agent/agent_server.cpp +++ b/be/src/agent/agent_server.cpp @@ -37,6 +37,7 @@ #include "common/config.h" #include "common/logging.h" #include "common/status.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "storage/olap_define.h" #include "storage/options.h" diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 65cc587ac1899d..23d707702db049 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -65,6 +65,7 @@ #include "io/fs/path.h" #include "io/fs/remote_file_system.h" #include "io/fs/s3_file_system.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/index_policy/index_policy_mgr.h" diff --git a/be/src/cloud/cloud_compaction_stop_token.cpp b/be/src/cloud/cloud_compaction_stop_token.cpp index d65072559ddb0e..f48a5b840ed956 100644 --- a/be/src/cloud/cloud_compaction_stop_token.cpp +++ b/be/src/cloud/cloud_compaction_stop_token.cpp @@ -22,6 +22,7 @@ #include "cloud/cloud_meta_mgr.h" #include "cloud/config.h" #include "common/logging.h" +#include "service/backend_options.h" namespace doris { diff --git a/be/src/cloud/cloud_index_change_compaction.cpp b/be/src/cloud/cloud_index_change_compaction.cpp index 001b9d64f4b23d..55654ccc958824 100644 --- a/be/src/cloud/cloud_index_change_compaction.cpp +++ b/be/src/cloud/cloud_index_change_compaction.cpp @@ -21,6 +21,7 @@ #include "cloud/config.h" #include "common/status.h" #include "cpp/sync_point.h" +#include "service/backend_options.h" namespace doris { diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index ea9c7b376f332e..dd1290f7024a54 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -57,7 +57,9 @@ #include "cpp/sync_point.h" #include "io/fs/obj_storage_client.h" #include "load/stream_load/stream_load_context.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" +#include "service/backend_options.h" #include "storage/olap_common.h" #include "storage/rowset/rowset.h" #include "storage/rowset/rowset_factory.h" diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index fe4aa32a19d5d4..9e992078050942 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -59,6 +59,7 @@ #include "load/memtable/memtable_flush_executor.h" #include "runtime/exec_env.h" #include "runtime/memory/cache_manager.h" +#include "service/backend_options.h" #include "storage/compaction/cumulative_compaction_policy.h" #include "storage/compaction/cumulative_compaction_time_series_policy.h" #include "storage/compaction_task_tracker.h" diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index 9ede36b988304d..4590d96fb4ca80 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -43,6 +43,7 @@ #include "common/logging.h" #include "cpp/sync_point.h" #include "io/cache/block_file_cache_downloader.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "service/backend_options.h" #include "storage/index/inverted/inverted_index_desc.h" diff --git a/be/src/common/signal_handler.h b/be/src/common/signal_handler.h index ca117ee2b1cef5..334b2ea99af6ce 100644 --- a/be/src/common/signal_handler.h +++ b/be/src/common/signal_handler.h @@ -33,6 +33,7 @@ #pragma once +#include // ARRAYSIZE_UNSAFE #include #include diff --git a/be/src/core/block/block.cpp b/be/src/core/block/block.cpp index 8b92c2e3db9156..7922aae07eeea0 100644 --- a/be/src/core/block/block.cpp +++ b/be/src/core/block/block.cpp @@ -20,6 +20,7 @@ #include "core/block/block.h" +#include #include #include #include diff --git a/be/src/core/data_type/data_type_timestamptz.cpp b/be/src/core/data_type/data_type_timestamptz.cpp index 30d13b1c6d75ee..7c6b4d249cb2a7 100644 --- a/be/src/core/data_type/data_type_timestamptz.cpp +++ b/be/src/core/data_type/data_type_timestamptz.cpp @@ -20,9 +20,16 @@ #include "core/data_type/data_type_timestamptz.h" +#include + #include "exprs/function/cast/cast_to_timestamptz.h" namespace doris { +void DataTypeTimeStampTz::to_pb_column_meta(PColumnMeta* col_meta) const { + DataTypeNumberBase::to_pb_column_meta(col_meta); + col_meta->mutable_decimal_param()->set_scale(_scale); +} + Field DataTypeTimeStampTz::get_field(const TExprNode& node) const { TimestampTzValue res; CastParameters params {.status = Status::OK(), .is_strict = true}; diff --git a/be/src/core/data_type/data_type_timestamptz.h b/be/src/core/data_type/data_type_timestamptz.h index b386402cb49696..bd508489295c20 100644 --- a/be/src/core/data_type/data_type_timestamptz.h +++ b/be/src/core/data_type/data_type_timestamptz.h @@ -60,10 +60,7 @@ class DataTypeTimeStampTz final : public DataTypeNumberBaseset_scale(_scale); } - void to_pb_column_meta(PColumnMeta* col_meta) const override { - DataTypeNumberBase::to_pb_column_meta(col_meta); - col_meta->mutable_decimal_param()->set_scale(_scale); - } + void to_pb_column_meta(PColumnMeta* col_meta) const override; UInt32 get_scale() const override { return _scale; } diff --git a/be/src/exec/exchange/vdata_stream_recvr.h b/be/src/exec/exchange/vdata_stream_recvr.h index ccdb849c576f08..da139218b6b59b 100644 --- a/be/src/exec/exchange/vdata_stream_recvr.h +++ b/be/src/exec/exchange/vdata_stream_recvr.h @@ -48,12 +48,12 @@ #include "runtime/runtime_profile.h" #include "runtime/task_execution_context.h" #include "runtime/thread_context.h" -#include "runtime/workload_group/workload_group.h" #include "util/stopwatch.hpp" namespace doris { class MemTracker; class PBlock; +class PTransmitDataParams; class MemTrackerLimiter; class RuntimeState; diff --git a/be/src/exec/operator/materialization_opertor.cpp b/be/src/exec/operator/materialization_opertor.cpp index 1a475af0bccfa6..7ac146aa75dcd2 100644 --- a/be/src/exec/operator/materialization_opertor.cpp +++ b/be/src/exec/operator/materialization_opertor.cpp @@ -33,6 +33,7 @@ #include "exec/operator/operator.h" #include "exec/rowid_fetcher.h" #include "exec/scan/file_scanner.h" +#include "runtime/workload_group/workload_group.h" #include "util/brpc_client_cache.h" #include "util/brpc_closure.h" #include "util/pretty_printer.h" diff --git a/be/src/exec/pipeline/dependency.h b/be/src/exec/pipeline/dependency.h index 0f2a4726a156e3..a8b7ad4402beb9 100644 --- a/be/src/exec/pipeline/dependency.h +++ b/be/src/exec/pipeline/dependency.h @@ -28,6 +28,7 @@ #include #include +#include #include #include #include diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index def2cf473aea77..2825091f190a8c 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -123,6 +123,7 @@ #include "exec/spill/spill_file.h" #include "io/fs/stream_load_pipe.h" #include "load/stream_load/new_load_stream_mgr.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/result_buffer_mgr.h" diff --git a/be/src/exec/pipeline/task_scheduler.h b/be/src/exec/pipeline/task_scheduler.h index 7d8a76ef584303..b44f0b123be211 100644 --- a/be/src/exec/pipeline/task_scheduler.h +++ b/be/src/exec/pipeline/task_scheduler.h @@ -31,7 +31,6 @@ #include "exec/pipeline/pipeline_task.h" #include "exec/pipeline/task_queue.h" #include "runtime/query_context.h" -#include "runtime/workload_group/workload_group.h" #include "util/thread.h" #include "util/uid_util.h" diff --git a/be/src/exec/rowid_fetcher.h b/be/src/exec/rowid_fetcher.h index 790f9cf17e7e4e..795854a0ebcf55 100644 --- a/be/src/exec/rowid_fetcher.h +++ b/be/src/exec/rowid_fetcher.h @@ -22,7 +22,9 @@ #include #include +#include #include +#include #include #include #include diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 8851d9b2c05724..6bdf487f20428b 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -88,6 +88,7 @@ #include "runtime/descriptors.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" +#include "service/backend_options.h" namespace cctz { class time_zone; diff --git a/be/src/exec/scan/meta_scanner.cpp b/be/src/exec/scan/meta_scanner.cpp index 3b82ff160021a2..f0dad1e0197127 100644 --- a/be/src/exec/scan/meta_scanner.cpp +++ b/be/src/exec/scan/meta_scanner.cpp @@ -38,6 +38,7 @@ #include "core/data_type/define_primitive_type.h" #include "core/types.h" #include "format/table/parquet_metadata_reader.h" +#include "runtime/cluster_info.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" diff --git a/be/src/exec/scan/scanner_context.cpp b/be/src/exec/scan/scanner_context.cpp index dfa5569fcb5e8e..7fbf6ed951c997 100644 --- a/be/src/exec/scan/scanner_context.cpp +++ b/be/src/exec/scan/scanner_context.cpp @@ -45,6 +45,8 @@ #include "runtime/exec_env.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "runtime/workload_management/resource_context.h" #include "storage/tablet/tablet.h" #include "util/time.h" #include "util/uid_util.h" @@ -53,6 +55,18 @@ namespace doris { using namespace std::chrono_literals; +// ==================== ScanTask ==================== +ScanTask::ScanTask(std::weak_ptr delegate_scanner) : scanner(delegate_scanner) { + _resource_ctx = thread_context()->resource_ctx(); + DorisMetrics::instance()->scanner_task_cnt->increment(1); +} + +ScanTask::~ScanTask() { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_resource_ctx->memory_context()->mem_tracker()); + DorisMetrics::instance()->scanner_task_cnt->increment(-1); + cached_block.reset(); +} + // ==================== ScannerContext ==================== ScannerContext::ScannerContext(RuntimeState* state, ScanLocalStateBase* local_state, const TupleDescriptor* output_tuple_desc, diff --git a/be/src/exec/scan/scanner_context.h b/be/src/exec/scan/scanner_context.h index 013deef35443a2..b5730897cdb286 100644 --- a/be/src/exec/scan/scanner_context.h +++ b/be/src/exec/scan/scanner_context.h @@ -43,6 +43,7 @@ namespace doris { +class ResourceContext; class RuntimeState; class TupleDescriptor; class WorkloadGroup; @@ -96,16 +97,9 @@ class ScanTask { COMPLETED, // finished with result or error, waiting to be collected by scan node EOS, // finished and no more data, waiting to be collected by scan node }; - ScanTask(std::weak_ptr delegate_scanner) : scanner(delegate_scanner) { - _resource_ctx = thread_context()->resource_ctx(); - DorisMetrics::instance()->scanner_task_cnt->increment(1); - } + ScanTask(std::weak_ptr delegate_scanner); - ~ScanTask() { - SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_resource_ctx->memory_context()->mem_tracker()); - DorisMetrics::instance()->scanner_task_cnt->increment(-1); - cached_block.reset(); - } + ~ScanTask(); private: // whether current scanner is finished diff --git a/be/src/exec/scan/task_executor/tools/simulator/simulation_split.h b/be/src/exec/scan/task_executor/tools/simulator/simulation_split.h index 8fa4c0cf555464..9e732a6858df5f 100644 --- a/be/src/exec/scan/task_executor/tools/simulator/simulation_split.h +++ b/be/src/exec/scan/task_executor/tools/simulator/simulation_split.h @@ -19,6 +19,7 @@ #include #include +#include #include #include #include diff --git a/be/src/exec/sink/autoinc_buffer.cpp b/be/src/exec/sink/autoinc_buffer.cpp index 645d2b73c34379..35670424b129df 100644 --- a/be/src/exec/sink/autoinc_buffer.cpp +++ b/be/src/exec/sink/autoinc_buffer.cpp @@ -24,6 +24,7 @@ #include "common/logging.h" #include "common/status.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_profile.h" #include "util/client_cache.h" diff --git a/be/src/exec/sink/vrow_distribution.cpp b/be/src/exec/sink/vrow_distribution.cpp index ac3ce5d8715061..6f952f31be8e05 100644 --- a/be/src/exec/sink/vrow_distribution.cpp +++ b/be/src/exec/sink/vrow_distribution.cpp @@ -35,6 +35,7 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "exec/sink/writer/vtablet_writer.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/query_context.h" #include "runtime/runtime_state.h" diff --git a/be/src/exec/sink/writer/async_result_writer.cpp b/be/src/exec/sink/writer/async_result_writer.cpp index 2fa2f6f92a7418..206f6adc8445d7 100644 --- a/be/src/exec/sink/writer/async_result_writer.cpp +++ b/be/src/exec/sink/writer/async_result_writer.cpp @@ -25,6 +25,7 @@ #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/runtime_state.h" +#include "runtime/workload_group/workload_group.h" namespace doris { class ObjectPool; diff --git a/be/src/exec/sink/writer/maxcompute/vmc_table_writer.cpp b/be/src/exec/sink/writer/maxcompute/vmc_table_writer.cpp index a7818ea01d27b2..df440d4b5db355 100644 --- a/be/src/exec/sink/writer/maxcompute/vmc_table_writer.cpp +++ b/be/src/exec/sink/writer/maxcompute/vmc_table_writer.cpp @@ -22,6 +22,7 @@ #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "format/transformer/vjni_format_transformer.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/uid_util.h" diff --git a/be/src/exec/sink/writer/vtablet_writer.cpp b/be/src/exec/sink/writer/vtablet_writer.cpp index 5bd43f88726625..dfd1aa03a1a529 100644 --- a/be/src/exec/sink/writer/vtablet_writer.cpp +++ b/be/src/exec/sink/writer/vtablet_writer.cpp @@ -65,6 +65,7 @@ #include "exec/sink/vtablet_finder.h" #include "exprs/vexpr.h" #include "exprs/vexpr_fwd.h" +#include "load/memtable/memtable_memory_limiter.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "runtime/memory/memory_reclamation.h" @@ -72,6 +73,7 @@ #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "runtime/thread_context.h" +#include "runtime/workload_group/workload_group.h" #include "service/backend_options.h" #include "storage/binlog.h" #include "storage/tablet_info.h" diff --git a/be/src/exec/sink/writer/vtablet_writer_v2.cpp b/be/src/exec/sink/writer/vtablet_writer_v2.cpp index 7eb0138beb714a..1b790f966ae578 100644 --- a/be/src/exec/sink/writer/vtablet_writer_v2.cpp +++ b/be/src/exec/sink/writer/vtablet_writer_v2.cpp @@ -54,6 +54,7 @@ #include "exec/sink/load_stream_stub.h" // IWYU pragma: keep #include "exec/sink/vtablet_block_convertor.h" #include "exec/sink/vtablet_finder.h" +#include "load/memtable/memtable_memory_limiter.h" namespace doris { diff --git a/be/src/exec/sink/writer/vwal_writer.cpp b/be/src/exec/sink/writer/vwal_writer.cpp index 3f3b78b211b3de..a1c8c1397b5a0e 100644 --- a/be/src/exec/sink/writer/vwal_writer.cpp +++ b/be/src/exec/sink/writer/vwal_writer.cpp @@ -23,6 +23,7 @@ #include #include "io/fs/encrypted_fs_factory.h" +#include "io/fs/local_file_system.h" #include "util/debug_points.h" namespace doris { diff --git a/be/src/exprs/aggregate/aggregate_function.h b/be/src/exprs/aggregate/aggregate_function.h index c556303476d9a9..ac780b7c025a0d 100644 --- a/be/src/exprs/aggregate/aggregate_function.h +++ b/be/src/exprs/aggregate/aggregate_function.h @@ -45,6 +45,7 @@ namespace doris { class Arena; class IColumn; class IDataType; +class QueryContext; struct AggregateFunctionAttr { bool is_window_function {false}; diff --git a/be/src/exprs/function/function_java_udf.cpp b/be/src/exprs/function/function_java_udf.cpp index 4a0aecff0862df..2c7e12494bb837 100644 --- a/be/src/exprs/function/function_java_udf.cpp +++ b/be/src/exprs/function/function_java_udf.cpp @@ -19,6 +19,7 @@ #include +#include #include #include @@ -29,6 +30,7 @@ #include "runtime/exec_env.h" #include "runtime/user_function_cache.h" #include "util/jni-util.h" +#include "util/threadpool.h" const char* EXECUTOR_CLASS = "org/apache/doris/udf/UdfExecutor"; const char* EXECUTOR_CTOR_SIGNATURE = "([B)V"; diff --git a/be/src/exprs/vectorized_agg_fn.h b/be/src/exprs/vectorized_agg_fn.h index 707bb468dc7c3a..caf8369dee2942 100644 --- a/be/src/exprs/vectorized_agg_fn.h +++ b/be/src/exprs/vectorized_agg_fn.h @@ -23,6 +23,7 @@ #include #include "common/be_mock_util.h" +#include "common/factory_creator.h" #include "common/status.h" #include "core/data_type/data_type.h" #include "exec/sort/sort_description.h" diff --git a/be/src/format/parquet/parquet_predicate.h b/be/src/format/parquet/parquet_predicate.h index 7ad8c847801bdd..a0f6c75bdefb8b 100644 --- a/be/src/format/parquet/parquet_predicate.h +++ b/be/src/format/parquet/parquet_predicate.h @@ -32,6 +32,7 @@ #include "format/parquet/parquet_column_convert.h" #include "format/parquet/parquet_common.h" #include "format/parquet/schema_desc.h" +#include "io/fs/file_reader.h" #include "storage/olap_scan_common.h" #include "storage/segment/row_ranges.h" #include "util/timezone_utils.h" diff --git a/be/src/format/table/paimon_jni_reader.cpp b/be/src/format/table/paimon_jni_reader.cpp index 9855b808f760c6..04184ed9b956a7 100644 --- a/be/src/format/table/paimon_jni_reader.cpp +++ b/be/src/format/table/paimon_jni_reader.cpp @@ -26,6 +26,7 @@ #include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" +#include "storage/options.h" #include "util/string_util.h" #include "util/uid_util.h" diff --git a/be/src/information_schema/schema_active_queries_scanner.cpp b/be/src/information_schema/schema_active_queries_scanner.cpp index bceac0347b517f..f6633b45e895dc 100644 --- a/be/src/information_schema/schema_active_queries_scanner.cpp +++ b/be/src/information_schema/schema_active_queries_scanner.cpp @@ -20,6 +20,7 @@ #include "core/block/block.h" #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" diff --git a/be/src/information_schema/schema_authentication_integrations_scanner.cpp b/be/src/information_schema/schema_authentication_integrations_scanner.cpp index 37b9e6811baebb..3f3943326470b9 100644 --- a/be/src/information_schema/schema_authentication_integrations_scanner.cpp +++ b/be/src/information_schema/schema_authentication_integrations_scanner.cpp @@ -23,6 +23,7 @@ #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" #include "information_schema/schema_helper.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" diff --git a/be/src/information_schema/schema_backend_configuration_scanner.cpp b/be/src/information_schema/schema_backend_configuration_scanner.cpp index ebdf6cc59de2e9..c266c0a9b7d260 100644 --- a/be/src/information_schema/schema_backend_configuration_scanner.cpp +++ b/be/src/information_schema/schema_backend_configuration_scanner.cpp @@ -24,6 +24,7 @@ #include "core/block/block.h" #include "core/data_type/define_primitive_type.h" #include "core/string_ref.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" diff --git a/be/src/information_schema/schema_backend_ms_rpc_table_throttlers_scanner.cpp b/be/src/information_schema/schema_backend_ms_rpc_table_throttlers_scanner.cpp index a19eea752c54e5..339fe0e9c94ede 100644 --- a/be/src/information_schema/schema_backend_ms_rpc_table_throttlers_scanner.cpp +++ b/be/src/information_schema/schema_backend_ms_rpc_table_throttlers_scanner.cpp @@ -26,6 +26,7 @@ #include "core/block/block.h" #include "core/data_type/define_primitive_type.h" #include "core/string_ref.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" diff --git a/be/src/information_schema/schema_database_properties_scanner.cpp b/be/src/information_schema/schema_database_properties_scanner.cpp index dcb4810f171bdb..9fb6c8364faf01 100644 --- a/be/src/information_schema/schema_database_properties_scanner.cpp +++ b/be/src/information_schema/schema_database_properties_scanner.cpp @@ -21,6 +21,7 @@ #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" #include "information_schema/schema_helper.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" diff --git a/be/src/information_schema/schema_file_cache_info_scanner.cpp b/be/src/information_schema/schema_file_cache_info_scanner.cpp index 46d94e0c7f4b6b..0ee8328dce643a 100644 --- a/be/src/information_schema/schema_file_cache_info_scanner.cpp +++ b/be/src/information_schema/schema_file_cache_info_scanner.cpp @@ -20,7 +20,10 @@ #include "core/block/block.h" #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" +#include "io/cache/block_file_cache_factory.h" #include "io/cache/file_cache_common.h" +#include "io/cache/fs_file_cache_storage.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" diff --git a/be/src/information_schema/schema_partitions_scanner.cpp b/be/src/information_schema/schema_partitions_scanner.cpp index 4939fb2970247f..506fdae3907744 100644 --- a/be/src/information_schema/schema_partitions_scanner.cpp +++ b/be/src/information_schema/schema_partitions_scanner.cpp @@ -25,6 +25,7 @@ #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" #include "information_schema/schema_helper.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" diff --git a/be/src/information_schema/schema_role_mappings_scanner.cpp b/be/src/information_schema/schema_role_mappings_scanner.cpp index 99e5211fbd88a0..f0c08f23230c16 100644 --- a/be/src/information_schema/schema_role_mappings_scanner.cpp +++ b/be/src/information_schema/schema_role_mappings_scanner.cpp @@ -23,6 +23,7 @@ #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" #include "information_schema/schema_helper.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" diff --git a/be/src/information_schema/schema_table_options_scanner.cpp b/be/src/information_schema/schema_table_options_scanner.cpp index e102853429b868..2c19a73e6f831e 100644 --- a/be/src/information_schema/schema_table_options_scanner.cpp +++ b/be/src/information_schema/schema_table_options_scanner.cpp @@ -21,6 +21,7 @@ #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" #include "information_schema/schema_helper.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" diff --git a/be/src/information_schema/schema_table_properties_scanner.cpp b/be/src/information_schema/schema_table_properties_scanner.cpp index cd6680be7601a4..81c4fb38374a6e 100644 --- a/be/src/information_schema/schema_table_properties_scanner.cpp +++ b/be/src/information_schema/schema_table_properties_scanner.cpp @@ -21,6 +21,7 @@ #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" #include "information_schema/schema_helper.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" diff --git a/be/src/information_schema/schema_table_stream_consumption_scanner.cpp b/be/src/information_schema/schema_table_stream_consumption_scanner.cpp index 5259b7b8db145b..c484533ae7a5d9 100644 --- a/be/src/information_schema/schema_table_stream_consumption_scanner.cpp +++ b/be/src/information_schema/schema_table_stream_consumption_scanner.cpp @@ -25,6 +25,7 @@ #include "core/string_ref.h" #include "gen_cpp/FrontendService_types.h" #include "information_schema/schema_helper.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" diff --git a/be/src/information_schema/schema_table_streams_scanner.cpp b/be/src/information_schema/schema_table_streams_scanner.cpp index 0c9697341ae929..520e170c338185 100644 --- a/be/src/information_schema/schema_table_streams_scanner.cpp +++ b/be/src/information_schema/schema_table_streams_scanner.cpp @@ -25,6 +25,7 @@ #include "core/string_ref.h" #include "gen_cpp/FrontendService_types.h" #include "information_schema/schema_helper.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" diff --git a/be/src/information_schema/schema_view_dependency_scanner.cpp b/be/src/information_schema/schema_view_dependency_scanner.cpp index 2a7f684b0e1f2b..a3e6842f736fb1 100644 --- a/be/src/information_schema/schema_view_dependency_scanner.cpp +++ b/be/src/information_schema/schema_view_dependency_scanner.cpp @@ -25,6 +25,7 @@ #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" #include "information_schema/schema_helper.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" diff --git a/be/src/information_schema/schema_workload_group_privileges.cpp b/be/src/information_schema/schema_workload_group_privileges.cpp index 627344036abc82..458d89239b6d06 100644 --- a/be/src/information_schema/schema_workload_group_privileges.cpp +++ b/be/src/information_schema/schema_workload_group_privileges.cpp @@ -20,6 +20,7 @@ #include "core/block/block.h" #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" diff --git a/be/src/information_schema/schema_workload_groups_scanner.cpp b/be/src/information_schema/schema_workload_groups_scanner.cpp index 7375809b45e538..527696de3cd759 100644 --- a/be/src/information_schema/schema_workload_groups_scanner.cpp +++ b/be/src/information_schema/schema_workload_groups_scanner.cpp @@ -20,6 +20,7 @@ #include "core/block/block.h" #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" diff --git a/be/src/information_schema/schema_workload_sched_policy_scanner.cpp b/be/src/information_schema/schema_workload_sched_policy_scanner.cpp index eb82b26b8769f6..12a73f8ec1928d 100644 --- a/be/src/information_schema/schema_workload_sched_policy_scanner.cpp +++ b/be/src/information_schema/schema_workload_sched_policy_scanner.cpp @@ -20,6 +20,7 @@ #include "core/block/block.h" #include "core/data_type/data_type_factory.hpp" #include "core/string_ref.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "util/client_cache.h" diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index 9a01d07d136a62..ac12f5211829b2 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/be/src/io/cache/peer_file_cache_reader.cpp b/be/src/io/cache/peer_file_cache_reader.cpp index ace961ede0640d..76cffab8d4467e 100644 --- a/be/src/io/cache/peer_file_cache_reader.cpp +++ b/be/src/io/cache/peer_file_cache_reader.cpp @@ -32,6 +32,9 @@ #include "runtime/exec_env.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" +#include "runtime/workload_group/workload_group.h" +#include "runtime/workload_management/io_throttle.h" +#include "runtime/workload_management/resource_context.h" #include "util/brpc_client_cache.h" #include "util/bvar_helper.h" #include "util/debug_points.h" diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index 7dee26bf99e67d..cf1cfa443dc210 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -49,6 +49,7 @@ #include "load/stream_load/stream_load_context.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" +#include "service/backend_options.h" #include "util/s3_uri.h" #include "util/s3_util.h" #include "util/uid_util.h" diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp index 386c3e4192c3d6..599507ad751873 100644 --- a/be/src/io/fs/buffered_reader.cpp +++ b/be/src/io/fs/buffered_reader.cpp @@ -35,7 +35,9 @@ #include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_management/io_throttle.h" +#include "runtime/workload_management/resource_context.h" #include "util/slice.h" #include "util/threadpool.h" namespace doris { diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp index 6e363636980d19..2f734c9356b663 100644 --- a/be/src/io/fs/hdfs_file_reader.cpp +++ b/be/src/io/fs/hdfs_file_reader.cpp @@ -34,7 +34,9 @@ #include "io/hdfs_util.h" #include "runtime/file_scan_profile.h" #include "runtime/thread_context.h" +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_management/io_throttle.h" +#include "runtime/workload_management/resource_context.h" #include "service/backend_options.h" namespace doris::io { diff --git a/be/src/io/fs/local_file_reader.cpp b/be/src/io/fs/local_file_reader.cpp index 8cdc1e67663527..50bac007224d97 100644 --- a/be/src/io/fs/local_file_reader.cpp +++ b/be/src/io/fs/local_file_reader.cpp @@ -37,7 +37,9 @@ #include "cpp/sync_point.h" #include "io/fs/err_utils.h" #include "runtime/thread_context.h" +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_management/io_throttle.h" +#include "runtime/workload_management/resource_context.h" #include "storage/data_dir.h" #include "storage/olap_common.h" #include "storage/options.h" diff --git a/be/src/io/fs/multi_table_pipe.cpp b/be/src/io/fs/multi_table_pipe.cpp index 92a333e1431c5b..8a8d9fd3393e87 100644 --- a/be/src/io/fs/multi_table_pipe.cpp +++ b/be/src/io/fs/multi_table_pipe.cpp @@ -28,6 +28,7 @@ #include "common/status.h" #include "load/stream_load/new_load_stream_mgr.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/runtime_state.h" diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index af8dde36d2df50..8a6e5c0fdc4978 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -40,7 +40,9 @@ #include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_management/io_throttle.h" +#include "runtime/workload_management/resource_context.h" #include "util/bvar_helper.h" #include "util/concurrency_stats.h" #include "util/debug_points.h" diff --git a/be/src/load/channel/load_channel.cpp b/be/src/load/channel/load_channel.cpp index 9ec0de1943e288..4b483824d36852 100644 --- a/be/src/load/channel/load_channel.cpp +++ b/be/src/load/channel/load_channel.cpp @@ -28,6 +28,7 @@ #include "runtime/fragment_mgr.h" #include "runtime/memory/mem_tracker.h" #include "runtime/thread_context.h" +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_group/workload_group_manager.h" #include "storage/storage_engine.h" #include "util/debug_points.h" diff --git a/be/src/load/channel/load_channel.h b/be/src/load/channel/load_channel.h index 2702cf192fecf3..77550eb3317a9a 100644 --- a/be/src/load/channel/load_channel.h +++ b/be/src/load/channel/load_channel.h @@ -29,6 +29,7 @@ #include "common/status.h" #include "runtime/runtime_profile.h" +#include "runtime/workload_group/workload_group_fwd.h" #include "runtime/workload_management/resource_context.h" #include "util/uid_util.h" diff --git a/be/src/load/group_commit/group_commit_mgr.cpp b/be/src/load/group_commit/group_commit_mgr.cpp index b0dd97ed1303d2..60b1cef7e55cba 100644 --- a/be/src/load/group_commit/group_commit_mgr.cpp +++ b/be/src/load/group_commit/group_commit_mgr.cpp @@ -27,6 +27,7 @@ #include "common/config.h" #include "common/status.h" #include "exec/pipeline/dependency.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/memory/mem_tracker_limiter.h" diff --git a/be/src/load/group_commit/wal/wal_manager.cpp b/be/src/load/group_commit/wal/wal_manager.cpp index e7962cc3396ce1..826c9781a20e2e 100644 --- a/be/src/load/group_commit/wal/wal_manager.cpp +++ b/be/src/load/group_commit/wal/wal_manager.cpp @@ -33,8 +33,10 @@ #include "io/fs/local_file_system.h" #include "load/group_commit/wal/wal_dirs_info.h" #include "load/group_commit/wal/wal_reader.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" +#include "storage/options.h" #include "util/parse_util.h" namespace doris { diff --git a/be/src/load/group_commit/wal/wal_table.cpp b/be/src/load/group_commit/wal/wal_table.cpp index 060df525050807..34109affe9b656 100644 --- a/be/src/load/group_commit/wal/wal_table.cpp +++ b/be/src/load/group_commit/wal/wal_table.cpp @@ -23,6 +23,7 @@ #include "io/fs/local_file_system.h" #include "io/fs/stream_load_pipe.h" #include "load/group_commit/wal/wal_manager.h" +#include "runtime/cluster_info.h" #include "runtime/fragment_mgr.h" #include "service/http/action/http_stream.h" #include "service/http/action/stream_load.h" diff --git a/be/src/load/group_commit/wal/wal_writer.cpp b/be/src/load/group_commit/wal/wal_writer.cpp index 9147eb56a17c5e..57f5e6c59c2b14 100644 --- a/be/src/load/group_commit/wal/wal_writer.cpp +++ b/be/src/load/group_commit/wal/wal_writer.cpp @@ -29,6 +29,7 @@ #include "io/fs/local_file_system.h" #include "io/fs/path.h" #include "load/group_commit/wal/wal_manager.h" +#include "runtime/cluster_info.h" #include "storage/storage_engine.h" #include "util/thrift_rpc_helper.h" diff --git a/be/src/load/memtable/memtable.cpp b/be/src/load/memtable/memtable.cpp index c774e9ce2bc24e..5530cee63aa61c 100644 --- a/be/src/load/memtable/memtable.cpp +++ b/be/src/load/memtable/memtable.cpp @@ -37,6 +37,7 @@ #include "runtime/exec_env.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" +#include "runtime/workload_management/resource_context.h" #include "storage/olap_define.h" #include "storage/tablet/tablet_schema.h" #include "util/debug_points.h" diff --git a/be/src/load/memtable/memtable_flush_executor.cpp b/be/src/load/memtable/memtable_flush_executor.cpp index b2959fbdb0ae3f..41f8c1da67d3cc 100644 --- a/be/src/load/memtable/memtable_flush_executor.cpp +++ b/be/src/load/memtable/memtable_flush_executor.cpp @@ -31,6 +31,8 @@ #include "common/signal_handler.h" #include "load/memtable/memtable.h" #include "runtime/thread_context.h" +#include "runtime/workload_group/workload_group.h" +#include "runtime/workload_management/resource_context.h" #include "storage/binlog.h" #include "storage/rowset/group_rowset_writer.h" #include "storage/rowset/rowset_writer.h" diff --git a/be/src/load/memtable/memtable_memory_limiter.h b/be/src/load/memtable/memtable_memory_limiter.h index 11c5c0fdd61413..aad457b5031322 100644 --- a/be/src/load/memtable/memtable_memory_limiter.h +++ b/be/src/load/memtable/memtable_memory_limiter.h @@ -19,11 +19,14 @@ #include +#include #include +#include +#include #include "common/status.h" #include "runtime/memory/mem_tracker.h" -#include "runtime/workload_group/workload_group.h" +#include "runtime/workload_group/workload_group_fwd.h" #include "util/countdown_latch.h" #include "util/stopwatch.hpp" diff --git a/be/src/load/memtable/memtable_writer.cpp b/be/src/load/memtable/memtable_writer.cpp index 0555a813b0bef2..5a6eb301cf10ae 100644 --- a/be/src/load/memtable/memtable_writer.cpp +++ b/be/src/load/memtable/memtable_writer.cpp @@ -35,6 +35,8 @@ #include "load/memtable/memtable_memory_limiter.h" #include "runtime/exec_env.h" #include "runtime/memory/mem_tracker.h" +#include "runtime/workload_group/workload_group.h" +#include "runtime/workload_management/resource_context.h" #include "service/backend_options.h" #include "storage/rowset/beta_rowset_writer.h" #include "storage/rowset/group_rowset_writer.h" @@ -64,6 +66,14 @@ MemTableWriter::~MemTableWriter() { _mem_table.reset(); } +uint64_t MemTableWriter::workload_group_id() const { + auto wg = _resource_ctx->workload_group(); + if (wg != nullptr) { + return wg->id(); + } + return 0; +} + Status MemTableWriter::init(std::shared_ptr rowset_writer, TabletSchemaSPtr tablet_schema, std::shared_ptr partial_update_info, diff --git a/be/src/load/memtable/memtable_writer.h b/be/src/load/memtable/memtable_writer.h index eacbba402e7ed2..aa59a7fae6e9d4 100644 --- a/be/src/load/memtable/memtable_writer.h +++ b/be/src/load/memtable/memtable_writer.h @@ -103,13 +103,7 @@ class MemTableWriter { uint64_t flush_running_count() const; - uint64_t workload_group_id() const { - auto wg = _resource_ctx->workload_group(); - if (wg != nullptr) { - return wg->id(); - } - return 0; - } + uint64_t workload_group_id() const; private: Status _flush_memtable(); diff --git a/be/src/load/routine_load/routine_load_task_executor.cpp b/be/src/load/routine_load/routine_load_task_executor.cpp index 1c1937673ea89b..f1d126f8643bd6 100644 --- a/be/src/load/routine_load/routine_load_task_executor.cpp +++ b/be/src/load/routine_load/routine_load_task_executor.cpp @@ -50,6 +50,7 @@ #include "load/stream_load/new_load_stream_mgr.h" #include "load/stream_load/stream_load_context.h" #include "load/stream_load/stream_load_executor.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/memory/memory_profile.h" #include "service/backend_options.h" diff --git a/be/src/load/stream_load/stream_load_executor.cpp b/be/src/load/stream_load/stream_load_executor.cpp index 9b307c932d4776..7ea6f8147b86f5 100644 --- a/be/src/load/stream_load/stream_load_executor.cpp +++ b/be/src/load/stream_load/stream_load_executor.cpp @@ -44,6 +44,7 @@ #include "load/message_body_sink.h" #include "load/stream_load/new_load_stream_mgr.h" #include "load/stream_load/stream_load_context.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/runtime_state.h" diff --git a/be/src/runtime/exec_env.cpp b/be/src/runtime/exec_env.cpp index 3177c73cdc0fbd..1e96caff334800 100644 --- a/be/src/runtime/exec_env.cpp +++ b/be/src/runtime/exec_env.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include "common/config.h" @@ -28,7 +29,10 @@ #include "exec/exchange/vdata_stream_mgr.h" #include "exec/sink/delta_writer_v2_pool.h" #include "exec/sink/load_stream_map_pool.h" +#include "io/cache/fs_file_cache_storage.h" #include "load/channel/load_stream_mgr.h" +#include "load/memtable/memtable_memory_limiter.h" +#include "runtime/cluster_info.h" #include "runtime/fragment_mgr.h" #include "runtime/frontend_info.h" #include "storage/index/index_writer.h" // TmpFileDirs, completed here rather than in the header @@ -36,6 +40,7 @@ #include "storage/storage_engine.h" #include "storage/tablet/tablet_manager.h" #include "util/debug_util.h" +#include "util/threadpool.h" // ThreadPool must be complete: unique_ptr member assignment #include "util/time.h" namespace doris { @@ -54,6 +59,18 @@ void ExecEnv::set_storage_engine(std::unique_ptr&& engine) { void ExecEnv::set_write_cooldown_meta_executors() { _write_cooldown_meta_executors = std::make_unique(); } +void ExecEnv::set_memtable_memory_limiter(MemTableMemoryLimiter* limiter) { + _memtable_memory_limiter.reset(limiter); +} +void ExecEnv::set_file_cache_open_fd_cache(std::unique_ptr&& fd_cache) { + _file_cache_open_fd_cache = std::move(fd_cache); +} +void ExecEnv::set_non_block_close_thread_pool(std::unique_ptr&& pool) { + _non_block_close_thread_pool = std::move(pool); +} +void ExecEnv::set_s3_file_upload_thread_pool(std::unique_ptr&& pool) { + _s3_file_upload_thread_pool = std::move(pool); +} #endif // BE_TEST Result ExecEnv::get_tablet(int64_t tablet_id, SyncRowsetStats* sync_stats, @@ -87,7 +104,7 @@ void ExecEnv::clear_stream_mgr() { std::vector ExecEnv::get_frontends() { std::lock_guard lg(_frontends_lock); std::vector infos; - for (const auto& cur_fe : _frontends) { + for (const auto& cur_fe : *_frontends) { infos.push_back(cur_fe.second.info); } return infos; @@ -98,17 +115,17 @@ void ExecEnv::update_frontends(const std::vector& new_fe_infos) { std::set dropped_fes; - for (const auto& cur_fe : _frontends) { + for (const auto& cur_fe : *_frontends) { dropped_fes.insert(cur_fe.first); } for (const auto& coming_fe_info : new_fe_infos) { - auto itr = _frontends.find(coming_fe_info.coordinator_address); + auto itr = _frontends->find(coming_fe_info.coordinator_address); - if (itr == _frontends.end()) { + if (itr == _frontends->end()) { LOG(INFO) << "A completely new frontend, " << PrintFrontendInfo(coming_fe_info); - _frontends.insert(std::pair( + _frontends->insert(std::pair( coming_fe_info.coordinator_address, FrontendInfo {coming_fe_info, GetCurrentTimeMicros() / 1000, /*first time*/ GetCurrentTimeMicros() / 1000 /*last time*/})); @@ -138,7 +155,7 @@ void ExecEnv::update_frontends(const std::vector& new_fe_infos) { for (const auto& dropped_fe : dropped_fes) { LOG(INFO) << "Frontend " << PrintThriftNetworkAddress(dropped_fe) << " has already been dropped, remove it"; - _frontends.erase(dropped_fe); + _frontends->erase(dropped_fe); } } @@ -148,7 +165,7 @@ std::map ExecEnv::get_running_frontends() { const int expired_duration = config::fe_expire_duration_seconds * 1000; const auto now = GetCurrentTimeMicros() / 1000; - for (const auto& pair : _frontends) { + for (const auto& pair : *_frontends) { auto& brpc_addr = pair.first; auto& fe_info = pair.second; diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index affa6a0f0489ad..885828b8eff443 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -17,27 +17,19 @@ #pragma once -#include - #include #include #include #include #include +#include #include #include #include "common/config.h" #include "common/multi_version.h" #include "common/status.h" -#include "information_schema/schema_routine_load_job_scanner.h" -#include "io/cache/fs_file_cache_storage.h" -#include "load/memtable/memtable_memory_limiter.h" -#include "runtime/cluster_info.h" -#include "runtime/frontend_info.h" // TODO(zhiqiang): find a way to remove this include header -#include "storage/options.h" #include "storage/tablet/tablet_fwd.h" -#include "util/threadpool.h" namespace orc { class MemoryPool; @@ -58,7 +50,11 @@ struct WriteCooldownMetaExecutors; class TokenBucketRateLimiterHolder; using S3RateLimiterHolder = TokenBucketRateLimiterHolder; class MSRpcRateLimitServices; +class ThreadPool; +struct StorePath; +struct CachePath; namespace io { +class FDCache; class FileCacheFactory; class HdfsMgr; class PackedFileManager; @@ -95,6 +91,7 @@ class LoadPathMgr; class NewLoadStreamMgr; class MemTrackerLimiter; class MemTracker; +class MemTableMemoryLimiter; struct TrackerLimiterGroup; class BaseStorageEngine; class ResultBufferMgr; @@ -114,6 +111,12 @@ class PFunctionService_Stub; template class ClientCache; class HeartbeatFlags; +class ClusterInfo; +struct FrontendInfo; +// Thrift-generated types (gen_cpp), forward-declared so their headers stay out +// of every TU that includes ExecEnv. +class TFrontendInfo; +class TNetworkAddress; class FrontendServiceClient; class FileMetaCache; class GroupCommitMgr; @@ -325,9 +328,9 @@ class ExecEnv { void set_tmp_file_dir(std::unique_ptr tmp_file_dirs); void set_ready() { _s_ready = true; } void set_not_ready() { _s_ready = false; } - void set_memtable_memory_limiter(MemTableMemoryLimiter* limiter) { - _memtable_memory_limiter.reset(limiter); - } + // Defined out of line: resetting the unique_ptr deletes the old pointee, + // which would require MemTableMemoryLimiter to be complete here. + void set_memtable_memory_limiter(MemTableMemoryLimiter* limiter); void set_cluster_info(ClusterInfo* cluster_info) { this->_cluster_info = cluster_info; } void set_new_load_stream_mgr(std::unique_ptr&& new_load_stream_mgr); void clear_new_load_stream_mgr(); @@ -359,16 +362,14 @@ class ExecEnv { _s_tracking_memory.store(tracking_memory, std::memory_order_release); } void set_orc_memory_pool(orc::MemoryPool* pool) { _orc_memory_pool = pool; } - void set_non_block_close_thread_pool(std::unique_ptr&& pool) { - _non_block_close_thread_pool = std::move(pool); - } - void set_s3_file_upload_thread_pool(std::unique_ptr&& pool) { - _s3_file_upload_thread_pool = std::move(pool); - } + // Defined out of line: assigning the unique_ptr destroys the old pointee, + // which would require ThreadPool to be complete here. + void set_non_block_close_thread_pool(std::unique_ptr&& pool); + void set_s3_file_upload_thread_pool(std::unique_ptr&& pool); void set_file_cache_factory(io::FileCacheFactory* factory) { _file_cache_factory = factory; } - void set_file_cache_open_fd_cache(std::unique_ptr&& fd_cache) { - _file_cache_open_fd_cache = std::move(fd_cache); - } + // Defined out of line: assigning the unique_ptr destroys the old pointee, + // which would require io::FDCache to be complete here. + void set_file_cache_open_fd_cache(std::unique_ptr&& fd_cache); #endif // WARN: The following setter methods are intended for use in test code and // offline tools (like meta_tool) ONLY. They should NOT be called in the @@ -539,8 +540,10 @@ class ExecEnv { std::unique_ptr _write_cooldown_meta_executors; std::mutex _frontends_lock; - // ip:brpc_port -> frontend_indo - std::map _frontends; + // ip:brpc_port -> frontend_info. Held behind unique_ptr because std::map + // requires a complete mapped type, and frontend_info.h would drag + // gen_cpp/HeartbeatService_types.h into every TU that includes ExecEnv. + std::unique_ptr> _frontends; GroupCommitMgr* _group_commit_mgr = nullptr; CdcClientMgr* _cdc_client_mgr = nullptr; diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 97d068f0754f2e..00351e1b1aac10 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -79,6 +79,7 @@ #include "runtime/exec_env.h" #include "runtime/external_scan_context_mgr.h" #include "runtime/fragment_mgr.h" +#include "runtime/frontend_info.h" #include "runtime/heartbeat_flags.h" #include "runtime/index_policy/index_policy_mgr.h" #include "runtime/memory/cache_manager.h" @@ -189,7 +190,7 @@ ThreadPool* ExecEnv::non_block_close_thread_pool() { return _non_block_close_thread_pool.get(); } -ExecEnv::ExecEnv() = default; +ExecEnv::ExecEnv() : _frontends(std::make_unique>()) {} ExecEnv::~ExecEnv() { destroy(); diff --git a/be/src/runtime/fragment_mgr.h b/be/src/runtime/fragment_mgr.h index ab78c18555a640..cb69312c20b598 100644 --- a/be/src/runtime/fragment_mgr.h +++ b/be/src/runtime/fragment_mgr.h @@ -54,6 +54,7 @@ extern bvar::Status g_fragment_last_active_time; class PipelineFragmentContext; class QueryContext; class ExecEnv; +struct FrontendInfo; class ThreadPool; class PExecPlanFragmentStartRequest; class PMergeFilterRequest; diff --git a/be/src/runtime/memory/global_memory_arbitrator.h b/be/src/runtime/memory/global_memory_arbitrator.h index 5058fe8a33aa2f..ce49e99061b04c 100644 --- a/be/src/runtime/memory/global_memory_arbitrator.h +++ b/be/src/runtime/memory/global_memory_arbitrator.h @@ -17,6 +17,12 @@ #pragma once +#include +#include +#include +#include +#include + #include "runtime/process_profile.h" #include "util/mem_info.h" diff --git a/be/src/runtime/memory/memory_profile.cpp b/be/src/runtime/memory/memory_profile.cpp index 35f3ee372b4f57..7790ee0fe062c3 100644 --- a/be/src/runtime/memory/memory_profile.cpp +++ b/be/src/runtime/memory/memory_profile.cpp @@ -18,6 +18,7 @@ #include "runtime/memory/memory_profile.h" #include "bvar/reducer.h" +#include "load/memtable/memtable_memory_limiter.h" #include "runtime/exec_env.h" #include "runtime/memory/global_memory_arbitrator.h" #include "runtime/memory/jemalloc_control.h" diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp index 3a0f0c7972fc6a..5de03560c289d2 100644 --- a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp +++ b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp @@ -22,6 +22,8 @@ #include "common/exception.h" #include "common/signal_handler.h" #include "runtime/exec_env.h" +#include "runtime/workload_group/workload_group.h" +#include "util/pretty_printer.h" namespace doris { @@ -102,4 +104,87 @@ void ThreadMemTrackerMgr::detach_limiter_tracker() { _last_attach_snapshots_stack.pop_back(); } +doris::Status ThreadMemTrackerMgr::try_reserve(int64_t size, TryReserveChecker checker) { + DCHECK(size >= 0); + CHECK(init()); + DCHECK(_limiter_tracker); + memory_orphan_check(); + // if _reserved_mem not equal to 0, repeat reserve, + // _untracked_mem store bytes that not synchronized to process reserved memory. + flush_untracked_mem(); + auto wg_ptr = _wg_wptr.lock(); + + bool task_limit_checker = static_cast(checker) & 1; + bool workload_group_limit_checker = static_cast(checker) & 2; + bool process_limit_checker = static_cast(checker) & 4; + + if (task_limit_checker) { + if (!_limiter_tracker->try_reserve(size)) { + auto err_msg = fmt::format( + "reserve memory failed, size: {}, because query memory exceeded, memory " + "tracker: {}, " + "consumption: {}, limit: {}, peak: {}", + PrettyPrinter::print_bytes(size), _limiter_tracker->label(), + PrettyPrinter::print_bytes(_limiter_tracker->consumption()), + PrettyPrinter::print_bytes(_limiter_tracker->limit()), + PrettyPrinter::print_bytes(_limiter_tracker->peak_consumption())); + return doris::Status::Error(err_msg); + } + } else { + _limiter_tracker->reserve(size); + } + + if (wg_ptr) { + if (workload_group_limit_checker) { + if (!wg_ptr->try_add_wg_refresh_interval_memory_growth(size)) { + auto err_msg = fmt::format( + "reserve memory failed, size: {}, because workload group memory exceeded, " + "workload group: {}", + PrettyPrinter::print_bytes(size), wg_ptr->memory_debug_string()); + _limiter_tracker->release(size); // rollback + _limiter_tracker->shrink_reserved(size); // rollback + return doris::Status::Error(err_msg); + } + } else { + wg_ptr->add_wg_refresh_interval_memory_growth(size); + } + } + + if (process_limit_checker) { + if (!doris::GlobalMemoryArbitrator::try_reserve_process_memory(size)) { + auto err_msg = fmt::format( + "reserve memory failed, size: {}, because proccess memory exceeded, {}", + PrettyPrinter::print_bytes(size), + GlobalMemoryArbitrator::process_mem_log_str()); + _limiter_tracker->release(size); // rollback + _limiter_tracker->shrink_reserved(size); // rollback + if (wg_ptr) { + wg_ptr->sub_wg_refresh_interval_memory_growth(size); // rollback + } + return doris::Status::Error(err_msg); + } + } else { + doris::GlobalMemoryArbitrator::reserve_process_memory(size); + } + + _reserved_mem += size; + DCHECK(_reserved_mem >= 0); + return doris::Status::OK(); +} + +void ThreadMemTrackerMgr::shrink_reserved() { + if (_reserved_mem != 0) { + memory_orphan_check(); + doris::GlobalMemoryArbitrator::shrink_process_reserved(_reserved_mem + _untracked_mem); + _limiter_tracker->shrink_reserved(_reserved_mem + _untracked_mem); + _limiter_tracker->release(_reserved_mem); + auto wg_ptr = _wg_wptr.lock(); + if (wg_ptr) { + wg_ptr->sub_wg_refresh_interval_memory_growth(_reserved_mem); + } + _untracked_mem = 0; + _reserved_mem = 0; + } +} + } // namespace doris diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h b/be/src/runtime/memory/thread_mem_tracker_mgr.h index a24e32b205abe7..d9a915c439bc62 100644 --- a/be/src/runtime/memory/thread_mem_tracker_mgr.h +++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h @@ -34,7 +34,7 @@ #include "runtime/memory/global_memory_arbitrator.h" #include "runtime/memory/mem_tracker.h" #include "runtime/memory/mem_tracker_limiter.h" -#include "runtime/workload_group/workload_group.h" +#include "runtime/workload_group/workload_group_fwd.h" #include "util/stack_util.h" namespace doris { @@ -303,87 +303,4 @@ inline void ThreadMemTrackerMgr::flush_untracked_mem() { _stop_consume = false; } -inline doris::Status ThreadMemTrackerMgr::try_reserve(int64_t size, TryReserveChecker checker) { - DCHECK(size >= 0); - CHECK(init()); - DCHECK(_limiter_tracker); - memory_orphan_check(); - // if _reserved_mem not equal to 0, repeat reserve, - // _untracked_mem store bytes that not synchronized to process reserved memory. - flush_untracked_mem(); - auto wg_ptr = _wg_wptr.lock(); - - bool task_limit_checker = static_cast(checker) & 1; - bool workload_group_limit_checker = static_cast(checker) & 2; - bool process_limit_checker = static_cast(checker) & 4; - - if (task_limit_checker) { - if (!_limiter_tracker->try_reserve(size)) { - auto err_msg = fmt::format( - "reserve memory failed, size: {}, because query memory exceeded, memory " - "tracker: {}, " - "consumption: {}, limit: {}, peak: {}", - PrettyPrinter::print_bytes(size), _limiter_tracker->label(), - PrettyPrinter::print_bytes(_limiter_tracker->consumption()), - PrettyPrinter::print_bytes(_limiter_tracker->limit()), - PrettyPrinter::print_bytes(_limiter_tracker->peak_consumption())); - return doris::Status::Error(err_msg); - } - } else { - _limiter_tracker->reserve(size); - } - - if (wg_ptr) { - if (workload_group_limit_checker) { - if (!wg_ptr->try_add_wg_refresh_interval_memory_growth(size)) { - auto err_msg = fmt::format( - "reserve memory failed, size: {}, because workload group memory exceeded, " - "workload group: {}", - PrettyPrinter::print_bytes(size), wg_ptr->memory_debug_string()); - _limiter_tracker->release(size); // rollback - _limiter_tracker->shrink_reserved(size); // rollback - return doris::Status::Error(err_msg); - } - } else { - wg_ptr->add_wg_refresh_interval_memory_growth(size); - } - } - - if (process_limit_checker) { - if (!doris::GlobalMemoryArbitrator::try_reserve_process_memory(size)) { - auto err_msg = fmt::format( - "reserve memory failed, size: {}, because proccess memory exceeded, {}", - PrettyPrinter::print_bytes(size), - GlobalMemoryArbitrator::process_mem_log_str()); - _limiter_tracker->release(size); // rollback - _limiter_tracker->shrink_reserved(size); // rollback - if (wg_ptr) { - wg_ptr->sub_wg_refresh_interval_memory_growth(size); // rollback - } - return doris::Status::Error(err_msg); - } - } else { - doris::GlobalMemoryArbitrator::reserve_process_memory(size); - } - - _reserved_mem += size; - DCHECK(_reserved_mem >= 0); - return doris::Status::OK(); -} - -inline void ThreadMemTrackerMgr::shrink_reserved() { - if (_reserved_mem != 0) { - memory_orphan_check(); - doris::GlobalMemoryArbitrator::shrink_process_reserved(_reserved_mem + _untracked_mem); - _limiter_tracker->shrink_reserved(_reserved_mem + _untracked_mem); - _limiter_tracker->release(_reserved_mem); - auto wg_ptr = _wg_wptr.lock(); - if (wg_ptr) { - wg_ptr->sub_wg_refresh_interval_memory_growth(_reserved_mem); - } - _untracked_mem = 0; - _reserved_mem = 0; - } -} - } // namespace doris diff --git a/be/src/runtime/query_cache/query_cache.h b/be/src/runtime/query_cache/query_cache.h index f727159eac2915..ed353c08ad7350 100644 --- a/be/src/runtime/query_cache/query_cache.h +++ b/be/src/runtime/query_cache/query_cache.h @@ -18,6 +18,8 @@ #pragma once #include +#include +#include #include #include #include diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp index b593e3d3a56c76..8dce687a47dd46 100644 --- a/be/src/runtime/query_context.cpp +++ b/be/src/runtime/query_context.cpp @@ -37,6 +37,7 @@ #include "exec/pipeline/pipeline_fragment_context.h" #include "exec/runtime_filter/runtime_filter_definitions.h" #include "exec/spill/spill_file_manager.h" +#include "io/cache/block_file_cache_factory.h" #include "io/cache/remote_scan_cache_write_limiter.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" @@ -44,6 +45,7 @@ #include "runtime/runtime_query_statistics_mgr.h" #include "runtime/runtime_state.h" #include "runtime/thread_context.h" +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_group/workload_group_manager.h" #include "runtime/workload_management/query_task_controller.h" #include "storage/olap_common.h" diff --git a/be/src/runtime/query_context.h b/be/src/runtime/query_context.h index 2dedffb75762c3..89e10a22753dca 100644 --- a/be/src/runtime/query_context.h +++ b/be/src/runtime/query_context.h @@ -39,7 +39,7 @@ #include "runtime/exec_env.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/runtime_predicate.h" -#include "runtime/workload_group/workload_group.h" +#include "runtime/workload_group/workload_group_fwd.h" #include "runtime/workload_management/resource_context.h" #include "util/hash_util.hpp" #include "util/threadpool.h" diff --git a/be/src/runtime/runtime_query_statistics_mgr.cpp b/be/src/runtime/runtime_query_statistics_mgr.cpp index 1533897855c307..b23edea3c54075 100644 --- a/be/src/runtime/runtime_query_statistics_mgr.cpp +++ b/be/src/runtime/runtime_query_statistics_mgr.cpp @@ -37,7 +37,9 @@ #include "common/status.h" #include "core/block/block.h" #include "information_schema/schema_scanner_helper.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" +#include "runtime/workload_group/workload_group.h" #include "util/client_cache.h" #include "util/debug_util.h" #include "util/threadpool.h" diff --git a/be/src/runtime/runtime_query_statistics_mgr.h b/be/src/runtime/runtime_query_statistics_mgr.h index 84546fb3d2b6ff..62660c848e7d69 100644 --- a/be/src/runtime/runtime_query_statistics_mgr.h +++ b/be/src/runtime/runtime_query_statistics_mgr.h @@ -34,6 +34,7 @@ namespace doris { class Block; +class TReportExecStatusParams; class RuntimeQueryStatisticsMgr { public: diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 0551c2e533689c..cf844dd8c45dcb 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -43,14 +43,16 @@ #include "common/factory_creator.h" #include "common/status.h" #include "exec/scan/vector_search_user_params.h" -#include "io/fs/s3_file_system.h" #include "runtime/runtime_profile.h" #include "runtime/task_execution_context.h" -#include "runtime/workload_group/workload_group.h" +#include "runtime/workload_group/workload_group_fwd.h" #include "util/debug_util.h" #include "util/timezone_utils.h" namespace doris { +namespace io { +class S3FileSystem; +} // namespace io class RuntimeFilter; inline int32_t get_execution_rpc_timeout_ms(int32_t execution_timeout_sec) { diff --git a/be/src/runtime/small_file_mgr.cpp b/be/src/runtime/small_file_mgr.cpp index f383a89e846162..7df6ffbdc51937 100644 --- a/be/src/runtime/small_file_mgr.cpp +++ b/be/src/runtime/small_file_mgr.cpp @@ -38,6 +38,7 @@ #include "common/status.h" #include "io/fs/file_system.h" #include "io/fs/local_file_system.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "service/http/http_client.h" #include "util/md5.h" diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index 19976172c06715..e0a1f2ad592908 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -48,6 +48,7 @@ #include "io/fs/remote_file_system.h" #include "io/fs/s3_file_system.h" #include "io/hdfs_builder.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "service/http/http_client.h" #include "storage/data_dir.h" diff --git a/be/src/runtime/thread_context.cpp b/be/src/runtime/thread_context.cpp index 83f38369d6cf86..da2a377c409391 100644 --- a/be/src/runtime/thread_context.cpp +++ b/be/src/runtime/thread_context.cpp @@ -18,12 +18,34 @@ #include "runtime/thread_context.h" #include "common/signal_handler.h" +#include "runtime/exec_env.h" #include "runtime/query_context.h" #include "runtime/runtime_state.h" +#include "runtime/workload_management/resource_context.h" namespace doris { class MemTracker; +void ThreadContext::attach_task(const std::shared_ptr& rc) { + // will only attach_task at the beginning of the thread function, there should be no duplicate attach_task. + DCHECK(resource_ctx_ == nullptr); + // Validation of `rc` and its sub-objects is performed by the + // AttachTask::init() / SwitchResourceContext constructor entry + // points before any thread-local or signal mutation, so a thrown + // FatalError does not leak thread-local handle counts or leave a + // stale signal task id behind. + resource_ctx_ = rc; + thread_mem_tracker_mgr->attach_limiter_tracker(rc->memory_context()->mem_tracker(), + rc->workload_group()); + thread_mem_tracker_mgr->enable_wait_gc(); +} + +std::shared_ptr ThreadContext::_make_orphan_resource_ctx() { + auto ctx = ResourceContext::create_shared(); + ctx->memory_context()->set_mem_tracker(ExecEnv::GetInstance()->orphan_mem_tracker()); + return ctx; +} + void AttachTask::init(const std::shared_ptr& rc) { // Validate the ResourceContext chain before mutating any thread-local // or signal state. If any link is null we throw immediately, so the diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h index 332cd0f6545685..b310de8dd53f2c 100644 --- a/be/src/runtime/thread_context.h +++ b/be/src/runtime/thread_context.h @@ -27,10 +27,8 @@ #include "common/exception.h" #include "common/logging.h" #include "common/macros.h" -#include "runtime/exec_env.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/memory/thread_mem_tracker_mgr.h" -#include "runtime/workload_management/resource_context.h" #include "util/defer_op.h" // IWYU pragma: keep // Used to tracking query/load/compaction/e.g. execution thread memory usage. @@ -135,6 +133,8 @@ namespace doris { class ThreadContext; class MemTracker; +class QueryContext; +class ResourceContext; class RuntimeState; class SwitchResourceContext; @@ -165,19 +165,7 @@ class ThreadContext { ~ThreadContext() = default; - void attach_task(const std::shared_ptr& rc) { - // will only attach_task at the beginning of the thread function, there should be no duplicate attach_task. - DCHECK(resource_ctx_ == nullptr); - // Validation of `rc` and its sub-objects is performed by the - // AttachTask::init() / SwitchResourceContext constructor entry - // points before any thread-local or signal mutation, so a thrown - // FatalError does not leak thread-local handle counts or leave a - // stale signal task id behind. - resource_ctx_ = rc; - thread_mem_tracker_mgr->attach_limiter_tracker(rc->memory_context()->mem_tracker(), - rc->workload_group()); - thread_mem_tracker_mgr->enable_wait_gc(); - } + void attach_task(const std::shared_ptr& rc); void detach_task() { resource_ctx_.reset(); @@ -193,12 +181,8 @@ class ThreadContext { #endif if (is_attach_task()) { return resource_ctx_; - } else { - auto ctx = ResourceContext::create_shared(); - ctx->memory_context()->set_mem_tracker( - doris::ExecEnv::GetInstance()->orphan_mem_tracker()); - return ctx; } + return _make_orphan_resource_ctx(); } static std::string get_thread_id() { @@ -218,6 +202,11 @@ class ThreadContext { private: friend class SwitchResourceContext; + + // Cold fallback for threads without an attached task; defined in the .cpp + // so this header does not need the full ResourceContext / ExecEnv types. + static std::shared_ptr _make_orphan_resource_ctx(); + std::shared_ptr resource_ctx_; }; diff --git a/be/src/runtime/workload_group/workload_group.h b/be/src/runtime/workload_group/workload_group.h index 2a310bf66a0f32..bff74bdeef5b3a 100644 --- a/be/src/runtime/workload_group/workload_group.h +++ b/be/src/runtime/workload_group/workload_group.h @@ -30,6 +30,7 @@ #include "common/factory_creator.h" #include "common/status.h" +#include "runtime/workload_group/workload_group_fwd.h" #include "service/backend_options.h" #include "util/hash_util.hpp" @@ -269,8 +270,6 @@ class WorkloadGroup : public std::enable_shared_from_this { std::shared_ptr _wg_metrics {nullptr}; }; -using WorkloadGroupPtr = std::shared_ptr; - struct WorkloadGroupInfo { const uint64_t id = 0; const std::string name = ""; diff --git a/be/src/runtime/workload_group/workload_group_fwd.h b/be/src/runtime/workload_group/workload_group_fwd.h new file mode 100644 index 00000000000000..a117c8f139f8b7 --- /dev/null +++ b/be/src/runtime/workload_group/workload_group_fwd.h @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris { + +class WorkloadGroup; + +using WorkloadGroupPtr = std::shared_ptr; + +} // namespace doris diff --git a/be/src/runtime/workload_group/workload_group_manager.cpp b/be/src/runtime/workload_group/workload_group_manager.cpp index bfc3b94509d8f3..2a283f29972261 100644 --- a/be/src/runtime/workload_group/workload_group_manager.cpp +++ b/be/src/runtime/workload_group/workload_group_manager.cpp @@ -30,6 +30,7 @@ #include "exec/pipeline/task_scheduler.h" #include "exec/scan/scanner_scheduler.h" #include "information_schema/schema_scanner_helper.h" +#include "runtime/cluster_info.h" #include "runtime/memory/global_memory_arbitrator.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/workload_group/workload_group.h" diff --git a/be/src/runtime/workload_management/cpu_context.cpp b/be/src/runtime/workload_management/cpu_context.cpp index ee7a0024b0c43b..2e112dcd7b0ca9 100644 --- a/be/src/runtime/workload_management/cpu_context.cpp +++ b/be/src/runtime/workload_management/cpu_context.cpp @@ -19,6 +19,7 @@ #include +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_management/resource_context.h" namespace doris { diff --git a/be/src/runtime/workload_management/cpu_context.h b/be/src/runtime/workload_management/cpu_context.h index 2c20d86b2f5024..9f6d51d1541bce 100644 --- a/be/src/runtime/workload_management/cpu_context.h +++ b/be/src/runtime/workload_management/cpu_context.h @@ -19,7 +19,6 @@ #include "common/factory_creator.h" #include "runtime/runtime_profile.h" -#include "runtime/workload_group/workload_group.h" namespace doris { diff --git a/be/src/runtime/workload_management/query_task_controller.cpp b/be/src/runtime/workload_management/query_task_controller.cpp index 8e0e5c59584868..b089d77261024d 100644 --- a/be/src/runtime/workload_management/query_task_controller.cpp +++ b/be/src/runtime/workload_management/query_task_controller.cpp @@ -21,6 +21,7 @@ #include "exec/pipeline/pipeline_fragment_context.h" #include "runtime/query_context.h" +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_management/task_controller.h" namespace doris { diff --git a/be/src/runtime/workload_management/resource_context.cpp b/be/src/runtime/workload_management/resource_context.cpp index 2eb753fc6796fd..85378ad287ef2c 100644 --- a/be/src/runtime/workload_management/resource_context.cpp +++ b/be/src/runtime/workload_management/resource_context.cpp @@ -17,9 +17,11 @@ #include "runtime/workload_management/resource_context.h" +#include #include #include +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_management/query_task_controller.h" #include "util/time.h" diff --git a/be/src/runtime/workload_management/resource_context.h b/be/src/runtime/workload_management/resource_context.h index 969e899da64bf4..114b5b88367267 100644 --- a/be/src/runtime/workload_management/resource_context.h +++ b/be/src/runtime/workload_management/resource_context.h @@ -17,14 +17,12 @@ #pragma once -#include - #include #include "common/factory_creator.h" #include "common/multi_version.h" #include "runtime/runtime_profile.h" -#include "runtime/workload_group/workload_group.h" +#include "runtime/workload_group/workload_group_fwd.h" #include "runtime/workload_management/cpu_context.h" #include "runtime/workload_management/io_context.h" #include "runtime/workload_management/memory_context.h" @@ -32,6 +30,8 @@ namespace doris { +class TQueryStatistics; + // Every task should have its own resource context. And BE may adjust the resource // context during running. // ResourceContext contains many contexts or controller, the task could implements their diff --git a/be/src/runtime/workload_management/workload_sched_policy.cpp b/be/src/runtime/workload_management/workload_sched_policy.cpp index a3cbc003ac08a5..4504a7a849b449 100644 --- a/be/src/runtime/workload_management/workload_sched_policy.cpp +++ b/be/src/runtime/workload_management/workload_sched_policy.cpp @@ -17,6 +17,7 @@ #include "runtime/workload_management/workload_sched_policy.h" +#include "runtime/workload_group/workload_group.h" #include "runtime/workload_management/resource_context.h" #include "runtime/workload_management/task_controller.h" #include "util/time.h" diff --git a/be/src/runtime/workload_management/workload_sched_policy_mgr.cpp b/be/src/runtime/workload_management/workload_sched_policy_mgr.cpp index cba5ad50b560fe..95d48634e03bd1 100644 --- a/be/src/runtime/workload_management/workload_sched_policy_mgr.cpp +++ b/be/src/runtime/workload_management/workload_sched_policy_mgr.cpp @@ -22,6 +22,7 @@ #include "common/config.h" #include "runtime/fragment_mgr.h" #include "runtime/workload_management/resource_context.h" +#include "util/thread.h" namespace doris { diff --git a/be/src/runtime/workload_management/workload_sched_policy_mgr.h b/be/src/runtime/workload_management/workload_sched_policy_mgr.h index ecdb0913bb9219..b914d7b2b3a643 100644 --- a/be/src/runtime/workload_management/workload_sched_policy_mgr.h +++ b/be/src/runtime/workload_management/workload_sched_policy_mgr.h @@ -17,12 +17,16 @@ #pragma once +#include + #include "runtime/exec_env.h" #include "runtime/workload_management/workload_sched_policy.h" #include "util/countdown_latch.h" namespace doris { +class Thread; + class WorkloadSchedPolicyMgr { public: WorkloadSchedPolicyMgr() : _stop_latch(0) {} diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt index 64ed8651417748..938a3711c43748 100644 --- a/be/src/service/CMakeLists.txt +++ b/be/src/service/CMakeLists.txt @@ -65,7 +65,13 @@ if (${MAKE_TEST} STREQUAL "OFF" AND ${BUILD_BENCHMARK} STREQUAL "OFF") # The same library also duplicates zstd, lz4, snappy, bzip2 and zlib symbols. Those are C # ABIs, stable across versions and layout-free, so they are left alone until something # shows otherwise -- unlike RocksDB, whose C++ objects are what actually corrupt. - target_link_options(doris_be PRIVATE "-Wl,--exclude-libs,librocksdb.a") + # + # Linux-only: Apple's ld rejects --exclude-libs (a GNU ld option), and macOS does not + # need it -- dyld's two-level namespace already binds each loaded library to the copy + # it was linked against, so a JNI library carrying its own RocksDB never resolves to ours. + if (NOT OS_MACOSX) + target_link_options(doris_be PRIVATE "-Wl,--exclude-libs,librocksdb.a") + endif() target_link_libraries(doris_be ${DORIS_LINK_LIBS} diff --git a/be/src/service/http/action/compaction_action.cpp b/be/src/service/http/action/compaction_action.cpp index 83371b0995fd28..2c0ba26657153b 100644 --- a/be/src/service/http/action/compaction_action.cpp +++ b/be/src/service/http/action/compaction_action.cpp @@ -32,6 +32,7 @@ #include "common/logging.h" #include "common/metrics/doris_metrics.h" #include "common/status.h" +#include "service/backend_options.h" #include "service/http/http_channel.h" #include "service/http/http_headers.h" #include "service/http/http_request.h" diff --git a/be/src/service/http/action/http_stream.cpp b/be/src/service/http/action/http_stream.cpp index 3dab083ae72286..cd80517942e4fc 100644 --- a/be/src/service/http/action/http_stream.cpp +++ b/be/src/service/http/action/http_stream.cpp @@ -46,6 +46,7 @@ #include "load/stream_load/stream_load_context.h" #include "load/stream_load/stream_load_executor.h" #include "load/stream_load/stream_load_recorder.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "service/http/http_channel.h" diff --git a/be/src/service/http/action/stream_load.cpp b/be/src/service/http/action/stream_load.cpp index d5739f96155f50..1b67543f21536c 100644 --- a/be/src/service/http/action/stream_load.cpp +++ b/be/src/service/http/action/stream_load.cpp @@ -55,6 +55,7 @@ #include "load/stream_load/stream_load_context.h" #include "load/stream_load/stream_load_executor.h" #include "load/stream_load/stream_load_recorder.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "service/http/http_channel.h" #include "service/http/http_common.h" diff --git a/be/src/service/http/http_client.cpp b/be/src/service/http/http_client.cpp index 31022bd04743ab..7141e6f6012d0a 100644 --- a/be/src/service/http/http_client.cpp +++ b/be/src/service/http/http_client.cpp @@ -27,6 +27,8 @@ #include "common/cast_set.h" #include "common/config.h" #include "common/status.h" +#include "io/fs/local_file_system.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "service/http/http_headers.h" #include "util/security.h" diff --git a/be/src/service/http/http_handler_with_auth.cpp b/be/src/service/http/http_handler_with_auth.cpp index 4a30b5640b8905..23be67c654c5d1 100644 --- a/be/src/service/http/http_handler_with_auth.cpp +++ b/be/src/service/http/http_handler_with_auth.cpp @@ -19,6 +19,7 @@ #include +#include "runtime/cluster_info.h" #include "service/http/http_channel.h" #include "service/http/utils.h" #include "util/client_cache.h" diff --git a/be/src/storage/index/indexed_column_reader.cpp b/be/src/storage/index/indexed_column_reader.cpp index 29510f21452c2e..dad71453409bf7 100644 --- a/be/src/storage/index/indexed_column_reader.cpp +++ b/be/src/storage/index/indexed_column_reader.cpp @@ -22,6 +22,7 @@ #include #include "common/status.h" +#include "io/fs/file_reader.h" #include "io/io_common.h" #include "storage/key_coder.h" #include "storage/olap_common.h" diff --git a/be/src/storage/index/inverted/inverted_index_common_impl.h b/be/src/storage/index/inverted/inverted_index_common_impl.h index 021474158b36c0..fa9aa0792156d1 100644 --- a/be/src/storage/index/inverted/inverted_index_common_impl.h +++ b/be/src/storage/index/inverted/inverted_index_common_impl.h @@ -17,7 +17,18 @@ #pragma once +// CLucene is third-party code and is not clean under -Wconversion (which +// -Wshorten-64-to-32 belongs to). Whether its first expansion lands inside +// someone else's suppressed region depends on include order, so suppress it +// deliberately here. +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wconversion" +#endif #include +#ifdef __clang__ +#pragma clang diagnostic pop +#endif #include "common/logging.h" #include "storage/index/inverted/inverted_index_common.h" diff --git a/be/src/storage/olap_server.cpp b/be/src/storage/olap_server.cpp index f562f3d96f4915..21f682228bd192 100644 --- a/be/src/storage/olap_server.cpp +++ b/be/src/storage/olap_server.cpp @@ -58,6 +58,7 @@ #include "load/memtable/memtable_flush_executor.h" #include "runtime/memory/cache_manager.h" #include "runtime/memory/global_memory_arbitrator.h" +#include "service/backend_options.h" #include "storage/compaction/cold_data_compaction.h" #include "storage/compaction/compaction_permit_limiter.h" #include "storage/compaction/cumulative_compaction.h" diff --git a/be/src/storage/rowset/rowset_writer_context.h b/be/src/storage/rowset/rowset_writer_context.h index 58de04f8f26a5f..10b537522d7521 100644 --- a/be/src/storage/rowset/rowset_writer_context.h +++ b/be/src/storage/rowset/rowset_writer_context.h @@ -30,6 +30,7 @@ #include "io/fs/encrypted_fs_factory.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" #include "io/fs/packed_file_system.h" #include "runtime/exec_env.h" #include "storage/binlog.h" diff --git a/be/src/storage/rowset_version_mgr.cpp b/be/src/storage/rowset_version_mgr.cpp index aa6ed733c39658..d323382d5f14fe 100644 --- a/be/src/storage/rowset_version_mgr.cpp +++ b/be/src/storage/rowset_version_mgr.cpp @@ -39,6 +39,7 @@ #include "cloud/config.h" #include "common/status.h" #include "cpp/sync_point.h" +#include "runtime/cluster_info.h" #include "service/backend_options.h" #include "service/internal_service.h" #include "storage/olap_common.h" diff --git a/be/src/storage/segment/segment.h b/be/src/storage/segment/segment.h index c46f5dad268356..2c0b05a3516e2f 100644 --- a/be/src/storage/segment/segment.h +++ b/be/src/storage/segment/segment.h @@ -34,6 +34,7 @@ #include "common/status.h" // Status #include "core/column/column.h" #include "core/data_type/data_type.h" +#include "io/cache/file_cache_common.h" // io::UInt128Wrapper returned by value #include "io/fs/file_reader.h" #include "io/fs/file_reader_writer_fwd.h" #include "io/fs/file_system.h" diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index 84dd82fc932d4f..f473737d1de998 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -76,6 +76,7 @@ #include "runtime/runtime_predicate.h" #include "runtime/runtime_state.h" #include "runtime/thread_context.h" +#include "service/backend_options.h" #include "storage/binlog.h" #include "storage/compaction/collection_similarity.h" #include "storage/id_manager.h" diff --git a/be/src/storage/storage_engine.cpp b/be/src/storage/storage_engine.cpp index 2bc9e46b101378..8f95f4d4492526 100644 --- a/be/src/storage/storage_engine.cpp +++ b/be/src/storage/storage_engine.cpp @@ -59,6 +59,7 @@ #include "io/fs/local_file_system.h" #include "load/memtable/memtable_flush_executor.h" #include "load/stream_load/stream_load_recorder.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" #include "storage/binlog.h" #include "storage/data_dir.h" diff --git a/be/src/storage/task/engine_clone_task.cpp b/be/src/storage/task/engine_clone_task.cpp index e6fe3d3031d352..a214d4d81678f4 100644 --- a/be/src/storage/task/engine_clone_task.cpp +++ b/be/src/storage/task/engine_clone_task.cpp @@ -45,6 +45,7 @@ #include "io/fs/file_system.h" #include "io/fs/local_file_system.h" #include "io/fs/path.h" +#include "runtime/cluster_info.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" #include "service/http/http_client.h" diff --git a/be/src/udf/python/python_udf_meta.cpp b/be/src/udf/python/python_udf_meta.cpp index 4f21a045ab13b2..4e42197aab5e0c 100644 --- a/be/src/udf/python/python_udf_meta.cpp +++ b/be/src/udf/python/python_udf_meta.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include diff --git a/be/src/util/async_io.h b/be/src/util/async_io.h index 57587164b35964..a3bb60321bf5b1 100644 --- a/be/src/util/async_io.h +++ b/be/src/util/async_io.h @@ -19,6 +19,9 @@ #include +#include +#include + #include "io/fs/file_system.h" #include "runtime/thread_context.h" #include "storage/olap_define.h" diff --git a/be/src/util/brpc_client_cache.h b/be/src/util/brpc_client_cache.h index aa0ce4524e0596..510fbb4afa727f 100644 --- a/be/src/util/brpc_client_cache.h +++ b/be/src/util/brpc_client_cache.h @@ -44,6 +44,7 @@ #include "runtime/exec_env.h" #include "service/backend_options.h" #include "util/client_connection_provider.h" +#include "util/defer_op.h" #include "util/dns_cache.h" #include "util/network_util.h" diff --git a/be/test/cloud/cloud_compaction_test.cpp b/be/test/cloud/cloud_compaction_test.cpp index f7916ab511ef55..7a661f2e837b37 100644 --- a/be/test/cloud/cloud_compaction_test.cpp +++ b/be/test/cloud/cloud_compaction_test.cpp @@ -29,6 +29,7 @@ #include "cloud/cloud_tablet.h" #include "cloud/cloud_tablet_mgr.h" #include "cloud/config.h" +#include "io/fs/s3_file_system.h" #include "json2pb/json_to_pb.h" #include "storage/olap_common.h" #include "storage/rowset/rowset_factory.h" @@ -36,6 +37,7 @@ #include "storage/storage_policy.h" #include "storage/tablet/tablet_meta.h" #include "util/defer_op.h" +#include "util/s3_util.h" #include "util/uid_util.h" namespace doris { diff --git a/be/test/cloud/cloud_snapshot_mgr_test.cpp b/be/test/cloud/cloud_snapshot_mgr_test.cpp index 5b097be04d4685..f547f5e9f5bfa5 100644 --- a/be/test/cloud/cloud_snapshot_mgr_test.cpp +++ b/be/test/cloud/cloud_snapshot_mgr_test.cpp @@ -21,6 +21,8 @@ #include "cloud/cloud_storage_engine.h" #include "io/fs/remote_file_system.h" +#include "io/fs/s3_file_system.h" +#include "util/s3_util.h" namespace doris { using namespace cloud; diff --git a/be/test/exec/operator/spillable_operator_test_helper.cpp b/be/test/exec/operator/spillable_operator_test_helper.cpp index d51d6d17be16c5..dd39a8aa6b3b83 100644 --- a/be/test/exec/operator/spillable_operator_test_helper.cpp +++ b/be/test/exec/operator/spillable_operator_test_helper.cpp @@ -28,6 +28,7 @@ #include #include +#include "io/fs/local_file_system.h" #include "testutil/creators.h" namespace doris { diff --git a/be/test/exec/pipeline/multi_cast_data_streamer_test.cpp b/be/test/exec/pipeline/multi_cast_data_streamer_test.cpp index e9f1a58aeccf78..7d052419608d82 100644 --- a/be/test/exec/pipeline/multi_cast_data_streamer_test.cpp +++ b/be/test/exec/pipeline/multi_cast_data_streamer_test.cpp @@ -24,6 +24,7 @@ #include "exec/pipeline/dependency.h" #include "exec/spill/spill_file_manager.h" +#include "io/fs/local_file_system.h" #include "runtime/runtime_profile.h" #include "storage/olap_define.h" #include "testutil/column_helper.h" diff --git a/be/test/exec/scan/scanner_context_test.cpp b/be/test/exec/scan/scanner_context_test.cpp index 8be021454dd7b6..4eb76c27a219c9 100644 --- a/be/test/exec/scan/scanner_context_test.cpp +++ b/be/test/exec/scan/scanner_context_test.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "common/object_pool.h" diff --git a/be/test/format/native/native_reader_writer_test.cpp b/be/test/format/native/native_reader_writer_test.cpp index 3d8e9bf8f63e67..70e5e54dbd2800 100644 --- a/be/test/format/native/native_reader_writer_test.cpp +++ b/be/test/format/native/native_reader_writer_test.cpp @@ -37,6 +37,7 @@ #include "format/native/native_format.h" #include "format/native/native_reader.h" #include "format/transformer/vnative_transformer.h" +#include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "io/fs/path.h" #include "runtime/runtime_state.h" diff --git a/be/test/format/orc/orc_read_lines.cpp b/be/test/format/orc/orc_read_lines.cpp index be9bc0eae79251..25bffa06e8c12a 100644 --- a/be/test/format/orc/orc_read_lines.cpp +++ b/be/test/format/orc/orc_read_lines.cpp @@ -46,6 +46,7 @@ #include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" +#include "service/backend_options.h" #include "storage/segment/column_reader.h" #include "testutil/desc_tbl_builder.h" diff --git a/be/test/format/parquet/parquet_read_lines.cpp b/be/test/format/parquet/parquet_read_lines.cpp index 5511e5c6944695..835ae04bdf1434 100644 --- a/be/test/format/parquet/parquet_read_lines.cpp +++ b/be/test/format/parquet/parquet_read_lines.cpp @@ -48,6 +48,7 @@ #include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" +#include "service/backend_options.h" #include "storage/segment/column_reader.h" #include "util/timezone_utils.h" diff --git a/be/test/format/transformer/vorc_transformer_test.cpp b/be/test/format/transformer/vorc_transformer_test.cpp index 5f181e891fbeca..bb11d99f5023e5 100644 --- a/be/test/format/transformer/vorc_transformer_test.cpp +++ b/be/test/format/transformer/vorc_transformer_test.cpp @@ -36,6 +36,7 @@ #include "core/data_type/data_type_struct.h" #include "core/data_type_serde/orc_serde_utils.h" #include "format/table/iceberg/schema_parser.h" +#include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "runtime/runtime_state.h" #include "testutil/mock/mock_slot_ref.h" diff --git a/be/test/format_v2/native/native_reader_test.cpp b/be/test/format_v2/native/native_reader_test.cpp index 2745ecf852c38b..d3633956083b95 100644 --- a/be/test/format_v2/native/native_reader_test.cpp +++ b/be/test/format_v2/native/native_reader_test.cpp @@ -38,6 +38,7 @@ #include "exprs/vexpr_context.h" #include "format/native/native_format.h" #include "format_v2/column_mapper.h" +#include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "io/io_common.h" #include "runtime/descriptors.h" diff --git a/be/test/io/cache/block_file_cache_test.cpp b/be/test/io/cache/block_file_cache_test.cpp index 9b35ae10a325d1..22e6bae5d80be7 100644 --- a/be/test/io/cache/block_file_cache_test.cpp +++ b/be/test/io/cache/block_file_cache_test.cpp @@ -21,7 +21,9 @@ #include #include +#include #include +#include #include "io/cache/block_file_cache_test_common.h" #include "io/cache/remote_scan_cache_write_limiter.h" diff --git a/be/test/io/cache/block_file_cache_ttl_mgr_test.cpp b/be/test/io/cache/block_file_cache_ttl_mgr_test.cpp index 56f6d3d43d68e3..3aa00e67ab4b8c 100644 --- a/be/test/io/cache/block_file_cache_ttl_mgr_test.cpp +++ b/be/test/io/cache/block_file_cache_ttl_mgr_test.cpp @@ -36,6 +36,7 @@ #include "io/cache/file_block.h" #include "io/cache/file_cache_common.h" #include "runtime/exec_env.h" +#include "runtime/workload_group/workload_group_fwd.h" #include "storage/storage_engine.h" #include "storage/tablet/base_tablet.h" #include "util/slice.h" diff --git a/be/test/io/cache/cached_remote_file_reader_lock_wait_test.cpp b/be/test/io/cache/cached_remote_file_reader_lock_wait_test.cpp index 97b846a1db2e73..e007df2ccd1286 100644 --- a/be/test/io/cache/cached_remote_file_reader_lock_wait_test.cpp +++ b/be/test/io/cache/cached_remote_file_reader_lock_wait_test.cpp @@ -36,6 +36,7 @@ #include "io/cache/block_file_cache_factory.h" #include "io/cache/cached_remote_file_reader.h" #include "io/cache/file_cache_common.h" +#include "io/cache/fs_file_cache_storage.h" #include "io/fs/file_reader.h" #include "runtime/exec_env.h" #include "runtime/thread_context.h" diff --git a/be/test/io/client/s3_file_system_test.cpp b/be/test/io/client/s3_file_system_test.cpp index 1cc11876bde4c8..375ff3ff57f8e2 100644 --- a/be/test/io/client/s3_file_system_test.cpp +++ b/be/test/io/client/s3_file_system_test.cpp @@ -30,6 +30,7 @@ #include "common/config.h" #include "cpp/sync_point.h" +#include "io/fs/file_reader.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" #include "io/fs/obj_storage_client.h" diff --git a/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp b/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp index c0b27ac39c659b..9afa6c3cb939f0 100644 --- a/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp +++ b/be/test/load/delta_writer/delta_writer_cluster_key_test.cpp @@ -42,6 +42,7 @@ #include "gtest/gtest_pred_impl.h" #include "io/fs/local_file_system.h" #include "load/delta_writer/delta_writer.h" +#include "load/memtable/memtable_memory_limiter.h" #include "runtime/descriptor_helper.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" diff --git a/be/test/load/delta_writer/delta_writer_test.cpp b/be/test/load/delta_writer/delta_writer_test.cpp index 65b05dde7f1ece..e2b9419b8973d5 100644 --- a/be/test/load/delta_writer/delta_writer_test.cpp +++ b/be/test/load/delta_writer/delta_writer_test.cpp @@ -44,6 +44,7 @@ #include "exprs/function/cast/cast_to_datev2_impl.hpp" #include "gtest/gtest_pred_impl.h" #include "io/fs/local_file_system.h" +#include "load/memtable/memtable_memory_limiter.h" #include "runtime/descriptor_helper.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" diff --git a/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp b/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp index 70dd2e874bafc0..f1cf2424baa899 100644 --- a/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp +++ b/be/test/runtime/fragment_mgr_cross_cluster_cancel_test.cpp @@ -21,6 +21,7 @@ #include "runtime/descriptor_helper.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" +#include "runtime/frontend_info.h" #include "runtime/workload_group/workload_group_manager.h" namespace doris { diff --git a/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp b/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp index cbefa422ad2a77..716e5f2e9a4958 100644 --- a/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp +++ b/be/test/runtime/memory/thread_mem_tracker_mgr_test.cpp @@ -24,6 +24,7 @@ #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" #include "runtime/workload_group/workload_group_manager.h" +#include "runtime/workload_management/resource_context.h" namespace doris { diff --git a/be/test/runtime/routine_load_task_executor_test.cpp b/be/test/runtime/routine_load_task_executor_test.cpp index c50a010c01e022..fcbf2a1520bb76 100644 --- a/be/test/runtime/routine_load_task_executor_test.cpp +++ b/be/test/runtime/routine_load_task_executor_test.cpp @@ -33,6 +33,7 @@ #include "gtest/gtest_pred_impl.h" #include "load/stream_load/new_load_stream_mgr.h" #include "load/stream_load/stream_load_executor.h" +#include "runtime/cluster_info.h" #include "runtime/exec_env.h" namespace doris { diff --git a/be/test/runtime/snapshot_loader_test.cpp b/be/test/runtime/snapshot_loader_test.cpp index fb54b9e0a3280d..c3bb1046cda067 100644 --- a/be/test/runtime/snapshot_loader_test.cpp +++ b/be/test/runtime/snapshot_loader_test.cpp @@ -45,6 +45,7 @@ #include "io/fs/file_reader.h" #include "io/fs/local_file_system.h" #include "load/delta_writer/delta_writer.h" +#include "load/memtable/memtable_memory_limiter.h" #include "runtime/cluster_info.h" #include "runtime/descriptor_helper.h" #include "runtime/descriptors.h" diff --git a/be/test/runtime/thread_context_test.cpp b/be/test/runtime/thread_context_test.cpp index 88e8b5b64ba9ed..eb35894f710606 100644 --- a/be/test/runtime/thread_context_test.cpp +++ b/be/test/runtime/thread_context_test.cpp @@ -22,6 +22,7 @@ #include "gtest/gtest_pred_impl.h" #include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/workload_management/resource_context.h" namespace doris { diff --git a/be/test/runtime/workload_group/workload_group_manager_test.cpp b/be/test/runtime/workload_group/workload_group_manager_test.cpp index 397fedec71c3b0..5207daf1a9b530 100644 --- a/be/test/runtime/workload_group/workload_group_manager_test.cpp +++ b/be/test/runtime/workload_group/workload_group_manager_test.cpp @@ -34,6 +34,7 @@ #include "common/status.h" #include "exec/pipeline/dependency.h" #include "exec/spill/spill_file_manager.h" +#include "load/memtable/memtable_memory_limiter.h" #include "runtime/exec_env.h" #include "runtime/query_context.h" #include "runtime/runtime_query_statistics_mgr.h" diff --git a/be/test/service/http/file_cache_action_test.cpp b/be/test/service/http/file_cache_action_test.cpp index ce4f8b02372728..36ceeca0205afd 100644 --- a/be/test/service/http/file_cache_action_test.cpp +++ b/be/test/service/http/file_cache_action_test.cpp @@ -31,6 +31,7 @@ #include "io/cache/block_file_cache.h" #include "io/cache/block_file_cache_factory.h" #include "io/cache/file_cache_common.h" +#include "io/cache/fs_file_cache_storage.h" #include "runtime/exec_env.h" #include "service/http/http_request.h" #include "util/slice.h" diff --git a/be/test/service/http/http_client_test.cpp b/be/test/service/http/http_client_test.cpp index 5e3d5f1988412d..7b6cb2fd9f7b6c 100644 --- a/be/test/service/http/http_client_test.cpp +++ b/be/test/service/http/http_client_test.cpp @@ -29,6 +29,7 @@ #include #include "gtest/gtest_pred_impl.h" +#include "io/fs/local_file_system.h" #include "runtime/exec_env.h" #include "service/backend_service.h" #include "service/http/ev_http_server.h" diff --git a/be/test/storage/index/inverted/query_v2/boolean_query_test.cpp b/be/test/storage/index/inverted/query_v2/boolean_query_test.cpp index c9ceaba5288399..539e49e49e2225 100644 --- a/be/test/storage/index/inverted/query_v2/boolean_query_test.cpp +++ b/be/test/storage/index/inverted/query_v2/boolean_query_test.cpp @@ -24,6 +24,7 @@ #include #include "common/status.h" +#include "io/fs/local_file_system.h" #include "storage/index/index_query_context.h" #include "storage/index/inverted/analyzer/custom_analyzer.h" #include "storage/index/inverted/query_v2/bit_set_query/bit_set_query.h" diff --git a/be/test/storage/index/inverted/query_v2/multi_phrase_query_test.cpp b/be/test/storage/index/inverted/query_v2/multi_phrase_query_test.cpp index bb156f3e7d8b15..2d8b34b2bb7a77 100644 --- a/be/test/storage/index/inverted/query_v2/multi_phrase_query_test.cpp +++ b/be/test/storage/index/inverted/query_v2/multi_phrase_query_test.cpp @@ -24,6 +24,7 @@ #include #include "common/status.h" +#include "io/fs/local_file_system.h" #include "storage/index/index_query_context.h" #include "storage/index/inverted/analyzer/custom_analyzer.h" #include "storage/index/inverted/query/query_info.h" diff --git a/be/test/storage/index/inverted/query_v2/phrase_query_test.cpp b/be/test/storage/index/inverted/query_v2/phrase_query_test.cpp index f427152a585924..b974daee6a073a 100644 --- a/be/test/storage/index/inverted/query_v2/phrase_query_test.cpp +++ b/be/test/storage/index/inverted/query_v2/phrase_query_test.cpp @@ -24,6 +24,7 @@ #include #include "common/status.h" +#include "io/fs/local_file_system.h" #include "storage/index/index_query_context.h" #include "storage/index/inverted/analyzer/custom_analyzer.h" #include "storage/index/inverted/query/query_info.h" diff --git a/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp b/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp index f84972d84e8bef..397ecdae63b784 100644 --- a/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp +++ b/be/test/storage/index/inverted/query_v2/regexp_query_test.cpp @@ -25,6 +25,7 @@ #include #include "common/status.h" +#include "io/fs/local_file_system.h" #include "storage/index/index_query_context.h" #include "storage/index/inverted/analyzer/custom_analyzer.h" #include "storage/index/inverted/query_v2/regexp_query/regexp_weight.h" diff --git a/be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp b/be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp index af119cdd260217..ca89835fc1c7de 100644 --- a/be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp +++ b/be/test/storage/index/inverted/query_v2/wildcard_query_test.cpp @@ -25,6 +25,7 @@ #include #include "common/status.h" +#include "io/fs/local_file_system.h" #include "storage/index/index_query_context.h" #include "storage/index/inverted/analyzer/custom_analyzer.h" #include "storage/index/inverted/query_v2/wildcard_query/wildcard_weight.h" diff --git a/be/test/storage/metadata_adder_test.cpp b/be/test/storage/metadata_adder_test.cpp index b436a6c670cb2d..6b175d00e6f87e 100644 --- a/be/test/storage/metadata_adder_test.cpp +++ b/be/test/storage/metadata_adder_test.cpp @@ -21,6 +21,8 @@ #include "core/data_type/data_type_factory.hpp" #include "core/data_type/define_primitive_type.h" +#include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/tablet/tablet_schema.h" #include "storage/tablet/tablet_schema_helper.h" diff --git a/be/test/storage/schema_change/engine_storage_migration_task_test.cpp b/be/test/storage/schema_change/engine_storage_migration_task_test.cpp index b90e4e3346d6fb..4e0e1aabdf777b 100644 --- a/be/test/storage/schema_change/engine_storage_migration_task_test.cpp +++ b/be/test/storage/schema_change/engine_storage_migration_task_test.cpp @@ -39,6 +39,7 @@ #include "gtest/gtest_pred_impl.h" #include "io/fs/local_file_system.h" #include "load/delta_writer/delta_writer.h" +#include "load/memtable/memtable_memory_limiter.h" #include "runtime/descriptor_helper.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" diff --git a/be/test/storage/segment/segment_cache_test.cpp b/be/test/storage/segment/segment_cache_test.cpp index 4d00d403e41a55..9b116c1fe6b80f 100644 --- a/be/test/storage/segment/segment_cache_test.cpp +++ b/be/test/storage/segment/segment_cache_test.cpp @@ -42,6 +42,7 @@ #include "gtest/gtest_pred_impl.h" #include "io/fs/local_file_system.h" #include "load/delta_writer/delta_writer.h" +#include "load/memtable/memtable_memory_limiter.h" #include "runtime/descriptor_helper.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" diff --git a/be/test/storage/tablet/tablet_cooldown_test.cpp b/be/test/storage/tablet/tablet_cooldown_test.cpp index e3c53fe7de211a..6b39da97938374 100644 --- a/be/test/storage/tablet/tablet_cooldown_test.cpp +++ b/be/test/storage/tablet/tablet_cooldown_test.cpp @@ -50,6 +50,7 @@ #include "io/fs/path.h" #include "io/fs/remote_file_system.h" #include "load/delta_writer/delta_writer.h" +#include "load/memtable/memtable_memory_limiter.h" #include "runtime/descriptor_helper.h" #include "runtime/descriptors.h" #include "storage/olap_common.h" diff --git a/be/test/testutil/run_all_tests.cpp b/be/test/testutil/run_all_tests.cpp index c4413f5add0b7b..693accfdd7e2f0 100644 --- a/be/test/testutil/run_all_tests.cpp +++ b/be/test/testutil/run_all_tests.cpp @@ -40,6 +40,7 @@ #include "util/cpu_info.h" #include "util/disk_info.h" #include "util/mem_info.h" +#include "util/threadpool.h" int main(int argc, char** argv) { SCOPED_INIT_THREAD_CONTEXT(); diff --git a/build-support/check-header-deps.py b/build-support/check-header-deps.py index 315f63eaa3bb44..c91a3f25b4e0fd 100755 --- a/build-support/check-header-deps.py +++ b/build-support/check-header-deps.py @@ -39,6 +39,10 @@ import sys INCLUDE = re.compile(r'^\s*#\s*include\s+"([^"]+)"') +# Generated headers are conventionally angle-included; they never include project +# headers back, so capturing the edge (without resolving it) is enough to let a +# rule name a gen_cpp header as a forbidden target. +GEN_INCLUDE = re.compile(r"^\s*#\s*include\s+<(gen_cpp/[^>]+)>") SOURCE_ROOTS = ("be/src", "be/test") INCLUDE_ROOT = "be/src" @@ -61,6 +65,138 @@ "them; reaching the index implementation headers from here puts the whole " "index writer stack (and CLucene) in front of most of the backend", ), + ( + "runtime/exec_env.h", + "information_schema/", + set(), + "ExecEnv only names RoutineLoadTaskExecutor (forward-declared); the schema " + "scanner headers carry gen_cpp/FrontendService_types.h, which must not ride " + "into the ~1000 TUs that include ExecEnv transitively", + ), + ( + "runtime/exec_env.h", + "io/cache/", + set(), + "ExecEnv holds the file-cache machinery as pointers and forward-declares " + "io::FDCache and io::FileCacheFactory; fs_file_cache_storage.h used to drag " + "gen_cpp/internal_service.pb.h, descriptors.pb.h and the io/fs family into " + "the ~1060 TUs that include ExecEnv", + ), + ( + "runtime/exec_env.h", + "load/memtable/", + set(), + "ExecEnv only names MemTableMemoryLimiter through a unique_ptr member and " + "accessors (forward-declared, setter defined out of line); the memtable " + "stack must not ride the ExecEnv superhighway", + ), + ( + "runtime/exec_env.h", + "runtime/frontend_info.h", + set(), + "FrontendInfo embeds TFrontendInfo by value, so frontend_info.h carries " + "gen_cpp/HeartbeatService_types.h and AgentService_types.h; ExecEnv keeps " + "the frontends map behind a unique_ptr and forward-declares the types", + ), + ( + "runtime/exec_env.h", + "runtime/cluster_info.h", + set(), + "ExecEnv only holds ClusterInfo* (forward-declared); cluster_info.h carries " + "gen_cpp/Types_types.h, which must not enter every TU through this header", + ), + ( + "runtime/exec_env.h", + "util/threadpool.h", + set(), + "ExecEnv holds every pool as unique_ptr with .get() accessors " + "and forward-declares the type (assigning setters defined out of line); " + "threadpool.h carries thread.h, metrics.h and the blocking-queue family, " + "which must not ride into the ~1060 TUs that include ExecEnv", + ), + ( + "runtime/exec_env.h", + "storage/options.h", + set(), + "ExecEnv stores StorePath/CachePath only inside std::vector members and " + "reference-returning accessors, which work with forward declarations; " + "options.h carries gen_cpp/Types_types.h and io/cache/file_cache_common.h " + "into every TU that includes ExecEnv", + ), + ( + "runtime/exec_env.h", + "gen_cpp/", + { + # Status embeds TStatus/PStatus; these two ride in through + # common/status.h and are the only generated headers ExecEnv may keep. + "gen_cpp/Status_types.h", + "gen_cpp/types.pb.h", + }, + "ExecEnv reaches ~1060 TUs, so any generated protobuf/thrift header it " + "pulls in is reparsed by most of the backend; every thrift struct it " + "stores is behind a pointer or forward declaration", + ), + ( + "runtime/thread_context.h", + "runtime/workload_group/", + set(), + "ThreadContext is included by nearly every TU and only holds WorkloadGroup " + "through weak_ptr; workload_group.h carries gen_cpp/BackendService_types.h " + "(the whole thrift type universe), so it must stay out of this superhighway", + ), + ( + "runtime/runtime_state.h", + "runtime/workload_group/", + set(), + "RuntimeState only returns WorkloadGroupPtr by declaration; keeping " + "workload_group.h (and its thrift payload) out of it keeps the exec layer " + "from re-spreading BackendService_types.h", + ), + ( + "runtime/runtime_state.h", + "io/fs/s3_file_system.h", + set(), + "RuntimeState holds the error-log S3 filesystem only behind a shared_ptr " + "(forward-declared, dereferenced in runtime_state.cpp); s3_file_system.h " + "carries util/s3_util.h, the AWS SDK surface and gen_cpp/cloud.pb.h, " + "which must not ride into the ~1060 TUs that include RuntimeState", + ), + ( + "runtime/thread_context.h", + "runtime/workload_management/", + set(), + "ThreadContext stores ResourceContext behind a shared_ptr (forward-declared; " + "attach_task and the orphan fallback are defined out of line) and the " + "SCOPED/LIMIT macros only expand at call sites; resource_context.h used to " + "carry the whole workload_management family plus task_controller's " + "PaloInternalService_types.h into nearly every TU", + ), + ( + "runtime/thread_context.h", + "gen_cpp/", + { + # Status embeds TStatus/PStatus (ride in through common/status.h); + # TUniqueId lives in Types_types.h; the profile family rides in + # through runtime_profile.h held by the memory-tracker chain. + "gen_cpp/Status_types.h", + "gen_cpp/types.pb.h", + "gen_cpp/Types_types.h", + "gen_cpp/Metrics_types.h", + "gen_cpp/RuntimeProfile_types.h", + "gen_cpp/runtime_profile.pb.h", + }, + "ThreadContext reaches ~1000 TUs; any generated protobuf/thrift header " + "beyond the status/types/profile carriers listed here is reparsed by " + "most of the backend", + ), + ( + "runtime/workload_management/resource_context.h", + "gen_cpp/data.pb.h", + set(), + "resource_context.h references no data.pb symbol (TQueryStatistics is " + "thrift and forward-declared); this was a dead include spreading PBlock " + "and segment_v2.pb.h to ~595 TUs through thread_context.h", + ), ] # Forward-declaration headers are the sanctioned way through a barrier: they carry @@ -80,7 +216,10 @@ def load_includes(): with open(path, encoding="utf-8", errors="ignore") as handle: includes[path] = [ match.group(1) - for match in (INCLUDE.match(line) for line in handle) + for match in ( + INCLUDE.match(line) or GEN_INCLUDE.match(line) + for line in handle + ) if match ] return includes diff --git a/build-support/compile-bench/README.md b/build-support/compile-bench/README.md new file mode 100644 index 00000000000000..eec937f55942e0 --- /dev/null +++ b/build-support/compile-bench/README.md @@ -0,0 +1,122 @@ + + +# BE compile benchmark (`build.sh --compile-bench`) + +A timing framework for analyzing and optimizing the **BE C++ build speed**. +It answers two questions: + +1. **Which build phase** is slow (gensrc / submodules / cmake configure / compile)? +2. **Which translation unit / header / template** is slow, and why? + +## Usage + +```bash +# One cold, cache-free benchmark build of the BE, then print + save a report +./build.sh --compile-bench + +# Additionally collect clang -ftime-trace data (per-header / per-template costs) +COMPILE_BENCH_TRACE=ON ./build.sh --compile-bench + +# The usual knobs still apply +BUILD_TYPE=Release ./build.sh --compile-bench -j 16 +``` + +Results are stored per run in `be/compile-bench-results//`: + +| file | content | +|--------------------|----------------------------------------------------------------| +| `report.txt` | the human-readable report (also printed at the end of the run) | +| `summary.json` | machine-readable summary, used by `report.py compare` | +| `meta.tsv` | commit, toolchain, `-j`, PCH on/off, ... of the run | +| `phases.tsv` | raw phase timestamps | +| `compile_log.jsonl`| one record per compiler/linker invocation (wall/user/sys/rss) | +| `ninja_log.txt` | copy of the build dir's `.ninja_log` | + +Compare two runs (e.g. before/after an optimization attempt): + +```bash +python3 build-support/compile-bench/report.py compare \ + be/compile-bench-results/ be/compile-bench-results/ +``` + +Re-generate a report from raw data (e.g. after an interrupted run, using the +still-existing build dir): + +```bash +python3 build-support/compile-bench/report.py be/compile-bench-results/ \ + --build-dir be/build_Release_compile_bench --build-status interrupted +``` + +## How caches are kept out of the measurement + +Benchmark numbers are only comparable when every run does the same cold work: + +- **ccache is fully disabled**: the timing wrapper replaces ccache as + `CMAKE__COMPILER_LAUNCHER`, and `CCACHE_DISABLE=1` is exported as a + belt-and-braces measure for anything else that might call ccache. +- **A dedicated build dir** `be/build__compile_bench` is deleted and + recreated on every run: no incremental objects, no CMake cache, no stale + ninja state. Your normal `be/build_` dir is left untouched, so day-to-day + incremental builds are not harmed by benchmarking. +- **BE-only scope**: FE, cloud, java extensions, cdc client, UI and output + packaging are forcibly skipped; the run ends right after the C++ build and + the report, before `install`. +- PCH (`ENABLE_PCH`) stays at its normal default on purpose — the PCH is + generated inside the fresh build dir each run, so it is cold work, and it is + part of the real build being optimized. Override with `ENABLE_PCH=OFF` to + compare with/without PCH. + +Not isolated (by design, both are outside the BE compile and near-constant): +the `contrib/datasketches-cpp` mini-build reuses its own build dir, and +`--clean` is not required (passing it additionally rebuilds gensrc from +scratch; the timed BE phases are unaffected). + +## What the report contains + +- **Phases**: gensrc, contrib submodules, datasketches install, cmake + configure, build — with durations and share of total wall time. +- **Build summary**: TU count, sum of per-TU wall/cpu time, effective + parallelism, slowest TU, most memory-hungry TU, link times with peak RSS. +- **Top N slowest translation units** with wall/user/sys/maxrss per file. +- **Wall time by directory** (top-level and second-level under `be/src`) — + tells you which module to attack first. +- **Last finishers** from `.ninja_log` — the critical-path tail the whole + build waits on (typically the monster TUs and the final link). +- With `COMPILE_BENCH_TRACE=ON` (clang only): **top headers by inclusive parse + time**, **top template instantiations**, and a **frontend vs backend split** + for the slowest TUs — this is what tells you *why* a file is slow + (header/include cost vs template instantiation vs codegen). + +## Implementation notes + +- `cc-timing-wrapper.py` sits in the compiler-launcher slot, `fork/exec`s the + real compiler with inherited stdio, and records wall time plus + `rusage` (user/sys/maxrss) per invocation into `compile_log.jsonl`. + It is a strict pass-through: exit codes and diagnostics are unchanged, and + without `DORIS_COMPILE_BENCH_DIR` set it `exec`s straight through. +- Linker launchers (`CMAKE__LINKER_LAUNCHER`) need CMake >= 3.21; with an + older CMake, link times still show up via the ninja log, just without RSS. +- `-ftime-trace` JSON files land next to the object files in the (transient) + bench build dir and are aggregated immediately at the end of the run; expect + roughly 1-2 GB of temporary JSON for a full BE build. Header/template times + are *inclusive* (nested includes count into their parents), so they rank + hotspots but do not add up to wall time. +- Timings from a `make` (non-ninja) build lose only the "last finishers" + section; per-TU data comes from the wrapper and is generator-independent. diff --git a/build-support/compile-bench/bench-lib.sh b/build-support/compile-bench/bench-lib.sh new file mode 100644 index 00000000000000..eca2e03b1e42c8 --- /dev/null +++ b/build-support/compile-bench/bench-lib.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +############################################################## +# Helper library for `build.sh --compile-bench`. +# This file is sourced by build.sh, not executed. +# +# One benchmark run collects everything under +# be/compile-bench-results// +# meta.tsv keyvalue facts about the run +# phases.tsv phasestart_msend_ms +# compile_log.jsonl one line per compiler/linker call +# (written by cc-timing-wrapper.py) +# ninja_log.txt copy of the build dir's .ninja_log +# report.txt human readable report (report.py) +# summary.json machine readable summary (report.py) +############################################################## + +COMPILE_BENCH_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +_compile_bench_now_ms() { + python3 -c 'import time; print(int(time.time() * 1000))' +} + +# compile_bench_init +# Prepares the run directory, disables every compile cache and installs the +# timing wrapper as the compiler launcher. Mutates: +# CMAKE_USE_CCACHE_CXX / CMAKE_USE_CCACHE_C (launcher slot: ccache -> wrapper) +# COMPILE_BENCH_CMAKE_ARGS (extra -D args for the BE cmake call) +# COMPILE_BENCH_BUILD_DIR (dedicated always-cold build dir) +# EXTRA_CXX_FLAGS (+ -ftime-trace when requested) +compile_bench_init() { + local doris_home="$1" + + if ! command -v python3 &>/dev/null; then + echo "ERROR: --compile-bench requires python3 (timing wrapper and report generator)." + exit 1 + fi + + COMPILE_BENCH_RUN_ID="$(date -u '+%Y%m%d_%H%M%S')" + COMPILE_BENCH_RUN_DIR="${doris_home}/be/compile-bench-results/${COMPILE_BENCH_RUN_ID}" + mkdir -p "${COMPILE_BENCH_RUN_DIR}" + + local cmake_build_type="${BUILD_TYPE:-Release}" + COMPILE_BENCH_BUILD_DIR="${doris_home}/be/build_${cmake_build_type}_compile_bench" + + # Consumed by cc-timing-wrapper.py in every compiler process. + export DORIS_COMPILE_BENCH_DIR="${COMPILE_BENCH_RUN_DIR}" + + # Kill every compile cache: + # 1. the launcher slot below no longer contains ccache, and + # 2. CCACHE_DISABLE turns any leftover ccache call into a pass-through. + export CCACHE_DISABLE=1 + + COMPILE_BENCH_WRAPPER="${COMPILE_BENCH_LIB_DIR}/cc-timing-wrapper.py" + CMAKE_USE_CCACHE_CXX="-DCMAKE_CXX_COMPILER_LAUNCHER=${COMPILE_BENCH_WRAPPER}" + CMAKE_USE_CCACHE_C="-DCMAKE_C_COMPILER_LAUNCHER=${COMPILE_BENCH_WRAPPER}" + # Linker launcher needs CMake >= 3.21; older CMake silently ignores the + # variables and link times are then only visible through the ninja log. + COMPILE_BENCH_CMAKE_ARGS=( + "-DCMAKE_CXX_LINKER_LAUNCHER=${COMPILE_BENCH_WRAPPER}" + "-DCMAKE_C_LINKER_LAUNCHER=${COMPILE_BENCH_WRAPPER}" + ) + + COMPILE_BENCH_TIME_TRACE='OFF' + if [[ "${COMPILE_BENCH_TRACE:-OFF}" == 'ON' ]]; then + if [[ "${DORIS_TOOLCHAIN}" == "gcc" ]]; then + echo "WARNING: COMPILE_BENCH_TRACE=ON needs clang (-ftime-trace); ignored with gcc." + else + COMPILE_BENCH_TIME_TRACE='ON' + EXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS:+${EXTRA_CXX_FLAGS} }-ftime-trace" + fi + fi + + local git_commit git_branch cxx_version ncpu + git_commit="$(git -C "${doris_home}" rev-parse --short HEAD 2>/dev/null || echo unknown)" + git_branch="$(git -C "${doris_home}" rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)" + cxx_version="$("${CXX}" --version 2>/dev/null | head -n1 || echo unknown)" + ncpu="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo unknown)" + + { + printf 'run_id\t%s\n' "${COMPILE_BENCH_RUN_ID}" + printf 'doris_home\t%s\n' "${doris_home}" + printf 'build_dir\t%s\n' "${COMPILE_BENCH_BUILD_DIR}" + printf 'date_utc\t%s\n' "$(date -u '+%Y-%m-%d %H:%M:%S')" + printf 'uname\t%s\n' "$(uname -sm)" + printf 'ncpu\t%s\n' "${ncpu}" + printf 'parallel\t%s\n' "${PARALLEL}" + printf 'build_type\t%s\n' "${cmake_build_type}" + printf 'generator\t%s\n' "${GENERATOR}" + printf 'build_system\t%s\n' "${BUILD_SYSTEM}" + printf 'toolchain\t%s\n' "${DORIS_TOOLCHAIN}" + printf 'cxx\t%s\n' "${CXX}" + printf 'cxx_version\t%s\n' "${cxx_version}" + printf 'enable_pch\t%s\n' "${ENABLE_PCH}" + printf 'use_avx2\t%s\n' "${USE_AVX2}" + printf 'time_trace\t%s\n' "${COMPILE_BENCH_TIME_TRACE}" + printf 'git_commit\t%s\n' "${git_commit}" + printf 'git_branch\t%s\n' "${git_branch}" + } >"${COMPILE_BENCH_RUN_DIR}/meta.tsv" + + COMPILE_BENCH_T0="$(_compile_bench_now_ms)" + + echo "***************************************" + echo "* BE compile benchmark mode (--compile-bench)" + echo "* run dir : ${COMPILE_BENCH_RUN_DIR}" + echo "* build dir : ${COMPILE_BENCH_BUILD_DIR} (recreated from scratch)" + echo "* ccache : DISABLED (timing wrapper installed as launcher)" + echo "* -ftime-trace: ${COMPILE_BENCH_TIME_TRACE}" + echo "* BE-only build: FE/cloud/java-extensions/packaging are skipped" + echo "***************************************" +} + +# compile_bench_phase_begin +compile_bench_phase_begin() { + COMPILE_BENCH_PHASE_NAME="$1" + COMPILE_BENCH_PHASE_START="$(_compile_bench_now_ms)" + echo "Compile-bench: phase '${COMPILE_BENCH_PHASE_NAME}' started" +} + +compile_bench_phase_end() { + local end_ms + end_ms="$(_compile_bench_now_ms)" + printf '%s\t%s\t%s\n' \ + "${COMPILE_BENCH_PHASE_NAME}" "${COMPILE_BENCH_PHASE_START}" "${end_ms}" \ + >>"${COMPILE_BENCH_RUN_DIR}/phases.tsv" + echo "Compile-bench: phase '${COMPILE_BENCH_PHASE_NAME}' finished in $(((end_ms - COMPILE_BENCH_PHASE_START) / 1000))s" +} + +# compile_bench_finish +# Preserves the ninja log and generates the report. Never fails the build +# beyond the exit code that is already being propagated by the caller. +compile_bench_finish() { + local build_dir="$1" + local build_rc="$2" + local end_ms + end_ms="$(_compile_bench_now_ms)" + printf 'total\t%s\t%s\n' "${COMPILE_BENCH_T0}" "${end_ms}" \ + >>"${COMPILE_BENCH_RUN_DIR}/phases.tsv" + + if [[ -f "${build_dir}/.ninja_log" ]]; then + cp -f "${build_dir}/.ninja_log" "${COMPILE_BENCH_RUN_DIR}/ninja_log.txt" + fi + + local build_status='ok' + if [[ "${build_rc}" -ne 0 ]]; then + build_status="failed(rc=${build_rc})" + fi + + set +e + python3 "${COMPILE_BENCH_LIB_DIR}/report.py" "${COMPILE_BENCH_RUN_DIR}" \ + --build-dir "${build_dir}" \ + --build-status "${build_status}" + local report_rc=$? + set -e + if [[ "${report_rc}" -ne 0 ]]; then + echo "WARNING: compile-bench report generation failed (rc=${report_rc});" + echo " raw timing data is still available in ${COMPILE_BENCH_RUN_DIR}" + fi + + echo "***************************************" + echo "* BE compile benchmark finished: ${build_status}" + echo "* report : ${COMPILE_BENCH_RUN_DIR}/report.txt" + echo "* summary: ${COMPILE_BENCH_RUN_DIR}/summary.json" + echo "* Compare two runs with:" + echo "* python3 build-support/compile-bench/report.py compare " + echo "***************************************" +} diff --git a/build-support/compile-bench/cc-timing-wrapper.py b/build-support/compile-bench/cc-timing-wrapper.py new file mode 100755 index 00000000000000..64b50da0ff1d63 --- /dev/null +++ b/build-support/compile-bench/cc-timing-wrapper.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Compiler/linker timing wrapper for `build.sh --compile-bench`. + +In compile-bench mode this script is installed as +CMAKE__COMPILER_LAUNCHER / CMAKE__LINKER_LAUNCHER, i.e. in the +slot where ccache would normally sit (compile-bench never uses ccache, so the +slot is free). Every underlying compiler/linker invocation is timed and one +JSON line is appended to $DORIS_COMPILE_BENCH_DIR/compile_log.jsonl: + + {"ts": ..., "kind": "compile|pch|link|other", "src": ..., "out": ..., + "cwd": ..., "compiler": ..., "wall_s": ..., "user_s": ..., "sys_s": ..., + "maxrss_mb": ..., "rc": ...} + +Non-interference guarantees: +- child stdout/stderr are fully inherited, diagnostics are unchanged +- the child's exit code is propagated verbatim (signals become 128+N) +- if DORIS_COMPILE_BENCH_DIR is not set, exec() straight through: zero overhead +- a bookkeeping failure never fails the build +""" + +import fcntl +import json +import os +import sys +import time + +SRC_EXTS = (".cpp", ".cc", ".cxx", ".c", ".mm", ".m", ".S", ".s") +HDR_EXTS = (".h", ".hpp", ".hxx", ".hh") +# Flags whose presence means "not a real compile/link" (probes, dep scans...). +NON_BUILD_FLAGS = ("-E", "-M", "-MM", "--version", "-dumpversion", "-dumpmachine") + + +def classify(args): + """Return (kind, src, out) extracted from a compiler command line. + + Only the `-o path` split form is handled: that is the only form CMake and + ninja generate. Flags that merely start with "-o" (-objc..., -only...) + must not be misparsed, so the fused "-opath" form is deliberately ignored. + """ + out = None + src = None + has_c = False + x_header = False + prev = None + for arg in args[1:]: + if prev == "-o": + out = arg + elif prev == "-x" and arg in ("c++-header", "c-header"): + x_header = True + elif not arg.startswith("-"): + if arg.endswith(SRC_EXTS): + src = arg + elif x_header and src is None and arg.endswith(HDR_EXTS): + src = arg + if arg == "-c": + has_c = True + prev = arg + + if x_header or (out is not None and out.endswith((".pch", ".gch"))): + kind = "pch" + elif any(a in NON_BUILD_FLAGS for a in args): + kind = "other" + elif has_c: + kind = "compile" + elif out is not None: + kind = "link" + else: + kind = "other" + return kind, src, out + + +def main(): + argv = sys.argv[1:] + if not argv: + sys.stderr.write("cc-timing-wrapper.py: missing compiler command\n") + return 2 + + log_dir = os.environ.get("DORIS_COMPILE_BENCH_DIR") + if not log_dir: + # Not in a bench run (or env was stripped): become the compiler. + os.execvp(argv[0], argv) + + start_ts = time.time() + t0 = time.monotonic() + pid = os.fork() + if pid == 0: + try: + os.execvp(argv[0], argv) + except OSError as exc: + sys.stderr.write( + "cc-timing-wrapper.py: failed to exec {}: {}\n".format(argv[0], exc) + ) + os._exit(127) + _, status, rusage = os.wait4(pid, 0) + wall_s = time.monotonic() - t0 + + if os.WIFSIGNALED(status): + rc = 128 + os.WTERMSIG(status) + else: + rc = os.WEXITSTATUS(status) + + kind, src, out = classify(argv) + # ru_maxrss is bytes on macOS, KiB on Linux. + if sys.platform == "darwin": + maxrss_mb = rusage.ru_maxrss / (1024.0 * 1024.0) + else: + maxrss_mb = rusage.ru_maxrss / 1024.0 + + record = { + "ts": round(start_ts, 3), + "kind": kind, + "src": src, + "out": out, + "cwd": os.getcwd(), + "compiler": os.path.basename(argv[0]), + "wall_s": round(wall_s, 3), + "user_s": round(rusage.ru_utime, 3), + "sys_s": round(rusage.ru_stime, 3), + "maxrss_mb": round(maxrss_mb, 1), + "rc": rc, + } + try: + with open(os.path.join(log_dir, "compile_log.jsonl"), "a") as fh: + fcntl.flock(fh.fileno(), fcntl.LOCK_EX) + fh.write(json.dumps(record, sort_keys=True) + "\n") + except OSError: + pass + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build-support/compile-bench/cut_impact.py b/build-support/compile-bench/cut_impact.py new file mode 100644 index 00000000000000..5507c01076acef --- /dev/null +++ b/build-support/compile-bench/cut_impact.py @@ -0,0 +1,489 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Estimate the blast radius of cutting one #include edge from a hub header. + +Before removing `#include "T"` from hub header H, you want to know: + 1. which TUs currently reach T only through H (they will stop seeing T); + 2. which of those TUs (or headers they keep) actually reference symbols + declared in T's include subtree, and therefore need a direct include + added ("seeding") before the edge can be cut safely. + +Data sources: + * `ninja -t deps` from a completed compile-bench build directory: the real, + per-TU flat header closure (what each TU actually included in this build + configuration). + * The parsed include graph of be/src + gensrc/build: edge structure, built + by scanning `#include` directives and resolving them the way the compiler + does (includer dir first for quoted includes, then -I roots). + +Per candidate TU the script runs two BFS traversals over the parsed graph, +both restricted to the TU's real closure: one with the edge, one without. +The difference is the set of headers this TU loses. Symbols defined in lost +headers are then matched (word-level, comments stripped) against the files +the TU keeps; every hit becomes a seeding suggestion "file F must directly +include header L". The result is an estimate — conditional includes the +parser cannot see and symbol matches inside string literals can produce +noise — so spot-check a sample (e.g. -fsyntax-only) before mass edits. + +Usage: + # single edge: what happens if H stops including T + python3 cut_impact.py edge runtime/exec_env.h \ + information_schema/schema_routine_load_job_scanner.h + + # rank every direct project include of a hub by cut impact + python3 cut_impact.py audit runtime/exec_env.h + + # machine-readable detail / manual-verification sample + python3 cut_impact.py edge H T --json out.json --sample 5 +""" + +import argparse +import collections +import json +import os +import random +import re +import subprocess +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(os.path.dirname(SCRIPT_DIR)) + +SOURCE_EXTS = (".cpp", ".cc", ".c", ".cxx") +HEADER_EXTS = (".h", ".hpp", ".hh", ".inc", ".ipp") + +INCLUDE_RE = re.compile(r'^\s*#\s*include\s+(["<])([^">]+)[">]', re.M) +COMMENT_RE = re.compile(r"//[^\n]*|/\*.*?\*/", re.S) +TOKEN_RE = re.compile(r"[A-Za-z_]\w{2,}") +# Type *definitions* only (a trailing '{' is required), not forward decls. +TYPE_DEF_RE = re.compile( + r"\b(?:class|struct|enum(?:\s+(?:class|struct))?)\s+" + r"(?:\[\[[^\]]*\]\]\s*|[A-Z_]{3,}\s+)?" # attributes / export macros + r"([A-Za-z_]\w*)\s*(?:final\s*)?(?::[^;{}]*)?\{" +) +USING_RE = re.compile(r"\busing\s+([A-Za-z_]\w*)\s*=") +TYPEDEF_RE = re.compile(r"\btypedef\b[^;]*?\b([A-Za-z_]\w*)\s*;") +DEFINE_RE = re.compile(r"^\s*#\s*define\s+([A-Za-z_]\w*)", re.M) + + +def read_text(path): + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + return f.read() + except OSError: + return "" + + +class IncludeGraph: + """Parsed include-edge structure over project files (be/src, gensrc/build).""" + + def __init__(self, roots): + self.roots = roots # ordered list of abs dirs, first match wins for display + self.files = {} # abs path -> True + self.includes = {} # abs path -> list of abs paths (project files only) + self._stripped = {} # abs path -> comment-stripped text (lazy) + self._tokens = {} # abs path -> set of identifier tokens (lazy) + self._symbols = {} # abs path -> set of defined top-level names (lazy) + self._scan() + self._parse_edges() + + def _scan(self): + for root in self.roots: + for dirpath, _dirnames, filenames in os.walk(root): + for name in filenames: + if name.endswith(SOURCE_EXTS) or name.endswith(HEADER_EXTS): + p = sys.intern(os.path.normpath(os.path.join(dirpath, name))) + self.files[p] = True + + def _parse_edges(self): + for path in self.files: + edges = [] + for m in INCLUDE_RE.finditer(read_text(path)): + quoted = m.group(1) == '"' + target = self.resolve(m.group(2), os.path.dirname(path), quoted) + if target is not None and target != path: + edges.append(target) + self.includes[path] = edges + + def resolve(self, inc, includer_dir, quoted=True): + if quoted: + cand = sys.intern(os.path.normpath(os.path.join(includer_dir, inc))) + if cand in self.files: + return cand + for root in self.roots: + cand = sys.intern(os.path.normpath(os.path.join(root, inc))) + if cand in self.files: + return cand + return None + + def display(self, path): + for root in self.roots: + if path.startswith(root + os.sep): + return os.path.relpath(path, root) + return path + + def subtree(self, start): + """All files reachable from `start` in the unrestricted parsed graph.""" + seen = {start} + queue = collections.deque([start]) + while queue: + for nxt in self.includes.get(queue.popleft(), ()): + if nxt not in seen: + seen.add(nxt) + queue.append(nxt) + return seen + + def stripped_text(self, path): + if path not in self._stripped: + self._stripped[path] = COMMENT_RE.sub(" ", read_text(path)) + return self._stripped[path] + + def tokens(self, path): + if path not in self._tokens: + self._tokens[path] = set(TOKEN_RE.findall(self.stripped_text(path))) + return self._tokens[path] + + def symbols(self, path): + """Top-level names *defined* by this header (types, aliases, macros).""" + if path not in self._symbols: + text = self.stripped_text(path) + names = set() + for regex in (TYPE_DEF_RE, USING_RE, TYPEDEF_RE, DEFINE_RE): + names.update(regex.findall(text)) + self._symbols[path] = {n for n in names if len(n) >= 3} + return self._symbols[path] + + +def load_tus(build_dir, graph): + """source abs path -> set of project files in its real (ninja) closure.""" + proc = subprocess.Popen( + ["ninja", "-C", build_dir, "-t", "deps"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + tus = {} + src_prefix = os.path.join(REPO_ROOT, "be", "src") + os.sep + gen_prefix = os.path.join(REPO_ROOT, "gensrc", "build") + os.sep + cur = None # dep set of the current block's TU; None while skipping a block + expect_source = False # the next indented line is the block's source file + for line in proc.stdout: + if line.startswith(" "): + p = line.strip() + if not os.path.isabs(p): + p = os.path.join(build_dir, p) + p = os.path.normpath(p) + if expect_source: # first dep line = the source file itself + expect_source = False + if p.startswith((src_prefix, gen_prefix)): + key = sys.intern(p) + cur = tus.setdefault(key, set()) + cur.add(key) + else: + cur = None # foreign TU (contrib etc.): skip whole block + elif cur is not None and p in graph.files: + cur.add(sys.intern(p)) + else: # block header ": #deps N, ... (VALID|STALE)" or noise + expect_source = line.rstrip().endswith("(VALID)") + cur = None + proc.wait() + return tus + + +def analyze_edge(graph, tus, edges, collect_seeds=True, freq_cap=250): + """Simulate removing include edges [(hub, cut), ...]; return per-TU losses + and seed advice. Symbols referenced by more than `freq_cap` files are + treated as noise (e.g. thrift's nested `enum type`) and ignored.""" + cut_set = set(edges) + targets = {c for _h, c in edges} + res = { + "edges": list(edges), + "candidates": 0, # TUs whose real closure contains any cut target + "affected": {}, # source -> sorted list of lost files + "unaffected": 0, # still reach everything via other paths + "parser_gap": [], # closure has a target but parsed BFS never saw it + "seeds": collections.defaultdict(lambda: collections.defaultdict(set)), + "seed_tu_count": collections.Counter(), # (file, lost) -> #TUs + "lost_spread": collections.Counter(), # lost file -> #TUs losing it + "tu_needs": {}, # source -> [(seed file, lost header), ...] + } + + def bfs(source, allowed, skip_edges): + seen = {source} + queue = collections.deque([source]) + while queue: + node = queue.popleft() + for nxt in graph.includes.get(node, ()): + if skip_edges and (node, nxt) in cut_set: + continue + if nxt in allowed and nxt not in seen: + seen.add(nxt) + queue.append(nxt) + return seen + + for source, closure in sorted(tus.items()): + if not targets & closure: + continue + res["candidates"] += 1 + before = bfs(source, closure, skip_edges=False) + if not targets & before: + res["parser_gap"].append(source) + continue + after = bfs(source, closure, skip_edges=True) + lost = before - after + if not lost: + res["unaffected"] += 1 + continue + res["affected"][source] = sorted(lost) + for f in lost: + res["lost_spread"][f] += 1 + + if collect_seeds and res["affected"]: + all_lost = set() + for lost in res["affected"].values(): + all_lost.update(lost) + # symbol -> headers (in the lost pool) defining it + sym_owners = collections.defaultdict(set) + for lf in all_lost: + for s in graph.symbols(lf): + sym_owners[s].add(lf) + symset = set(sym_owners) + # drop noise: symbols that occur in more files than freq_cap + sym_freq = collections.Counter() + file_hits = {} + for path in graph.files: + hit = graph.tokens(path) & symset + if hit: + file_hits[path] = hit + sym_freq.update(hit) + noisy = {s for s, n in sym_freq.items() if n > freq_cap} + # file -> {lost header -> matched symbols} + refs = collections.defaultdict(lambda: collections.defaultdict(set)) + header_refs = collections.defaultdict(set) # lost header -> {files} + for path, hit in file_hits.items(): + own = graph.symbols(path) if path.endswith(HEADER_EXTS) else () + for s in hit - noisy: + if s in own: # file defines this name itself (e.g. its own State) + continue + for owner in sym_owners[s]: + if path != owner: + refs[path][owner].add(s) + header_refs[owner].add(path) + for source, lost in res["affected"].items(): + kept = tus[source] - set(lost) + needs = [] + for lf in lost: + for f in header_refs.get(lf, ()): + if f in kept: + res["seeds"][f][lf].update(refs[f][lf]) + res["seed_tu_count"][(f, lf)] += 1 + needs.append((f, lf)) + if needs: + res["tu_needs"][source] = needs + return res + + +def print_edge_report(graph, res, sample=0, top=15): + d = graph.display + print("== Cut impact ==") + for hub, cut in res["edges"]: + print(f" {d(hub)} -/-> {d(cut)}") + n_aff = len(res["affected"]) + print(f"TUs whose real closure contains a cut header : {res['candidates']}") + print(f" affected (lose headers once the edge is cut) : {n_aff}") + print(f" unaffected (other include paths still exist) : {res['unaffected']}") + if res["parser_gap"]: + print(f" unanalyzable (parser gap, verify manually) : {len(res['parser_gap'])}") + for s in res["parser_gap"][:5]: + print(f" {d(s)}") + if not n_aff: + return + + for hub, _cut in res["edges"]: + hub_seed = res["seeds"].get(hub) + if hub_seed: + syms = sorted({s for ss in hub_seed.values() for s in ss}) + print(f"Hub self-check: {d(hub)} itself references: {', '.join(syms[:8])}") + print(" -> this hub still needs (some of) the cut subtree; NOT a dead include.") + else: + print(f"Hub self-check: {d(hub)} references nothing from the lost subtree " + "-> dead include for this hub itself.") + + print(f"\nLost-header spread (top {top} by #TUs losing it):") + for f, n in res["lost_spread"].most_common(top): + print(f" {n:5d} TUs lose {d(f)}") + + if res["seeds"]: + print("\nSeed list — add these direct includes BEFORE cutting the edge:") + rows = [] + for f, per_lost in res["seeds"].items(): + for lf, syms in per_lost.items(): + rows.append((res["seed_tu_count"][(f, lf)], f, lf, sorted(syms))) + rows.sort(key=lambda r: (-r[0], r[1])) + for n_tu, f, lf, syms in rows: + shown = ", ".join(syms[:6]) + (" …" if len(syms) > 6 else "") + print(f" {d(f)}\n + #include \"{d(lf)}\" [{n_tu} TU(s); uses: {shown}]") + seeded_files = len(res["seeds"]) + print(f" ({seeded_files} file(s) need seeding, {len(rows)} include line(s) total)") + else: + print("\nSeed list: EMPTY — no kept file references any lost symbol; " + "cut is predicted safe without preparatory edits.") + + clean = n_aff - len(res["tu_needs"]) + print(f"\nPer-TU verdict: {clean}/{n_aff} affected TUs reference nothing they lose " + f"(predicted to compile unchanged); {len(res['tu_needs'])} TU(s) rely on the " + "seeded file(s) above (per-TU detail in --json).") + + if sample: + print(f"\n-- Random sample of {min(sample, n_aff)} affected TU(s) for manual verification --") + for source in random.sample(sorted(res["affected"]), min(sample, n_aff)): + lost = res["affected"][source] + print(f" TU {d(source)} loses {len(lost)} header(s):") + for f in lost[:8]: + print(f" {d(f)}") + if len(lost) > 8: + print(f" … and {len(lost) - 8} more") + + +def edge_json(graph, res): + d = graph.display + return { + "edges": [[d(h), d(c)] for h, c in res["edges"]], + "candidates": res["candidates"], + "unaffected": res["unaffected"], + "parser_gap": [d(s) for s in res["parser_gap"]], + "affected": {d(s): [d(f) for f in lost] for s, lost in res["affected"].items()}, + "tu_needs": {d(s): [[d(f), d(lf)] for f, lf in needs] + for s, needs in res["tu_needs"].items()}, + "lost_spread": {d(f): n for f, n in res["lost_spread"].most_common()}, + "seeds": { + d(f): { + d(lf): {"symbols": sorted(syms), + "tu_count": res["seed_tu_count"][(f, lf)]} + for lf, syms in per_lost.items() + } + for f, per_lost in res["seeds"].items() + }, + } + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build-dir", + default=os.path.join(REPO_ROOT, "be", "build_Release_compile_bench"), + help="ninja build dir with a completed build (default: bench dir)") + sub = ap.add_subparsers(dest="mode", required=True) + ap_edge = sub.add_parser("edge", help="impact of cutting one include edge") + ap_edge.add_argument("hub", help="header that contains the include, e.g. runtime/exec_env.h") + ap_edge.add_argument("cut", help="the included header to remove, e.g. io/cache/fs_file_cache_storage.h") + ap_edge.add_argument("--and", dest="extra", nargs=2, action="append", default=[], + metavar=("HUB", "CUT"), + help="cut this edge too (repeatable): combined-scenario impact") + ap_edge.add_argument("--json", help="write full per-TU detail to this file") + ap_edge.add_argument("--sample", type=int, default=0, + help="print N random affected TUs for manual spot-checking") + ap_audit = sub.add_parser("audit", help="rank all direct project includes of a hub") + ap_audit.add_argument("hub") + ap_why = sub.add_parser("why", help="explain how a TU reaches a header (real closure)") + ap_why.add_argument("tu", help="TU source, e.g. runtime/exec_env.cpp") + ap_why.add_argument("target", help="header to explain, e.g. gen_cpp/FrontendService_types.h") + args = ap.parse_args() + + roots = [os.path.join(REPO_ROOT, "be", "src"), + os.path.join(REPO_ROOT, "gensrc", "build")] + extra_common = os.path.join(REPO_ROOT, "common") + if os.path.isdir(extra_common): + roots.append(extra_common) + + print("Parsing include graph …", file=sys.stderr) + graph = IncludeGraph(roots) + print(f" {len(graph.files)} project files", file=sys.stderr) + print("Loading real per-TU closures (ninja -t deps) …", file=sys.stderr) + tus = load_tus(args.build_dir, graph) + print(f" {len(tus)} TUs with valid deps", file=sys.stderr) + + def must_resolve(name): + p = graph.resolve(name, os.getcwd(), quoted=True) + if p is None: + sys.exit(f"error: cannot resolve '{name}' under {', '.join(roots)}") + return p + + if args.mode == "edge": + edges = [] + for h, c in [(args.hub, args.cut)] + args.extra: + hub, cut = must_resolve(h), must_resolve(c) + if cut not in graph.includes.get(hub, ()): + sys.exit(f"error: {graph.display(hub)} has no direct include of " + f"{graph.display(cut)}") + edges.append((hub, cut)) + res = analyze_edge(graph, tus, edges) + print_edge_report(graph, res, sample=args.sample) + if args.json: + with open(args.json, "w") as f: + json.dump(edge_json(graph, res), f, indent=1) + print(f"\nfull detail written to {args.json}") + elif args.mode == "why": + tu, target = must_resolve(args.tu), must_resolve(args.target) + closure = tus.get(tu) + if closure is None: + sys.exit(f"error: {graph.display(tu)} is not a compiled TU in this build") + if target not in closure: + print(f"{graph.display(tu)} does NOT include {graph.display(target)} in the real build.") + return + parent = {tu: None} + queue = collections.deque([tu]) + while queue and target not in parent: + node = queue.popleft() + for nxt in graph.includes.get(node, ()): + if nxt in closure and nxt not in parent: + parent[nxt] = node + queue.append(nxt) + if target not in parent: + print("in the real closure, but the parsed graph cannot trace a path " + "(conditional include the parser missed?)") + return + chain, node = [], target + while node is not None: + chain.append(node) + node = parent[node] + print("shortest include chain:") + for i, node in enumerate(reversed(chain)): + print(f" {' ' * i}{graph.display(node)}") + parents_in_closure = sorted( + p for p in closure + if target in graph.includes.get(p, ())) + print(f"\nall direct includers of {graph.display(target)} inside this TU's closure " + f"({len(parents_in_closure)}):") + for p in parents_in_closure: + print(f" {graph.display(p)}") + else: # audit + hub = must_resolve(args.hub) + targets = graph.includes.get(hub, []) + print(f"== Audit: {graph.display(hub)} — {len(targets)} direct project include(s) ==") + print(f"{'cut candidate':58s} {'cand':>5s} {'affect':>6s} {'seedF':>5s} top lost header") + for cut in targets: + res = analyze_edge(graph, tus, [(hub, cut)]) + top = res["lost_spread"].most_common(1) + top_s = f"{graph.display(top[0][0])}({top[0][1]})" if top else "-" + print(f"{graph.display(cut):58s} {res['candidates']:5d} " + f"{len(res['affected']):6d} {len(res['seeds']):5d} {top_s}") + + +if __name__ == "__main__": + main() diff --git a/build-support/compile-bench/report.py b/build-support/compile-bench/report.py new file mode 100755 index 00000000000000..d70ed286f004c9 --- /dev/null +++ b/build-support/compile-bench/report.py @@ -0,0 +1,533 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Report generator for `build.sh --compile-bench` runs. + +Generate a report for one run (done automatically at the end of a bench build, +can be re-run manually at any time): + + python3 report.py [--build-dir DIR] [--top N] [--build-status S] + +Compare two runs (arguments are run dirs or summary.json paths): + + python3 report.py compare + +Inputs inside (see bench-lib.sh): + meta.tsv, phases.tsv, compile_log.jsonl, ninja_log.txt +Optional, from the build dir when COMPILE_BENCH_TRACE=ON was used: + per-TU clang -ftime-trace JSON files (next to the object files) + +Outputs inside : report.txt (human) and summary.json (machine). +""" + +import argparse +import json +import os +import sys +from collections import defaultdict + +TOP_DIRS = 25 +TOP_HEADERS = 30 +TOP_TEMPLATES = 20 +TOP_TAIL = 10 +WIDTH = 78 + + +def section(title): + text = "-- " + title + " " + return text + "-" * max(0, WIDTH - len(text)) + + +def fmt_dur(seconds): + seconds = float(seconds) + if seconds < 0: + return "?" + if seconds < 60: + return "{:.1f}s".format(seconds) + minutes = int(seconds // 60) + if minutes < 60: + return "{}m{:02d}s".format(minutes, int(seconds % 60)) + return "{}h{:02d}m".format(minutes // 60, minutes % 60) + + +def read_meta(run_dir): + meta = {} + path = os.path.join(run_dir, "meta.tsv") + if os.path.isfile(path): + with open(path) as fh: + for line in fh: + parts = line.rstrip("\n").split("\t", 1) + if len(parts) == 2: + meta[parts[0]] = parts[1] + return meta + + +def read_phases(run_dir): + phases = [] + path = os.path.join(run_dir, "phases.tsv") + if os.path.isfile(path): + with open(path) as fh: + for line in fh: + parts = line.rstrip("\n").split("\t") + if len(parts) != 3: + continue + try: + start_ms, end_ms = int(parts[1]), int(parts[2]) + except ValueError: + continue + phases.append( + {"name": parts[0], "dur_s": (end_ms - start_ms) / 1000.0} + ) + return phases + + +def is_cmake_probe(record): + for key in ("src", "out", "cwd"): + value = record.get(key) or "" + if "CMakeScratch" in value or "CMakeTmp" in value: + return True + return False + + +def read_compile_log(run_dir): + records = [] + path = os.path.join(run_dir, "compile_log.jsonl") + if os.path.isfile(path): + with open(path) as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except ValueError: + continue + if not is_cmake_probe(record): + records.append(record) + return records + + +def read_ninja_log(run_dir, build_dir): + """Return {output_path: (start_ms, end_ms)}, deduped keeping the last entry.""" + path = os.path.join(run_dir, "ninja_log.txt") + if not os.path.isfile(path) and build_dir: + path = os.path.join(build_dir, ".ninja_log") + edges = {} + if not os.path.isfile(path): + return edges + with open(path) as fh: + for line in fh: + if line.startswith("#"): + continue + parts = line.rstrip("\n").split("\t") + if len(parts) < 4: + continue + try: + start_ms, end_ms = int(parts[0]), int(parts[1]) + except ValueError: + continue + edges[parts[3]] = (start_ms, end_ms) + return edges + + +def rel_path(path, meta): + if not path: + return "?" + doris_home = meta.get("doris_home") + if doris_home: + home = doris_home.rstrip("/") + "/" + if path.startswith(home): + return path[len(home):] + return path + + +def shorten_header(path, meta): + doris_home = (meta.get("doris_home") or "").rstrip("/") + if doris_home: + if path.startswith(doris_home + "/thirdparty/installed/include/"): + return "/" + path[len(doris_home + "/thirdparty/installed/include/"):] + if path.startswith(doris_home + "/"): + return path[len(doris_home) + 1:] + return path + + +def group_keys(rel_src): + """Return (level1, level2) directory grouping keys for a doris_home-relative source.""" + rel_dir = os.path.dirname(rel_src) + if rel_dir.startswith("be/src"): + prefix, rest = "be/src", rel_dir[len("be/src"):].strip("/") + else: + prefix, rest = "", rel_dir + parts = [p for p in rest.split("/") if p] + level1 = "/".join([prefix] + parts[:1]) if prefix else ("/".join(parts[:1]) or ".") + level2 = "/".join([prefix] + parts[:2]) if prefix else ("/".join(parts[:2]) or ".") + return level1 or ".", level2 or "." + + +def load_time_traces(compiles, meta, build_dir): + """Aggregate clang -ftime-trace JSONs written next to the object files. + + Durations reported by clang are microseconds. "Source" and template + instantiation timings are inclusive of nested work, so sums across headers + overlap; they rank hotspots, they are not additive wall time. + """ + headers = defaultdict(lambda: [0.0, 0]) # path -> [total_s, count] + templates = defaultdict(lambda: [0.0, 0]) # symbol -> [total_s, count] + tu_split = {} # src -> {total, frontend, backend} + parsed = 0 + for record in compiles: + out = record.get("out") + if not out: + continue + base = os.path.join(record.get("cwd") or build_dir or "", out) + trace_path = os.path.splitext(base)[0] + ".json" + if not os.path.isfile(trace_path): + continue + try: + with open(trace_path) as fh: + events = json.load(fh).get("traceEvents", []) + except (ValueError, OSError): + continue + parsed += 1 + maxima = defaultdict(float) + # clang >= 20 emits "Source" as async begin/end pairs (ph "b"/"e") that + # nest on one tid; older clang emits complete events with "dur". + source_stacks = defaultdict(list) + for event in events: + dur_s = event.get("dur", 0) / 1e6 + name = event.get("name", "") + detail = (event.get("args") or {}).get("detail", "") + if name == "Source": + phase = event.get("ph") + if phase == "b": + source_stacks[event.get("tid")].append((detail, event.get("ts", 0))) + continue + if phase == "e": + stack = source_stacks.get(event.get("tid")) + if not stack: + continue + detail, begin_ts = stack.pop() + dur_s = (event.get("ts", 0) - begin_ts) / 1e6 + if detail: + entry = headers[detail] + entry[0] += dur_s + entry[1] += 1 + elif name in ("InstantiateClass", "InstantiateFunction") and detail: + entry = templates[detail] + entry[0] += dur_s + entry[1] += 1 + elif name in ("ExecuteCompiler", "Frontend", "Backend", + "Total Frontend", "Total Backend"): + if dur_s > maxima[name]: + maxima[name] = dur_s + tu_split[record.get("src") or out] = { + "total_s": maxima["ExecuteCompiler"], + "frontend_s": max(maxima["Frontend"], maxima["Total Frontend"]), + "backend_s": max(maxima["Backend"], maxima["Total Backend"]), + } + return { + "parsed": parsed, + "headers": headers, + "templates": templates, + "tu_split": tu_split, + } + + +def build_report(run_dir, build_dir, top_n, build_status): + meta = read_meta(run_dir) + if not build_dir: + build_dir = meta.get("build_dir") + phases = read_phases(run_dir) + records = read_compile_log(run_dir) + ninja_edges = read_ninja_log(run_dir, build_dir) + + compiles = [r for r in records if r.get("kind") in ("compile", "pch")] + links = [r for r in records if r.get("kind") == "link"] + failed = [r for r in records if r.get("rc", 0) != 0] + + lines = [] + out = lines.append + out("=" * 78) + out(" Doris BE compile benchmark report run: {}".format( + meta.get("run_id", os.path.basename(run_dir.rstrip("/"))))) + out("=" * 78) + out(" build status : {}".format(build_status)) + for key in ("date_utc", "git_branch", "git_commit", "uname", "ncpu", "parallel", + "build_type", "generator", "toolchain", "cxx_version", "enable_pch", + "time_trace", "build_dir"): + if key in meta: + out(" {:<13}: {}".format(key, meta[key])) + + # ---- Phases ------------------------------------------------------------- + out("") + out(section("Phases")) + total_s = None + for phase in phases: + if phase["name"] == "total": + total_s = phase["dur_s"] + for phase in phases: + if phase["name"] == "total": + continue + pct = " ({:5.1f}%)".format(100.0 * phase["dur_s"] / total_s) if total_s else "" + out(" {:<24} {:>8}{}".format(phase["name"], fmt_dur(phase["dur_s"]), pct)) + if total_s is not None: + out(" {:<24} {:>8}".format("total (wall)", fmt_dur(total_s))) + + build_phase_s = None + for phase in phases: + if phase["name"] == "build": + build_phase_s = phase["dur_s"] + + # ---- Build summary ------------------------------------------------------ + out("") + out(section("Build summary")) + sum_wall = sum(r["wall_s"] for r in compiles) + sum_cpu = sum(r["user_s"] + r["sys_s"] for r in compiles) + out(" compile units (compile+pch) : {}".format(len(compiles))) + out(" sum of TU wall time : {}".format(fmt_dur(sum_wall))) + out(" sum of TU cpu time (user+sys) : {}".format(fmt_dur(sum_cpu))) + if build_phase_s: + out(" build phase wall : {}".format(fmt_dur(build_phase_s))) + edge_sum = sum_wall + sum(r["wall_s"] for r in links) + out(" effective parallelism : {:.1f}x (sum TU+link wall / build wall)" + .format(edge_sum / build_phase_s)) + if compiles: + slowest = max(compiles, key=lambda r: r["wall_s"]) + out(" slowest single TU : {} ({})".format( + fmt_dur(slowest["wall_s"]), rel_path(slowest.get("src"), meta))) + hungriest = max(compiles, key=lambda r: r.get("maxrss_mb", 0)) + out(" largest TU peak rss : {:.0f} MB ({})".format( + hungriest.get("maxrss_mb", 0), rel_path(hungriest.get("src"), meta))) + for link in sorted(links, key=lambda r: r["wall_s"], reverse=True)[:5]: + out(" link {:<24} : {} peak rss {:.0f} MB".format( + os.path.basename(link.get("out") or "?"), + fmt_dur(link["wall_s"]), link.get("maxrss_mb", 0))) + if failed: + out(" FAILED commands : {}".format(len(failed))) + for record in failed[:10]: + out(" rc={:<4} {}".format( + record.get("rc"), rel_path(record.get("src") or record.get("out"), meta))) + + # ---- Top slow TUs ------------------------------------------------------- + out("") + out(section("Top {} slowest translation units (wall)".format(top_n))) + out(" {:>4} {:>8} {:>8} {:>7} {:>9} {}".format( + "rank", "wall", "user", "sys", "maxrss", "file")) + ranked = sorted(compiles, key=lambda r: r["wall_s"], reverse=True) + for idx, record in enumerate(ranked[:top_n], 1): + out(" {:>4} {:>8} {:>8} {:>7} {:>7.0f}MB {}{}".format( + idx, fmt_dur(record["wall_s"]), fmt_dur(record["user_s"]), + fmt_dur(record["sys_s"]), record.get("maxrss_mb", 0), + rel_path(record.get("src"), meta), + " [pch]" if record.get("kind") == "pch" else "")) + + # ---- Directory rollup --------------------------------------------------- + for level, title in ((0, "top-level directory"), (1, "second-level directory")): + rollup = defaultdict(lambda: [0.0, 0]) + for record in compiles: + rel = rel_path(record.get("src") or record.get("out") or "?", meta) + key = group_keys(rel)[level] + entry = rollup[key] + entry[0] += record["wall_s"] + entry[1] += 1 + out("") + out(section("Wall time by {}".format(title))) + out(" {:>9} {:>6} {:>8} {}".format("wall-sum", "count", "avg", "directory")) + ordered = sorted(rollup.items(), key=lambda kv: kv[1][0], reverse=True) + for key, (wall, count) in ordered[:TOP_DIRS]: + out(" {:>9} {:>6} {:>8} {}".format( + fmt_dur(wall), count, fmt_dur(wall / count), key)) + + # ---- Ninja tail: what the build waits on at the end --------------------- + if ninja_edges: + out("") + out(section("Last finishers (critical-path tail, from .ninja_log)")) + out(" {:>10} {:>10} {:>8} {}".format("start", "end", "dur", "output")) + tail = sorted(ninja_edges.items(), key=lambda kv: kv[1][1], reverse=True) + for output, (start_ms, end_ms) in tail[:TOP_TAIL]: + out(" {:>10} {:>10} {:>8} {}".format( + fmt_dur(start_ms / 1000.0), fmt_dur(end_ms / 1000.0), + fmt_dur((end_ms - start_ms) / 1000.0), output)) + + # ---- Optional -ftime-trace analysis ------------------------------------- + trace = None + if meta.get("time_trace") == "ON" and build_dir: + trace = load_time_traces(compiles, meta, build_dir) + out("") + out(section("[-ftime-trace] parsed {} trace files".format(trace["parsed"]))) + if trace["parsed"]: + out("") + out(" Top headers by inclusive parse time (overlapping, ranks hotspots):") + out(" {:>9} {:>7} {:>8} {}".format("total", "count", "avg", "header")) + for path, (total, count) in sorted( + trace["headers"].items(), key=lambda kv: kv[1][0], + reverse=True)[:TOP_HEADERS]: + out(" {:>9} {:>7} {:>8} {}".format( + fmt_dur(total), count, fmt_dur(total / count), + shorten_header(path, meta))) + out("") + out(" Top template instantiations (inclusive):") + out(" {:>9} {:>7} {}".format("total", "count", "symbol")) + for symbol, (total, count) in sorted( + trace["templates"].items(), key=lambda kv: kv[1][0], + reverse=True)[:TOP_TEMPLATES]: + out(" {:>9} {:>7} {}".format(fmt_dur(total), count, symbol[:110])) + out("") + out(" Frontend (parse/instantiate) vs backend (codegen/opt) of slowest TUs:") + out(" {:>9} {:>9} {:>9} {}".format("total", "frontend", "backend", "file")) + for record in ranked[:15]: + split = trace["tu_split"].get(record.get("src") or "") + if not split: + continue + out(" {:>9} {:>9} {:>9} {}".format( + fmt_dur(split["total_s"]), fmt_dur(split["frontend_s"]), + fmt_dur(split["backend_s"]), rel_path(record.get("src"), meta))) + else: + out(" (no trace files found under {} - was the build dir wiped?)" + .format(build_dir)) + + out("") + out("=" * 78) + + summary = { + "meta": meta, + "build_status": build_status, + "phases": {p["name"]: round(p["dur_s"], 1) for p in phases}, + "totals": { + "compile_units": len(compiles), + "sum_tu_wall_s": round(sum_wall, 1), + "sum_tu_cpu_s": round(sum_cpu, 1), + "build_phase_s": round(build_phase_s, 1) if build_phase_s else None, + }, + "files": { + rel_path(r.get("src") or r.get("out"), meta): { + "wall_s": r["wall_s"], + "user_s": r["user_s"], + "maxrss_mb": r.get("maxrss_mb", 0), + "kind": r.get("kind"), + } + for r in compiles + links + }, + } + if trace and trace["parsed"]: + summary["headers_top"] = { + shorten_header(path, meta): round(total, 1) + for path, (total, _) in sorted( + trace["headers"].items(), key=lambda kv: kv[1][0], reverse=True)[:100] + } + summary["templates_top"] = { + symbol[:200]: round(total, 1) + for symbol, (total, _) in sorted( + trace["templates"].items(), key=lambda kv: kv[1][0], reverse=True)[:100] + } + return lines, summary + + +def cmd_report(args): + run_dir = args.run_dir + if not os.path.isdir(run_dir): + print("ERROR: run dir not found: {}".format(run_dir), file=sys.stderr) + return 1 + lines, summary = build_report(run_dir, args.build_dir, args.top, args.build_status) + report_path = os.path.join(run_dir, "report.txt") + with open(report_path, "w") as fh: + fh.write("\n".join(lines) + "\n") + with open(os.path.join(run_dir, "summary.json"), "w") as fh: + json.dump(summary, fh, indent=1, sort_keys=True) + print("\n".join(lines)) + print("Report written to {}".format(report_path)) + return 0 + + +def load_summary(path): + if os.path.isdir(path): + path = os.path.join(path, "summary.json") + with open(path) as fh: + return json.load(fh) + + +def cmd_compare(args): + old, new = load_summary(args.old), load_summary(args.new) + print("=" * 78) + print(" Compile benchmark comparison") + print(" old: {} ({})".format(old["meta"].get("run_id"), old["meta"].get("git_commit"))) + print(" new: {} ({})".format(new["meta"].get("run_id"), new["meta"].get("git_commit"))) + print("=" * 78) + + print("") + print(section("Phases")) + print(" {:<24} {:>9} {:>9} {:>9}".format("phase", "old", "new", "delta")) + for name in sorted(set(old["phases"]) | set(new["phases"])): + old_s, new_s = old["phases"].get(name), new["phases"].get(name) + delta = "" if old_s is None or new_s is None else "{:+.1f}s".format(new_s - old_s) + print(" {:<24} {:>9} {:>9} {:>9}".format( + name, + fmt_dur(old_s) if old_s is not None else "-", + fmt_dur(new_s) if new_s is not None else "-", + delta)) + + old_files, new_files = old.get("files", {}), new.get("files", {}) + both = set(old_files) & set(new_files) + deltas = sorted( + ((new_files[f]["wall_s"] - old_files[f]["wall_s"], f) for f in both), + key=lambda pair: pair[0]) + threshold = 0.5 + improved = [d for d in deltas if d[0] < -threshold] + regressed = [d for d in deltas if d[0] > threshold] + + print("") + print(section("Per-file wall time changes (threshold {:.1f}s)".format(threshold))) + print(" improved: {} regressed: {} only-in-old: {} only-in-new: {}".format( + len(improved), len(regressed), + len(set(old_files) - both), len(set(new_files) - both))) + for title, items in (("Top improvements", improved[:20]), + ("Top regressions", list(reversed(regressed[-20:])))): + print("") + print(" {}:".format(title)) + for delta, name in items: + print(" {:>+8.1f}s {:>8} -> {:<8} {}".format( + delta, fmt_dur(old_files[name]["wall_s"]), + fmt_dur(new_files[name]["wall_s"]), name)) + return 0 + + +def main(): + argv = sys.argv[1:] + if argv and argv[0] == "compare": + parser = argparse.ArgumentParser( + prog="report.py compare", description="compare two compile-bench runs") + parser.add_argument("old", help="run dir or summary.json of the baseline") + parser.add_argument("new", help="run dir or summary.json of the new run") + return cmd_compare(parser.parse_args(argv[1:])) + + if argv and argv[0] == "report": + argv = argv[1:] + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("run_dir", help="compile-bench run directory") + parser.add_argument("--build-dir", default=None, + help="BE build dir (default: build_dir from meta.tsv)") + parser.add_argument("--top", type=int, default=40, + help="how many slowest TUs to list (default 40)") + parser.add_argument("--build-status", default="ok", + help="build outcome recorded in the report") + return cmd_report(parser.parse_args(argv)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build-support/compile-bench/syntax_sweep.py b/build-support/compile-bench/syntax_sweep.py new file mode 100644 index 00000000000000..0e5e2b7d1c32ea --- /dev/null +++ b/build-support/compile-bench/syntax_sweep.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Parallel -fsyntax-only sweep over BE TUs using compile_commands.json. + +Validates include-structure changes (header cuts, forward-declaration swaps) +against every TU without mutating the ninja build state: each compile command +is replayed with -o/-c/-MD/-MT/-MF/-ftime-trace* stripped and -fsyntax-only +appended, so nothing is written to the build directory. Front-end-only checks +run in roughly half the time of a real compile and catch every missing-include +or missing-declaration fallout a cut can cause. + +Usage: + python3 syntax_sweep.py [--build-dir DIR] [--jobs N] [--filter SUBSTR] + [--fail-log FILE] + + --filter limits the sweep to TUs whose source path contains SUBSTR + (e.g. --filter load/memtable for a quick re-check of one subsystem). + +Exit code: 0 if every TU passes, 1 otherwise. +""" + +import argparse +import concurrent.futures +import json +import os +import re +import shlex +import subprocess +import sys +import time + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(os.path.dirname(SCRIPT_DIR)) +ANSI = re.compile(r"\x1b\[[0-9;]*m") + +STRIP_WITH_ARG = ("-o", "-MT", "-MF") +STRIP_FLAGS = ("-c", "-MD", "-MMD") + + +def mangle(cmd): + args = shlex.split(cmd) + out, skip = [], False + for a in args: + if skip: + skip = False + continue + if a in STRIP_WITH_ARG: + skip = True + continue + if a in STRIP_FLAGS or a.startswith("-ftime-trace"): + continue + out.append(a) + out.append("-fsyntax-only") + return out + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build-dir", + default=os.path.join(REPO_ROOT, "be", "build_Release_compile_bench")) + ap.add_argument("--jobs", type=int, default=max(2, (os.cpu_count() or 8) // 2)) + ap.add_argument("--filter", default="", + help="only sweep TUs whose path contains this substring") + ap.add_argument("--fail-log", default="", + help="write full stderr of every failing TU to this file") + args = ap.parse_args() + + src_prefix = os.path.join(REPO_ROOT, "be", "src") + os.sep + with open(os.path.join(args.build_dir, "compile_commands.json")) as f: + entries = [e for e in json.load(f) + if e["file"].startswith(src_prefix) and args.filter in e["file"]] + print(f"{len(entries)} TUs to check ({args.jobs} jobs)", flush=True) + t0 = time.time() + fails = [] + done = 0 + + def run(e): + p = subprocess.run(mangle(e["command"]), cwd=e["directory"], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + text=True) + return e["file"], p.returncode, p.stderr + + with concurrent.futures.ThreadPoolExecutor(args.jobs) as ex: + for src, rc, err in ex.map(run, entries): + done += 1 + if rc != 0: + fails.append((src, err)) + print(f"FAIL {src.replace(src_prefix, '')}", flush=True) + if done % 100 == 0: + print(f" …{done}/{len(entries)} ({time.time() - t0:.0f}s, " + f"{len(fails)} failures)", flush=True) + + print(f"\n==== {len(fails)} failing TU(s) of {len(entries)} " + f"in {time.time() - t0:.0f}s ====", flush=True) + if args.fail_log and fails: + with open(args.fail_log, "w") as f: + for src, err in fails: + f.write(f"===== {src}\n{ANSI.sub('', err)}\n") + for src, err in fails[:15]: + first = [l for l in ANSI.sub("", err).splitlines() if " error: " in l][:2] + print(src.replace(src_prefix, "")) + for l in first: + print(" ", l[:200]) + return 1 if fails else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build.sh b/build.sh index 0565ae006ac9d3..b7355bd86d213e 100755 --- a/build.sh +++ b/build.sh @@ -69,6 +69,10 @@ Usage: $0 --enable-dynamic-arch enable dynamic CPU detection in OpenBLAS. Default ON. --disable-dynamic-arch disable dynamic CPU detection in OpenBLAS. --clean clean and build target + --compile-bench BE compile-speed benchmark: cold, cache-free BE-only build + (fresh dedicated build dir, ccache disabled) with a per-phase + and per-file timing report. Implies --be; FE/cloud/java + extensions/packaging are skipped. For build speed analysis only. --output specify the output directory -j build Backend parallel @@ -85,6 +89,9 @@ Usage: $0 EXTRA_FE_MODULES Optional FE feature modules in feature=module_path format, separated by commas. EXTRA_BE_MODULES Optional BE feature modules in feature=module_path format, separated by commas. EXTRA_CLOUD_MODULES Optional CLOUD feature modules in feature=module_path format, separated by commas. + COMPILE_BENCH_TRACE If set COMPILE_BENCH_TRACE=ON together with --compile-bench (clang only), + compile with -ftime-trace and aggregate per-header/per-template costs + into the benchmark report. Default is OFF. Eg. $0 build all $0 --be build Backend @@ -101,6 +108,9 @@ Usage: $0 $0 --be --output PATH build Backend, the result will be output to PATH(relative paths are available) $0 --be-extension-ignore paimon-scanner build be-java-extensions, choose which modules to ignore. Multiple modules separated by commas, like --be-extension-ignore paimon-scanner,hadoop-hudi-scanner + $0 --compile-bench benchmark a cold cache-free BE build and report the slowest files + COMPILE_BENCH_TRACE=ON $0 --compile-bench benchmark and also collect clang -ftime-trace data + USE_AVX2=0 $0 --be build Backend and not using AVX2 instruction. USE_AVX2=0 STRIP_DEBUG_INFO=ON $0 build all and not using AVX2 instruction, and strip the debug info for Backend ARM_MARCH=armv8-a+crc+simd $0 --be build Backend with specified ARM architecture instruction set @@ -261,6 +271,7 @@ if ! OPTS="$(getopt \ -l 'enable-dynamic-arch' \ -l 'disable-dynamic-arch' \ -l 'clean' \ + -l 'compile-bench' \ -l 'coverage' \ -l 'help' \ -l 'output:' \ @@ -287,6 +298,7 @@ BUILD_COS_DEPENDENCIES=1 BUILD_HIVE_UDF=0 ENABLE_DYNAMIC_ARCH='ON' CLEAN=0 +COMPILE_BENCH=0 HELP=0 PARAMETER_COUNT="$#" PARAMETER_FLAG=0 @@ -393,6 +405,10 @@ else CLEAN=1 shift ;; + --compile-bench) + COMPILE_BENCH=1 + shift + ;; --coverage) DENABLE_CLANG_COVERAGE='ON' shift @@ -644,6 +660,7 @@ parse_extra_modules "BE_EXTRA" "${EXTRA_BE_MODULES}" "${DORIS_HOME}/be/src" "be" parse_extra_modules "CLOUD_EXTRA" "${EXTRA_CLOUD_MODULES}" "${DORIS_HOME}/cloud/src" "cloud" BE_EXTRA_CMAKE_ARGS=() +COMPILE_BENCH_CMAKE_ARGS=() for ((i = 0; i < ${#BE_EXTRA_FEATURE_KEYS[@]}; i++)); do feature_name="$(feature_to_cmake_name "${BE_EXTRA_FEATURE_KEYS[i]}")" BE_EXTRA_CMAKE_ARGS+=("-DENABLE_${feature_name}=ON") @@ -657,6 +674,22 @@ for ((i = 0; i < ${#CLOUD_EXTRA_FEATURE_KEYS[@]}; i++)); do CLOUD_EXTRA_CMAKE_ARGS+=("-D${feature_name}_MODULE_DIR=${CLOUD_EXTRA_MODULE_PATHS[i]}") done +if [[ "${COMPILE_BENCH}" -eq 1 ]]; then + # BE compile benchmark mode: measure a cold, cache-free BE C++ build. + # Everything that is not the BE C++ build would only add noise, so force + # a BE-only build regardless of the other options. + BUILD_BE=1 + BUILD_FE=0 + BUILD_CLOUD=0 + BUILD_HIVE_UDF=0 + BUILD_BE_JAVA_EXTENSIONS=0 + BUILD_BE_CDC_CLIENT=0 + OUTPUT_BE_BINARY=0 + # shellcheck source=build-support/compile-bench/bench-lib.sh + . "${DORIS_HOME}/build-support/compile-bench/bench-lib.sh" + compile_bench_init "${DORIS_HOME}" +fi + echo "Get params: BUILD_FE -- ${BUILD_FE} BUILD_BE -- ${BUILD_BE} @@ -705,7 +738,13 @@ echo "Feature List: ${DORIS_FEATURE_LIST}" if [[ "${CLEAN}" -eq 1 ]]; then clean_gensrc fi +if [[ "${COMPILE_BENCH}" -eq 1 ]]; then + compile_bench_phase_begin "gensrc" +fi bash "${DORIS_HOME}"/generated-source.sh noclean +if [[ "${COMPILE_BENCH}" -eq 1 ]]; then + compile_bench_phase_end +fi # Assesmble FE modules FE_MODULES='' @@ -769,17 +808,27 @@ FE_MODULES="$( # Clean and build Backend if [[ "${BUILD_BE}" -eq 1 ]]; then + if [[ "${COMPILE_BENCH}" -eq 1 ]]; then + compile_bench_phase_begin "datasketches_install" + fi echo "install datasketches-cpp to thirdparty path before build be" update_submodule "contrib/datasketches-cpp" "datasketches-cpp" "https://github.com/apache/datasketches-cpp/archive/refs/heads/master.tar.gz" cd "${DORIS_HOME}/contrib/datasketches-cpp" "${CMAKE_CMD}" -S . -B build/Release -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$TP_INSTALLED_DIR -DBUILD_TESTS=OFF "${CMAKE_CMD}" --build build/Release -t install cd "${DORIS_HOME}" + if [[ "${COMPILE_BENCH}" -eq 1 ]]; then + compile_bench_phase_end + compile_bench_phase_begin "contrib_submodules" + fi update_submodule "contrib/apache-orc" "apache-orc" "https://github.com/apache/doris-thirdparty/archive/refs/heads/orc.tar.gz" update_submodule "contrib/clucene" "clucene" "https://github.com/apache/doris-thirdparty/archive/refs/heads/clucene.tar.gz" update_submodule "contrib/openblas" "openblas" "https://github.com/apache/doris-thirdparty/archive/refs/heads/openblas.tar.gz" update_submodule "contrib/faiss" "faiss" "https://github.com/apache/doris-thirdparty/archive/refs/heads/faiss.tar.gz" + if [[ "${COMPILE_BENCH}" -eq 1 ]]; then + compile_bench_phase_end + fi if [[ -e "${DORIS_HOME}/gensrc/build/gen_cpp/version.h" ]]; then rm -f "${DORIS_HOME}/gensrc/build/gen_cpp/version.h" fi @@ -789,6 +838,13 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then if [[ "${CLEAN}" -eq 1 ]]; then clean_be fi + if [[ "${COMPILE_BENCH}" -eq 1 ]]; then + # Dedicated always-cold build dir: no reused objects, no reused CMake + # cache, and the developer's normal build dir stays untouched. + CMAKE_BUILD_DIR="${COMPILE_BENCH_BUILD_DIR}" + echo "Compile-bench: recreating build dir ${CMAKE_BUILD_DIR} from scratch" + rm -rf "${CMAKE_BUILD_DIR}" + fi MAKE_PROGRAM="$(command -v "${BUILD_SYSTEM}")" if [[ -z "${BUILD_FS_BENCHMARK}" ]]; then @@ -812,6 +868,9 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then mkdir -p "${CMAKE_BUILD_DIR}" cd "${CMAKE_BUILD_DIR}" + if [[ "${COMPILE_BENCH}" -eq 1 ]]; then + compile_bench_phase_begin "cmake_configure" + fi "${CMAKE_CMD}" -G "${GENERATOR}" \ -DCMAKE_MAKE_PROGRAM="${MAKE_PROGRAM}" \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ @@ -843,8 +902,25 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then -DENABLE_DYNAMIC_ARCH="${ENABLE_DYNAMIC_ARCH}" \ -DFAISS_ENABLE_GPU="${FAISS_ENABLE_GPU:-OFF}" \ "${BE_EXTRA_CMAKE_ARGS[@]}" \ + "${COMPILE_BENCH_CMAKE_ARGS[@]}" \ "${DORIS_HOME}/be" + if [[ "${COMPILE_BENCH}" -eq 1 ]]; then + compile_bench_phase_end + + compile_bench_phase_begin "build" + set +e + "${BUILD_SYSTEM}" -j "${PARALLEL}" + compile_bench_build_rc=$? + set -e + compile_bench_phase_end + + # Generate the timing report even for a failed build, then stop: + # install/packaging is out of scope for a compile benchmark. + compile_bench_finish "${CMAKE_BUILD_DIR}" "${compile_bench_build_rc}" + exit "${compile_bench_build_rc}" + fi + if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install