From 1017005acab374f7f16cadfa34728425ca1730ed Mon Sep 17 00:00:00 2001 From: namtran1812 <158846154+namtran1812@users.noreply.github.com> Date: Sun, 20 Sep 2026 14:24:55 -0400 Subject: [PATCH] [C++][Parquet] Avoid rebuilding column paths during schema initialization --- cpp/src/parquet/metadata_benchmark.cc | 5 +++++ cpp/src/parquet/schema.cc | 19 ++++++++++++++----- cpp/src/parquet/schema.h | 2 +- cpp/src/parquet/schema_test.cc | 16 ++++++++++++++++ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/cpp/src/parquet/metadata_benchmark.cc b/cpp/src/parquet/metadata_benchmark.cc index 97a99be798cb..2864e4347ba8 100644 --- a/cpp/src/parquet/metadata_benchmark.cc +++ b/cpp/src/parquet/metadata_benchmark.cc @@ -125,6 +125,11 @@ void WriteMetadataSetArgs(benchmark::internal::Benchmark* bench) { for (int num_row_groups : {1, 100}) { bench->Args({/*num_columns=*/1000, num_row_groups}); } + + // Stress very wide schemas separately with a single row group. + for (int num_columns : {5000, 10000, 20000, 50000}) { + bench->Args({num_columns, /*num_row_groups=*/1}); + } } void ReadMetadataSetArgs(benchmark::internal::Benchmark* bench) { diff --git a/cpp/src/parquet/schema.cc b/cpp/src/parquet/schema.cc index 3cb91f9a84eb..1af01c084e87 100644 --- a/cpp/src/parquet/schema.cc +++ b/cpp/src/parquet/schema.cc @@ -823,8 +823,9 @@ void SchemaDescriptor::Init(NodePtr schema) { group_node_ = static_cast(schema_.get()); leaves_.clear(); + std::string path; for (int i = 0; i < group_node_->field_count(); ++i) { - BuildTree(group_node_->field(i), 0, 0, group_node_->field(i)); + BuildTree(group_node_->field(i), 0, 0, group_node_->field(i), path); } } @@ -853,7 +854,14 @@ bool SchemaDescriptor::Equals(const SchemaDescriptor& other, } void SchemaDescriptor::BuildTree(const NodePtr& node, int16_t max_def_level, - int16_t max_rep_level, const NodePtr& base) { + int16_t max_rep_level, const NodePtr& base, + std::string& path) { + const size_t path_size = path.size(); + if (!path.empty()) { + path.push_back('.'); + } + path.append(node->name()); + if (node->is_optional()) { ++max_def_level; } else if (node->is_repeated()) { @@ -867,7 +875,7 @@ void SchemaDescriptor::BuildTree(const NodePtr& node, int16_t max_def_level, if (node->is_group()) { const GroupNode* group = static_cast(node.get()); for (int i = 0; i < group->field_count(); ++i) { - BuildTree(group->field(i), max_def_level, max_rep_level, base); + BuildTree(group->field(i), max_def_level, max_rep_level, base, path); } } else { node_to_leaf_index_[static_cast(node.get())] = @@ -876,9 +884,10 @@ void SchemaDescriptor::BuildTree(const NodePtr& node, int16_t max_def_level, // Primitive node, append to leaves leaves_.push_back(ColumnDescriptor(node, max_def_level, max_rep_level, this)); leaf_to_base_.emplace(static_cast(leaves_.size()) - 1, base); - leaf_to_idx_.emplace(node->path()->ToDotString(), - static_cast(leaves_.size()) - 1); + leaf_to_idx_.emplace(path, static_cast(leaves_.size()) - 1); } + + path.resize(path_size); } int SchemaDescriptor::GetColumnIndex(const PrimitiveNode& node) const { diff --git a/cpp/src/parquet/schema.h b/cpp/src/parquet/schema.h index 65732603ea1d..26120d130407 100644 --- a/cpp/src/parquet/schema.h +++ b/cpp/src/parquet/schema.h @@ -485,7 +485,7 @@ class PARQUET_EXPORT SchemaDescriptor { const schema::GroupNode* group_node_; void BuildTree(const schema::NodePtr& node, int16_t max_def_level, - int16_t max_rep_level, const schema::NodePtr& base); + int16_t max_rep_level, const schema::NodePtr& base, std::string& path); // Result of leaf node / tree analysis std::vector leaves_; diff --git a/cpp/src/parquet/schema_test.cc b/cpp/src/parquet/schema_test.cc index 6c8e6366adf8..268bda5f5ce0 100644 --- a/cpp/src/parquet/schema_test.cc +++ b/cpp/src/parquet/schema_test.cc @@ -870,6 +870,22 @@ TEST_F(TestSchemaDescriptor, BuildTree) { ASSERT_EQ(nleaves, descr_.num_columns()); } +TEST_F(TestSchemaDescriptor, ColumnIndexDuplicatePath) { + NodePtr first = Int32("duplicate", Repetition::REQUIRED); + NodePtr second = Int64("duplicate", Repetition::OPTIONAL); + NodePtr schema = GroupNode::Make("schema", Repetition::REQUIRED, {first, second}); + + descr_.Init(schema); + + ASSERT_EQ(2, descr_.num_columns()); + ASSERT_EQ("duplicate", descr_.Column(0)->path()->ToDotString()); + ASSERT_EQ("duplicate", descr_.Column(1)->path()->ToDotString()); + + // Duplicate paths are disambiguated by node identity. + ASSERT_EQ(0, descr_.ColumnIndex(*first)); + ASSERT_EQ(1, descr_.ColumnIndex(*second)); +} + TEST_F(TestSchemaDescriptor, HasRepeatedFields) { NodeVector fields; NodePtr schema;