From ff3cd4d81d45831f432bead428432bbcd8f476d0 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Thu, 27 Aug 2026 13:34:35 +0800 Subject: [PATCH 1/2] feat: optimize predicate in --- src/paimon/CMakeLists.txt | 2 + .../common/predicate/leaf_predicate_impl.h | 48 ++- src/paimon/common/predicate/literal_set.cpp | 384 ++++++++++++++++++ src/paimon/common/predicate/literal_set.h | 150 +++++++ .../common/predicate/literal_set_test.cpp | 332 +++++++++++++++ .../common/predicate/predicate_test.cpp | 63 +++ 6 files changed, 974 insertions(+), 5 deletions(-) create mode 100644 src/paimon/common/predicate/literal_set.cpp create mode 100644 src/paimon/common/predicate/literal_set.h create mode 100644 src/paimon/common/predicate/literal_set_test.cpp diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 1749634f4..6ed948d19 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -124,6 +124,7 @@ set(PAIMON_COMMON_SRCS common/predicate/less_than.cpp common/predicate/like.cpp common/predicate/literal_converter.cpp + common/predicate/literal_set.cpp common/predicate/literal.cpp common/predicate/not_equal.cpp common/predicate/not_in.cpp @@ -604,6 +605,7 @@ if(PAIMON_BUILD_TESTS) common/options/memory_size_test.cpp common/options/time_duration_test.cpp common/predicate/literal_converter_test.cpp + common/predicate/literal_set_test.cpp common/predicate/literal_test.cpp common/predicate/predicate_test.cpp common/predicate/predicate_utils_test.cpp diff --git a/src/paimon/common/predicate/leaf_predicate_impl.h b/src/paimon/common/predicate/leaf_predicate_impl.h index c9a8758f5..ad9d9ef03 100644 --- a/src/paimon/common/predicate/leaf_predicate_impl.h +++ b/src/paimon/common/predicate/leaf_predicate_impl.h @@ -20,12 +20,14 @@ #include #include +#include #include #include "arrow/api.h" #include "paimon/common/predicate/compound_function.h" #include "paimon/common/predicate/leaf_function.h" #include "paimon/common/predicate/literal_converter.h" +#include "paimon/common/predicate/literal_set.h" #include "paimon/common/predicate/predicate_filter.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/predicate/leaf_predicate.h" @@ -35,7 +37,8 @@ class LeafPredicateImpl : public LeafPredicate, public PredicateFilter { LeafPredicateImpl(const LeafFunction& leaf_function, int32_t field_index, const std::string& field_name, const FieldType& field_type, const std::vector& literals) - : LeafPredicate(leaf_function, field_index, field_name, field_type, literals) {} + : LeafPredicateImpl(leaf_function, field_index, field_name, field_type, literals, + BuildLiteralSet(leaf_function, field_type, literals)) {} const LeafFunction& GetLeafFunction() const { return leaf_function_; @@ -49,6 +52,11 @@ class LeafPredicateImpl : public LeafPredicate, public PredicateFilter { struct_array.fields().size())); } const auto& field_array = struct_array.field(field_index_); + if (literal_set_ && literal_set_->MatchesArrowType(*field_array)) { + std::vector is_valid(field_array->length(), 0); + PAIMON_RETURN_NOT_OK(literal_set_->TestArray(*field_array, negate_in_, &is_valid)); + return is_valid; + } return leaf_function_.Test(*field_array, literals_); } @@ -60,6 +68,9 @@ class LeafPredicateImpl : public LeafPredicate, public PredicateFilter { } PAIMON_ASSIGN_OR_RAISE(Literal value, LiteralConverter::ConvertLiteralsFromRow( schema, row, field_index_, field_type_)); + if (literal_set_ && value.GetType() == field_type_) { + return literal_set_->TestValue(value, negate_in_); + } return leaf_function_.Test(value, literals_); } @@ -86,14 +97,41 @@ class LeafPredicateImpl : public LeafPredicate, public PredicateFilter { return leaf_function_.Test(row_count, min_value, max_value, null_count, literals_); } + // Rebinding to another schema keeps the literals untouched, so the lookup structure is shared + // instead of rebuilt for every reader. std::shared_ptr NewLeafPredicate(int32_t new_field_index) const { - return std::make_shared(leaf_function_, new_field_index, field_name_, - field_type_, literals_); + return std::shared_ptr(new LeafPredicateImpl( + leaf_function_, new_field_index, field_name_, field_type_, literals_, literal_set_)); } std::shared_ptr NewLeafPredicate(const std::string& new_field_name) const { - return std::make_shared(leaf_function_, field_index_, new_field_name, - field_type_, literals_); + return std::shared_ptr(new LeafPredicateImpl( + leaf_function_, field_index_, new_field_name, field_type_, literals_, literal_set_)); } + + private: + LeafPredicateImpl(const LeafFunction& leaf_function, int32_t field_index, + const std::string& field_name, const FieldType& field_type, + const std::vector& literals, + std::shared_ptr literal_set) + : LeafPredicate(leaf_function, field_index, field_name, field_type, literals), + literal_set_(std::move(literal_set)), + negate_in_(leaf_function.GetType() == Function::Type::NOT_IN) {} + + static std::shared_ptr BuildLiteralSet(const LeafFunction& leaf_function, + const FieldType& field_type, + const std::vector& literals) { + Function::Type type = leaf_function.GetType(); + if (type != Function::Type::IN && type != Function::Type::NOT_IN) { + return nullptr; + } + return LiteralSet::CreateOrNull(field_type, literals); + } + + // Built once at construction time and never mutated afterwards, so concurrent `Test` calls on + // the same predicate stay safe. Null when the type or the literals are not supported, in which + // case every path falls back to `leaf_function_`. + std::shared_ptr literal_set_; + bool negate_in_; }; } // namespace paimon diff --git a/src/paimon/common/predicate/literal_set.cpp b/src/paimon/common/predicate/literal_set.cpp new file mode 100644 index 000000000..aea2fb408 --- /dev/null +++ b/src/paimon/common/predicate/literal_set.cpp @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/predicate/literal_set.h" + +#include +#include +#include +#include + +#include "arrow/array/array_binary.h" +#include "arrow/array/array_dict.h" +#include "arrow/array/array_primitive.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/field_type_utils.h" + +namespace paimon { +namespace { +// A dense bitmap is only worth it when the value range stays close to the literal count and small +// enough to keep the bitmap in cache friendly territory. +constexpr uint64_t DENSE_SPAN_FACTOR = 4; +constexpr uint64_t MAX_DENSE_SPAN = 1ULL << 20; + +// Maps an arrow array to the `FieldType` that `LiteralConverter::ConvertLiteralsFromArray` would +// produce for it. Types outside `LiteralSet`'s support are intentionally left unmapped: +// `field_type_` can never be one of them, so `MatchesArrowType` returns false and the caller falls +// back. +std::optional MapArrowType(const arrow::Array& array) { + switch (array.type_id()) { + case arrow::Type::type::BOOL: + return FieldType::BOOLEAN; + case arrow::Type::type::INT8: + return FieldType::TINYINT; + case arrow::Type::type::INT16: + return FieldType::SMALLINT; + case arrow::Type::type::INT32: + return FieldType::INT; + case arrow::Type::type::INT64: + return FieldType::BIGINT; + case arrow::Type::type::STRING: + return FieldType::STRING; + case arrow::Type::type::BINARY: + return FieldType::BINARY; + case arrow::Type::type::DATE32: + return FieldType::DATE; + case arrow::Type::type::DICTIONARY: { + const auto& dict_array = checked_cast(array); + const auto* dict_type = + checked_cast(dict_array.type().get()); + auto value_type_id = dict_type->value_type()->id(); + auto index_type_id = dict_type->index_type()->id(); + if ((value_type_id == arrow::Type::type::STRING && + index_type_id == arrow::Type::type::INT32) || + (value_type_id == arrow::Type::type::LARGE_STRING && + index_type_id == arrow::Type::type::INT64)) { + return FieldType::STRING; + } + return std::nullopt; + } + default: + return std::nullopt; + } +} +} // namespace + +std::shared_ptr LiteralSet::CreateOrNull(FieldType field_type, + const std::vector& literals) { + if (literals.empty()) { + return nullptr; + } + // A literal typed differently makes `Literal::CompareTo` fail, keep that on the fallback path. + for (const auto& literal : literals) { + if (!literal.IsNull() && literal.GetType() != field_type) { + return nullptr; + } + } + + std::shared_ptr literal_set(new LiteralSet(field_type)); + bool built = false; + switch (field_type) { + case FieldType::BOOLEAN: + built = literal_set->BuildBooleans(literals); + break; + case FieldType::TINYINT: + case FieldType::SMALLINT: + case FieldType::INT: + case FieldType::BIGINT: + case FieldType::DATE: + built = literal_set->BuildIntegers(literals); + break; + case FieldType::STRING: + case FieldType::BINARY: + built = literal_set->BuildBinaries(literals); + break; + default: + // FLOAT / DOUBLE carry NaN semantics, TIMESTAMP needs unit conversion and DECIMAL + // compares across scales, all of them keep the generic comparison path. + built = false; + break; + } + if (!built) { + return nullptr; + } + return literal_set; +} + +bool LiteralSet::BuildBooleans(const std::vector& literals) { + for (const auto& literal : literals) { + if (literal.IsNull()) { + has_null_literal_ = true; + } else if (literal.GetValue()) { + has_true_ = true; + } else { + has_false_ = true; + } + } + return true; +} + +bool LiteralSet::BuildIntegers(const std::vector& literals) { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + if (literal.IsNull()) { + has_null_literal_ = true; + continue; + } + switch (field_type_) { + case FieldType::TINYINT: + values.push_back(literal.GetValue()); + break; + case FieldType::SMALLINT: + values.push_back(literal.GetValue()); + break; + case FieldType::INT: + case FieldType::DATE: + values.push_back(literal.GetValue()); + break; + default: + values.push_back(literal.GetValue()); + break; + } + } + if (values.empty()) { + // Only null literals: `IN` never matches and `NOT IN` is always false, both handled without + // any lookup structure. + min_ = std::numeric_limits::max(); + max_ = std::numeric_limits::min(); + return true; + } + + const auto [min_it, max_it] = std::minmax_element(values.begin(), values.end()); + min_ = *min_it; + max_ = *max_it; + const uint64_t span = static_cast(max_) - static_cast(min_) + 1; + if (span <= MAX_DENSE_SPAN && span <= DENSE_SPAN_FACTOR * values.size()) { + dense_.assign(static_cast(span), false); + for (int64_t value : values) { + dense_[static_cast(value - min_)] = true; + } + } else { + sparse_.reserve(values.size()); + for (int64_t value : values) { + sparse_.insert(value); + } + } + return true; +} + +bool LiteralSet::BuildBinaries(const std::vector& literals) { + min_length_ = std::numeric_limits::max(); + max_length_ = 0; + for (const auto& literal : literals) { + if (literal.IsNull()) { + has_null_literal_ = true; + continue; + } + binary_storage_.push_back(literal.GetValue()); + const std::string& value = binary_storage_.back(); + min_length_ = std::min(min_length_, value.size()); + max_length_ = std::max(max_length_, value.size()); + if (value.empty()) { + has_empty_binary_ = true; + } else { + const auto first_byte = static_cast(value[0]); + first_byte_bitmap_[first_byte >> 6] |= 1ULL << (first_byte & 63); + } + binary_set_.insert(std::string_view(value)); + } + if (binary_storage_.empty()) { + // Only null literals, see `BuildIntegers`. Keeps `ContainsBinary` rejecting everything. + min_length_ = 1; + max_length_ = 0; + } + return true; +} + +bool LiteralSet::MatchesArrowType(const arrow::Array& array) const { + std::optional mapped = MapArrowType(array); + return mapped.has_value() && mapped.value() == field_type_; +} + +void LiteralSet::TestBooleanArray(const arrow::Array& array, bool negate, + std::vector* out) const { + const auto& typed = checked_cast(array); + for (int64_t i = 0; i < typed.length(); i++) { + if (typed.IsNull(i)) { + continue; + } + const bool contains = typed.Value(i) ? has_true_ : has_false_; + (*out)[i] = static_cast(contains != negate); + } +} + +template +void LiteralSet::TestIntegerArray(const arrow::Array& array, bool negate, + std::vector* out) const { + const auto& typed = checked_cast(array); + const auto* values = typed.raw_values(); + const int64_t length = typed.length(); + if (typed.null_count() == 0) { + for (int64_t i = 0; i < length; i++) { + (*out)[i] = + static_cast(ContainsInteger(static_cast(values[i])) != negate); + } + return; + } + for (int64_t i = 0; i < length; i++) { + if (typed.IsNull(i)) { + continue; + } + (*out)[i] = static_cast(ContainsInteger(static_cast(values[i])) != negate); + } +} + +template +void LiteralSet::TestBinaryArray(const arrow::Array& array, bool negate, + std::vector* out) const { + const auto& typed = checked_cast(array); + const int64_t length = typed.length(); + for (int64_t i = 0; i < length; i++) { + if (typed.IsNull(i)) { + continue; + } + (*out)[i] = static_cast(ContainsBinary(typed.GetView(i)) != negate); + } +} + +template +void LiteralSet::TestDictionaryArray(const arrow::DictionaryArray& dict_array, bool negate, + std::vector* out) const { + const auto& dictionary = checked_cast(*dict_array.dictionary()); + const auto& indices = checked_cast(*dict_array.indices()); + const int64_t dict_length = dictionary.length(); + // Probe the dictionary once and then only follow indices, O(dict_size + rows). + std::vector dict_hits(dict_length, 0); + for (int64_t i = 0; i < dict_length; i++) { + if (!dictionary.IsNull(i)) { + dict_hits[i] = static_cast(ContainsBinary(dictionary.GetView(i))); + } + } + for (int64_t i = 0; i < dict_array.length(); i++) { + if (dict_array.IsNull(i)) { + continue; + } + const int64_t dict_index = indices.Value(i); + const bool contains = + dict_index >= 0 && dict_index < dict_length && dict_hits[dict_index] != 0; + (*out)[i] = static_cast(contains != negate); + } +} + +Status LiteralSet::TestArray(const arrow::Array& array, bool negate, std::vector* out) const { + if (out == nullptr || static_cast(out->size()) != array.length()) { + return Status::Invalid("output buffer size must match the array length"); + } + if (!MatchesArrowType(array)) { + return Status::Invalid(fmt::format("literal set of type {} cannot probe arrow {} type", + FieldTypeUtils::FieldTypeToString(field_type_), + array.type()->ToString())); + } + if (negate && has_null_literal_) { + // `NotIn::InnerTest` returns false as soon as it meets a null literal, so no row matches. + return Status::OK(); + } + switch (array.type_id()) { + case arrow::Type::type::BOOL: + TestBooleanArray(array, negate, out); + break; + case arrow::Type::type::INT8: + TestIntegerArray(array, negate, out); + break; + case arrow::Type::type::INT16: + TestIntegerArray(array, negate, out); + break; + case arrow::Type::type::INT32: + TestIntegerArray(array, negate, out); + break; + case arrow::Type::type::INT64: + TestIntegerArray(array, negate, out); + break; + case arrow::Type::type::DATE32: + TestIntegerArray(array, negate, out); + break; + case arrow::Type::type::STRING: + TestBinaryArray(array, negate, out); + break; + case arrow::Type::type::BINARY: + TestBinaryArray(array, negate, out); + break; + case arrow::Type::type::DICTIONARY: { + const auto& dict_array = checked_cast(array); + const auto* dict_type = + checked_cast(dict_array.type().get()); + if (dict_type->value_type()->id() == arrow::Type::type::STRING) { + TestDictionaryArray(dict_array, negate, out); + } else { + TestDictionaryArray(dict_array, negate, + out); + } + break; + } + default: + return Status::Invalid( + fmt::format("Not support literal set on arrow {} type", array.type()->ToString())); + } + return Status::OK(); +} + +Result LiteralSet::TestValue(const Literal& value, bool negate) const { + if (value.IsNull()) { + return false; + } + if (value.GetType() != field_type_) { + return Status::Invalid( + fmt::format("cannot probe literal set of type {} with value {} of type {}", + FieldTypeUtils::FieldTypeToString(field_type_), value.ToString(), + FieldTypeUtils::FieldTypeToString(value.GetType()))); + } + if (negate && has_null_literal_) { + return false; + } + bool contains = false; + switch (field_type_) { + case FieldType::BOOLEAN: + contains = value.GetValue() ? has_true_ : has_false_; + break; + case FieldType::TINYINT: + contains = ContainsInteger(value.GetValue()); + break; + case FieldType::SMALLINT: + contains = ContainsInteger(value.GetValue()); + break; + case FieldType::INT: + case FieldType::DATE: + contains = ContainsInteger(value.GetValue()); + break; + case FieldType::BIGINT: + contains = ContainsInteger(value.GetValue()); + break; + default: + contains = ContainsBinary(value.GetValue()); + break; + } + return contains != negate; +} +} // namespace paimon diff --git a/src/paimon/common/predicate/literal_set.h b/src/paimon/common/predicate/literal_set.h new file mode 100644 index 000000000..bad1fa6bb --- /dev/null +++ b/src/paimon/common/predicate/literal_set.h @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/defs.h" +#include "paimon/predicate/literal.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace arrow { +class Array; +class DictionaryArray; +} // namespace arrow + +namespace paimon { +/// An immutable, type specialized lookup structure built once from the literals of an `IN` / +/// `NOT IN` predicate. +/// +/// `MultiLiteralsLeafFunction` materializes the whole column into `Literal` objects (one heap +/// allocation per row) and then linearly scans the literals for every row, which is +/// `O(rows * literals)` with a heap allocation per row. `LiteralSet` is built once when the +/// predicate is constructed and probes an arrow array in `O(rows)` without allocating per row. +/// +/// The class only covers the types where a bitwise / byte-wise equality is exactly equivalent to +/// `Literal::CompareTo(...) == 0`. Everything else makes `CreateOrNull` return `nullptr` so that +/// the caller falls back to `In` / `NotIn`. +class LiteralSet { + public: + /// Builds a lookup structure for `literals` of a column typed `field_type`. + /// + /// @return `nullptr` when the combination is not supported, in which case the caller must fall + /// back to the generic `LeafFunction` implementation. This never fails. + static std::shared_ptr CreateOrNull(FieldType field_type, + const std::vector& literals); + + /// Checks whether `array` maps to the exact `FieldType` this set was built for. The mapping is + /// identical to `LiteralConverter::ConvertLiteralsFromArray`, so a `false` result means the + /// fallback path would have produced a type mismatch error (or would not support the array at + /// all) and must be taken to keep the error behavior unchanged. + bool MatchesArrowType(const arrow::Array& array) const; + + /// Probes every non-null row of `array` against this set. + /// + /// @param negate `false` for `IN` semantics, `true` for `NOT IN` semantics. + /// @param out Must be sized `array.length()` with all elements pre-set to 0. Only non-null rows + /// are written, so null rows stay 0 (`IN` and `NOT IN` are both false on null). + Status TestArray(const arrow::Array& array, bool negate, std::vector* out) const; + + /// Probes a single value against this set. `value` must not be of a different type than the one + /// this set was built for. + Result TestValue(const Literal& value, bool negate) const; + + private: + explicit LiteralSet(FieldType field_type) : field_type_(field_type) {} + + // Each builder returns false when the literals cannot be represented, which turns into a + // `nullptr` from `CreateOrNull`. + bool BuildIntegers(const std::vector& literals); + bool BuildBooleans(const std::vector& literals); + bool BuildBinaries(const std::vector& literals); + + bool ContainsInteger(int64_t value) const { + if (value < min_ || value > max_) { + return false; + } + if (!dense_.empty()) { + return dense_[static_cast(value - min_)]; + } + return sparse_.find(value) != sparse_.end(); + } + + bool ContainsBinary(std::string_view value) const { + if (value.size() < min_length_ || value.size() > max_length_) { + return false; + } + if (value.empty()) { + return has_empty_binary_; + } + const auto first_byte = static_cast(value[0]); + if ((first_byte_bitmap_[first_byte >> 6] & (1ULL << (first_byte & 63))) == 0) { + return false; + } + return binary_set_.find(value) != binary_set_.end(); + } + + void TestBooleanArray(const arrow::Array& array, bool negate, std::vector* out) const; + + template + void TestIntegerArray(const arrow::Array& array, bool negate, std::vector* out) const; + + template + void TestBinaryArray(const arrow::Array& array, bool negate, std::vector* out) const; + + template + void TestDictionaryArray(const arrow::DictionaryArray& dict_array, bool negate, + std::vector* out) const; + + FieldType field_type_; + // Whether the literals contain a null, which makes `NOT IN` false for every row. + bool has_null_literal_ = false; + + // Integer family (TINYINT, SMALLINT, INT, BIGINT, DATE), all widened to int64_t. `min_` / + // `max_` reject out of range values without touching the set. Exactly one of `dense_` / + // `sparse_` is populated: `dense_` indexes by `value - min_` and needs no hashing at all. + int64_t min_ = 0; + int64_t max_ = 0; + std::vector dense_; + std::unordered_set sparse_; + + // BOOLEAN. + bool has_true_ = false; + bool has_false_ = false; + + // STRING / BINARY. `binary_storage_` owns the bytes (a deque never invalidates references on + // growth) and `binary_set_` views into it. Lengths and the first byte bitmap reject + // non-candidates before hashing. + std::deque binary_storage_; + std::unordered_set binary_set_; + size_t min_length_ = 0; + size_t max_length_ = 0; + std::array first_byte_bitmap_ = {0, 0, 0, 0}; + bool has_empty_binary_ = false; +}; +} // namespace paimon diff --git a/src/paimon/common/predicate/literal_set_test.cpp b/src/paimon/common/predicate/literal_set_test.cpp new file mode 100644 index 000000000..138006119 --- /dev/null +++ b/src/paimon/common/predicate/literal_set_test.cpp @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/predicate/literal_set.h" + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/array/array_dict.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/data/decimal.h" +#include "paimon/data/timestamp.h" +#include "paimon/defs.h" +#include "paimon/status.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +class LiteralSetTest : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + static Literal StringLiteral(const std::string& value) { + return Literal(FieldType::STRING, value.data(), value.size()); + } + + static Literal BinaryLiteral(const std::string& value) { + return Literal(FieldType::BINARY, value.data(), value.size()); + } + + // Probes `array` and returns the per row result, asserting the whole call succeeded. + static std::vector Probe(const LiteralSet& literal_set, + const std::shared_ptr& array, bool negate) { + std::vector is_valid(array->length(), 0); + Status status = literal_set.TestArray(*array, negate, &is_valid); + EXPECT_TRUE(status.ok()) << status.ToString(); + return is_valid; + } +}; + +TEST_F(LiteralSetTest, TestDenseIntegers) { + // 1..4 out of a span of 5 stays within the dense threshold. + auto literal_set = + LiteralSet::CreateOrNull(FieldType::INT, {Literal(1), Literal(2), Literal(3), Literal(5)}); + ASSERT_TRUE(literal_set); + + auto array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([0, 1, 2, 3, 4, 5, 6, null])") + .ValueOrDie(); + ASSERT_TRUE(literal_set->MatchesArrowType(*array)); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/false), + std::vector({0, 1, 1, 1, 0, 1, 0, 0})); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/true), + std::vector({1, 0, 0, 0, 1, 0, 1, 0})); +} + +TEST_F(LiteralSetTest, TestSparseIntegers) { + // A span of 2000001 for 3 literals is far beyond the dense threshold. + auto literal_set = LiteralSet::CreateOrNull( + FieldType::BIGINT, + {Literal(int64_t{-1000000}), Literal(int64_t{0}), Literal(int64_t{1000000})}); + ASSERT_TRUE(literal_set); + + auto array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::int64(), R"([-1000000, -999999, 0, 1, 1000000, null])") + .ValueOrDie(); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/false), std::vector({1, 0, 1, 0, 1, 0})); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/true), std::vector({0, 1, 0, 1, 0, 0})); +} + +TEST_F(LiteralSetTest, TestTinyIntAndSmallInt) { + auto tinyint_set = LiteralSet::CreateOrNull( + FieldType::TINYINT, {Literal(int8_t{-128}), Literal(int8_t{0}), Literal(int8_t{127})}); + ASSERT_TRUE(tinyint_set); + auto tinyint_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int8(), R"([-128, -127, 0, 126, 127])") + .ValueOrDie(); + ASSERT_TRUE(tinyint_set->MatchesArrowType(*tinyint_array)); + ASSERT_EQ(Probe(*tinyint_set, tinyint_array, /*negate=*/false), + std::vector({1, 0, 1, 0, 1})); + + auto smallint_set = LiteralSet::CreateOrNull( + FieldType::SMALLINT, {Literal(int16_t{-30000}), Literal(int16_t{30000})}); + ASSERT_TRUE(smallint_set); + auto smallint_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int16(), R"([-30000, 0, 30000, null])") + .ValueOrDie(); + ASSERT_TRUE(smallint_set->MatchesArrowType(*smallint_array)); + ASSERT_EQ(Probe(*smallint_set, smallint_array, /*negate=*/false), + std::vector({1, 0, 1, 0})); +} + +TEST_F(LiteralSetTest, TestDate) { + auto literal_set = LiteralSet::CreateOrNull( + FieldType::DATE, {Literal(FieldType::DATE, 100), Literal(FieldType::DATE, 20000)}); + ASSERT_TRUE(literal_set); + + auto array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::date32(), R"([100, 101, 20000, null])") + .ValueOrDie(); + ASSERT_TRUE(literal_set->MatchesArrowType(*array)); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/false), std::vector({1, 0, 1, 0})); + + ASSERT_OK_AND_ASSIGN(bool hit, + literal_set->TestValue(Literal(FieldType::DATE, 100), /*negate=*/false)); + ASSERT_TRUE(hit); + ASSERT_OK_AND_ASSIGN(bool miss, + literal_set->TestValue(Literal(FieldType::DATE, 101), /*negate=*/false)); + ASSERT_FALSE(miss); +} + +TEST_F(LiteralSetTest, TestBoolean) { + auto true_only = LiteralSet::CreateOrNull(FieldType::BOOLEAN, {Literal(true)}); + ASSERT_TRUE(true_only); + auto array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::boolean(), R"([true, false, null])") + .ValueOrDie(); + ASSERT_TRUE(true_only->MatchesArrowType(*array)); + ASSERT_EQ(Probe(*true_only, array, /*negate=*/false), std::vector({1, 0, 0})); + ASSERT_EQ(Probe(*true_only, array, /*negate=*/true), std::vector({0, 1, 0})); + + auto both = LiteralSet::CreateOrNull(FieldType::BOOLEAN, {Literal(true), Literal(false)}); + ASSERT_TRUE(both); + ASSERT_EQ(Probe(*both, array, /*negate=*/false), std::vector({1, 1, 0})); + ASSERT_EQ(Probe(*both, array, /*negate=*/true), std::vector({0, 0, 0})); +} + +TEST_F(LiteralSetTest, TestString) { + auto literal_set = LiteralSet::CreateOrNull( + FieldType::STRING, {StringLiteral("apple"), StringLiteral(""), StringLiteral("banana")}); + ASSERT_TRUE(literal_set); + + auto array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::utf8(), R"(["apple", "", "banana", "cherry", "app", null])") + .ValueOrDie(); + ASSERT_TRUE(literal_set->MatchesArrowType(*array)); + // "cherry" is rejected by the first byte bitmap, "app" by the length range. + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/false), std::vector({1, 1, 1, 0, 0, 0})); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/true), std::vector({0, 0, 0, 1, 1, 0})); + + ASSERT_OK_AND_ASSIGN(bool hit, literal_set->TestValue(StringLiteral("banana"), + /*negate=*/false)); + ASSERT_TRUE(hit); + ASSERT_OK_AND_ASSIGN(bool empty_hit, literal_set->TestValue(StringLiteral(""), + /*negate=*/false)); + ASSERT_TRUE(empty_hit); + ASSERT_OK_AND_ASSIGN(bool miss, literal_set->TestValue(StringLiteral("apples"), + /*negate=*/false)); + ASSERT_FALSE(miss); +} + +TEST_F(LiteralSetTest, TestStringWithoutEmptyLiteral) { + // An empty column value must not match when no empty literal was given, even though it passes + // the first byte bitmap trivially. + auto literal_set = LiteralSet::CreateOrNull(FieldType::STRING, {StringLiteral("a")}); + ASSERT_TRUE(literal_set); + auto array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["a", "", "b"])").ValueOrDie(); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/false), std::vector({1, 0, 0})); +} + +TEST_F(LiteralSetTest, TestBinary) { + auto literal_set = LiteralSet::CreateOrNull( + FieldType::BINARY, {BinaryLiteral(std::string("\x00\x01", 2)), BinaryLiteral("xyz")}); + ASSERT_TRUE(literal_set); + + arrow::BinaryBuilder builder; + ASSERT_TRUE(builder.Append(std::string("\x00\x01", 2)).ok()); + ASSERT_TRUE(builder.Append("xyz").ok()); + ASSERT_TRUE(builder.Append("xyw").ok()); + ASSERT_TRUE(builder.AppendNull().ok()); + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + + ASSERT_TRUE(literal_set->MatchesArrowType(*array)); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/false), std::vector({1, 1, 0, 0})); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/true), std::vector({0, 0, 1, 0})); +} + +TEST_F(LiteralSetTest, TestDictionaryString) { + auto literal_set = + LiteralSet::CreateOrNull(FieldType::STRING, {StringLiteral("a"), StringLiteral("c")}); + ASSERT_TRUE(literal_set); + + auto dictionary = + arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["a", "b", "c"])").ValueOrDie(); + auto indices = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([0, 1, 2, 2, null, 0])") + .ValueOrDie(); + auto dict_type = arrow::dictionary(arrow::int32(), arrow::utf8()); + auto array = arrow::DictionaryArray::FromArrays(dict_type, indices, dictionary).ValueOrDie(); + + ASSERT_TRUE(literal_set->MatchesArrowType(*array)); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/false), std::vector({1, 0, 1, 1, 0, 1})); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/true), std::vector({0, 1, 0, 0, 0, 0})); +} + +TEST_F(LiteralSetTest, TestNullLiteralIgnoredForIn) { + auto literal_set = LiteralSet::CreateOrNull( + FieldType::BIGINT, {Literal(int64_t{1}), Literal(FieldType::BIGINT), Literal(int64_t{3})}); + ASSERT_TRUE(literal_set); + + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::int64(), R"([1, 2, 3, null])") + .ValueOrDie(); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/false), std::vector({1, 0, 1, 0})); + // A null literal makes NOT IN false for every row. + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/true), std::vector({0, 0, 0, 0})); + + ASSERT_OK_AND_ASSIGN(bool not_in, literal_set->TestValue(Literal(int64_t{2}), + /*negate=*/true)); + ASSERT_FALSE(not_in); +} + +TEST_F(LiteralSetTest, TestOnlyNullLiterals) { + auto int_set = LiteralSet::CreateOrNull(FieldType::BIGINT, {Literal(FieldType::BIGINT)}); + ASSERT_TRUE(int_set); + auto int_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::int64(), R"([-9007199254740993, 0, 9007199254740993, null])") + .ValueOrDie(); + ASSERT_EQ(Probe(*int_set, int_array, /*negate=*/false), std::vector({0, 0, 0, 0})); + ASSERT_EQ(Probe(*int_set, int_array, /*negate=*/true), std::vector({0, 0, 0, 0})); + + auto string_set = LiteralSet::CreateOrNull(FieldType::STRING, {Literal(FieldType::STRING)}); + ASSERT_TRUE(string_set); + auto string_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["", "a", null])").ValueOrDie(); + ASSERT_EQ(Probe(*string_set, string_array, /*negate=*/false), std::vector({0, 0, 0})); +} + +TEST_F(LiteralSetTest, TestCreateOrNullUnsupported) { + // Empty literals. + ASSERT_FALSE(LiteralSet::CreateOrNull(FieldType::INT, {})); + // Literal type differs from the field type, `Literal::CompareTo` must keep reporting the error. + ASSERT_FALSE(LiteralSet::CreateOrNull(FieldType::BIGINT, {Literal(1)})); + ASSERT_FALSE(LiteralSet::CreateOrNull(FieldType::STRING, {Literal(int64_t{1})})); + // Types whose equality is not byte-wise. + ASSERT_FALSE(LiteralSet::CreateOrNull(FieldType::DOUBLE, {Literal(1.0)})); + ASSERT_FALSE(LiteralSet::CreateOrNull(FieldType::FLOAT, {Literal(1.0f)})); + ASSERT_FALSE(LiteralSet::CreateOrNull( + FieldType::DECIMAL, + {Literal(Decimal::FromUnscaledLong(/*unscaled_long=*/10, /*precision=*/10, /*scale=*/1))})); + ASSERT_FALSE( + LiteralSet::CreateOrNull(FieldType::TIMESTAMP, {Literal(Timestamp::FromEpochMillis(1))})); +} + +TEST_F(LiteralSetTest, TestMatchesArrowTypeMismatch) { + auto literal_set = LiteralSet::CreateOrNull(FieldType::BIGINT, {Literal(int64_t{1})}); + ASSERT_TRUE(literal_set); + + auto int32_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([1, 2])").ValueOrDie(); + ASSERT_FALSE(literal_set->MatchesArrowType(*int32_array)); + std::vector is_valid(int32_array->length(), 0); + ASSERT_FALSE(literal_set->TestArray(*int32_array, /*negate=*/false, &is_valid).ok()); + + auto double_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::float64(), R"([1.0])").ValueOrDie(); + ASSERT_FALSE(literal_set->MatchesArrowType(*double_array)); + + // Unsupported dictionary layout falls back as well. + auto dictionary = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int64(), R"([1, 2])").ValueOrDie(); + auto indices = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([0, 1])").ValueOrDie(); + auto dict_array = arrow::DictionaryArray::FromArrays( + arrow::dictionary(arrow::int32(), arrow::int64()), indices, dictionary) + .ValueOrDie(); + ASSERT_FALSE(literal_set->MatchesArrowType(*dict_array)); + + // A value of a different type must not be silently probed either. + ASSERT_FALSE(literal_set->TestValue(Literal(1), /*negate=*/false).ok()); +} + +TEST_F(LiteralSetTest, TestValueNull) { + auto literal_set = + LiteralSet::CreateOrNull(FieldType::BIGINT, {Literal(int64_t{1}), Literal(int64_t{3})}); + ASSERT_TRUE(literal_set); + ASSERT_OK_AND_ASSIGN(bool in, literal_set->TestValue(Literal(FieldType::BIGINT), + /*negate=*/false)); + ASSERT_FALSE(in); + ASSERT_OK_AND_ASSIGN(bool not_in, literal_set->TestValue(Literal(FieldType::BIGINT), + /*negate=*/true)); + ASSERT_FALSE(not_in); +} + +TEST_F(LiteralSetTest, TestSlicedArray) { + auto literal_set = LiteralSet::CreateOrNull(FieldType::INT, {Literal(2), Literal(4)}); + ASSERT_TRUE(literal_set); + auto array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([1, 2, 3, 4, 5, null])") + .ValueOrDie(); + auto sliced = array->Slice(2, 4); + ASSERT_EQ(Probe(*literal_set, sliced, /*negate=*/false), std::vector({0, 1, 0, 0})); + + auto string_set = LiteralSet::CreateOrNull(FieldType::STRING, {StringLiteral("c")}); + ASSERT_TRUE(string_set); + auto string_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["a", "b", "c", "d"])") + .ValueOrDie(); + auto sliced_string = string_array->Slice(1, 3); + ASSERT_EQ(Probe(*string_set, sliced_string, /*negate=*/false), std::vector({0, 1, 0})); +} + +TEST_F(LiteralSetTest, TestOutputBufferValidation) { + auto literal_set = LiteralSet::CreateOrNull(FieldType::INT, {Literal(1)}); + ASSERT_TRUE(literal_set); + auto array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([1, 2])").ValueOrDie(); + ASSERT_FALSE(literal_set->TestArray(*array, /*negate=*/false, nullptr).ok()); + std::vector too_small(1, 0); + ASSERT_FALSE(literal_set->TestArray(*array, /*negate=*/false, &too_small).ok()); +} +} // namespace paimon::test diff --git a/src/paimon/common/predicate/predicate_test.cpp b/src/paimon/common/predicate/predicate_test.cpp index fa4733cba..88b5cb3fc 100644 --- a/src/paimon/common/predicate/predicate_test.cpp +++ b/src/paimon/common/predicate/predicate_test.cpp @@ -29,10 +29,12 @@ #include "arrow/api.h" #include "arrow/array/array_nested.h" #include "arrow/ipc/json_simple.h" +#include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/common/data/binary_array.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/predicate/leaf_predicate_impl.h" #include "paimon/common/predicate/predicate_filter.h" #include "paimon/defs.h" #include "paimon/memory/memory_pool.h" @@ -938,6 +940,67 @@ TEST_F(PredicateTest, TestLargeNotInNull) { ASSERT_FALSE(StatsCheck(*predicate, 3ll, {FieldStats(29ll, 32ll, 0ll)})); } +TEST_F(PredicateTest, TestLargeStringIn) { + auto string_type = arrow::utf8(); + std::vector literals; + literals.reserve(1000); + for (int32_t i = 0; i < 1000; i++) { + std::string value = fmt::format("key-{}", i); + literals.emplace_back(FieldType::STRING, value.data(), value.size()); + } + auto in_base = + PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, literals); + auto in_predicate = std::dynamic_pointer_cast(in_base); + ASSERT_TRUE(in_predicate); + auto not_in_base = PredicateBuilder::NotIn(/*field_index=*/0, /*field_name=*/"f0", + FieldType::STRING, literals); + auto not_in_predicate = std::dynamic_pointer_cast(not_in_base); + ASSERT_TRUE(not_in_predicate); + + auto f0 = arrow::ipc::internal::json::ArrayFromJSON( + string_type, R"(["key-0", "key-999", "key-1000", "other", "", null])") + .ValueOrDie(); + std::shared_ptr src_type = arrow::struct_({arrow::field("f0", string_type)}); + std::shared_ptr struct_array = + arrow::StructArray::Make({f0}, src_type->fields()).ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(auto in_valid, in_predicate->Test(*struct_array)); + ASSERT_EQ(in_valid, std::vector({1, 1, 0, 0, 0, 0})); + ASSERT_OK_AND_ASSIGN(auto not_in_valid, not_in_predicate->Test(*struct_array)); + ASSERT_EQ(not_in_valid, std::vector({0, 0, 1, 1, 1, 0})); +} + +TEST_F(PredicateTest, TestInAfterRebind) { + auto bigint_type = arrow::int64(); + auto predicate_base = PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f0", + FieldType::BIGINT, {Literal(1l), Literal(3l)}); + auto leaf_predicate = std::dynamic_pointer_cast(predicate_base); + ASSERT_TRUE(leaf_predicate); + + // Rebinding shares the prebuilt lookup structure, results must stay identical. + auto renamed = leaf_predicate->NewLeafPredicate(/*new_field_name=*/"f1"); + ASSERT_EQ(renamed->FieldName(), "f1"); + auto rebound = renamed->NewLeafPredicate(/*new_field_index=*/1); + ASSERT_EQ(rebound->FieldIndex(), 1); + + auto f0 = + arrow::ipc::internal::json::ArrayFromJSON(bigint_type, R"([3, 2, 1, 0])").ValueOrDie(); + auto f1 = + arrow::ipc::internal::json::ArrayFromJSON(bigint_type, R"([1, 2, 3, null])").ValueOrDie(); + std::shared_ptr src_type = + arrow::struct_({arrow::field("f0", bigint_type), arrow::field("f1", bigint_type)}); + std::shared_ptr struct_array = + arrow::StructArray::Make({f0, f1}, src_type->fields()).ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(auto is_valid, rebound->Test(*struct_array)); + ASSERT_EQ(is_valid, std::vector({1, 0, 1, 0})); + + auto arrow_schema = arrow::schema( + arrow::FieldVector({arrow::field("f0", bigint_type), arrow::field("f1", bigint_type)})); + ASSERT_TRUE(rebound->Test(arrow_schema, CreateBigIntRow({0, 1})).value()); + ASSERT_FALSE(rebound->Test(arrow_schema, CreateBigIntRow({1, 2})).value()); +} + TEST_F(PredicateTest, TestAnd) { auto bigint_type = arrow::int64(); ASSERT_OK_AND_ASSIGN( From f9f09f5034fdd9961f113717ab763f157cb5ee6a Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 31 Aug 2026 14:17:28 +0800 Subject: [PATCH 2/2] fix --- src/paimon/common/predicate/literal_set.cpp | 18 ++++++-- src/paimon/common/predicate/literal_set.h | 3 ++ .../common/predicate/literal_set_test.cpp | 46 +++++++++++++++++++ .../common/predicate/predicate_test.cpp | 29 ++++++++++++ 4 files changed, 91 insertions(+), 5 deletions(-) diff --git a/src/paimon/common/predicate/literal_set.cpp b/src/paimon/common/predicate/literal_set.cpp index aea2fb408..a151e5f01 100644 --- a/src/paimon/common/predicate/literal_set.cpp +++ b/src/paimon/common/predicate/literal_set.cpp @@ -169,7 +169,13 @@ bool LiteralSet::BuildIntegers(const std::vector& literals) { const auto [min_it, max_it] = std::minmax_element(values.begin(), values.end()); min_ = *min_it; max_ = *max_it; - const uint64_t span = static_cast(max_) - static_cast(min_) + 1; + // For the full int64 range `max_ - min_ + 1` wraps the unsigned span to zero, which would + // create an empty dense bitmap and index out of bounds. Such a span always exceeds the dense + // threshold, so force the sparse path whenever the subtraction overflows int64_t. + const bool span_overflows = min_ < 0 && max_ > std::numeric_limits::max() + min_; + const uint64_t span = span_overflows + ? MAX_DENSE_SPAN + 1 + : static_cast(max_) - static_cast(min_) + 1; if (span <= MAX_DENSE_SPAN && span <= DENSE_SPAN_FACTOR * values.size()) { dense_.assign(static_cast(span), false); for (int64_t value : values) { @@ -269,12 +275,14 @@ void LiteralSet::TestDictionaryArray(const arrow::DictionaryArray& dict_array, b const auto& dictionary = checked_cast(*dict_array.dictionary()); const auto& indices = checked_cast(*dict_array.indices()); const int64_t dict_length = dictionary.length(); - // Probe the dictionary once and then only follow indices, O(dict_size + rows). + // Probe the dictionary once and then only follow indices, O(dict_size + rows). A null + // dictionary value reads back as an empty string through `GetLiteralFromDictionaryArray` + // (its offsets are equal), so probe it as an empty value to keep the semantics identical. std::vector dict_hits(dict_length, 0); for (int64_t i = 0; i < dict_length; i++) { - if (!dictionary.IsNull(i)) { - dict_hits[i] = static_cast(ContainsBinary(dictionary.GetView(i))); - } + const std::string_view value = + dictionary.IsNull(i) ? std::string_view() : dictionary.GetView(i); + dict_hits[i] = static_cast(ContainsBinary(value)); } for (int64_t i = 0; i < dict_array.length(); i++) { if (dict_array.IsNull(i)) { diff --git a/src/paimon/common/predicate/literal_set.h b/src/paimon/common/predicate/literal_set.h index bad1fa6bb..3b2552c26 100644 --- a/src/paimon/common/predicate/literal_set.h +++ b/src/paimon/common/predicate/literal_set.h @@ -78,6 +78,9 @@ class LiteralSet { private: explicit LiteralSet(FieldType field_type) : field_type_(field_type) {} + // `binary_set_` views into `binary_storage_`, so copying would leave dangling views. + LiteralSet(const LiteralSet&) = delete; + LiteralSet& operator=(const LiteralSet&) = delete; // Each builder returns false when the literals cannot be represented, which turns into a // `nullptr` from `CreateOrNull`. diff --git a/src/paimon/common/predicate/literal_set_test.cpp b/src/paimon/common/predicate/literal_set_test.cpp index 138006119..c2808ad1c 100644 --- a/src/paimon/common/predicate/literal_set_test.cpp +++ b/src/paimon/common/predicate/literal_set_test.cpp @@ -19,6 +19,7 @@ #include "paimon/common/predicate/literal_set.h" #include +#include #include #include #include @@ -87,6 +88,29 @@ TEST_F(LiteralSetTest, TestSparseIntegers) { ASSERT_EQ(Probe(*literal_set, array, /*negate=*/true), std::vector({0, 1, 0, 1, 0, 0})); } +TEST_F(LiteralSetTest, TestInt64FullSpan) { + // The full int64 range used to wrap the dense span to zero and index out of bounds. + auto literal_set = + LiteralSet::CreateOrNull(FieldType::BIGINT, {Literal(std::numeric_limits::min()), + Literal(std::numeric_limits::max())}); + ASSERT_TRUE(literal_set); + + auto array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::int64(), R"([-9223372036854775808, 0, 9223372036854775807, null])") + .ValueOrDie(); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/false), std::vector({1, 0, 1, 0})); + ASSERT_EQ(Probe(*literal_set, array, /*negate=*/true), std::vector({0, 1, 0, 0})); + + ASSERT_OK_AND_ASSIGN(bool min_hit, + literal_set->TestValue(Literal(std::numeric_limits::min()), + /*negate=*/false)); + ASSERT_OK_AND_ASSIGN(bool max_hit, + literal_set->TestValue(Literal(std::numeric_limits::max()), + /*negate=*/false)); + ASSERT_TRUE(min_hit); + ASSERT_TRUE(max_hit); +} + TEST_F(LiteralSetTest, TestTinyIntAndSmallInt) { auto tinyint_set = LiteralSet::CreateOrNull( FieldType::TINYINT, {Literal(int8_t{-128}), Literal(int8_t{0}), Literal(int8_t{127})}); @@ -214,6 +238,28 @@ TEST_F(LiteralSetTest, TestDictionaryString) { ASSERT_EQ(Probe(*literal_set, array, /*negate=*/true), std::vector({0, 1, 0, 0, 0, 0})); } +TEST_F(LiteralSetTest, TestDictionaryWithNullValue) { + // A null dictionary value reads back as an empty string on the fallback path, so rows that + // reference it must probe against the empty literal. + auto dictionary = + arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"([null, "a"])").ValueOrDie(); + auto indices = arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([0, 1, null, 0])") + .ValueOrDie(); + auto dict_type = arrow::dictionary(arrow::int32(), arrow::utf8()); + auto array = arrow::DictionaryArray::FromArrays(dict_type, indices, dictionary).ValueOrDie(); + + auto set_with_empty = + LiteralSet::CreateOrNull(FieldType::STRING, {StringLiteral("a"), StringLiteral("")}); + ASSERT_TRUE(set_with_empty); + ASSERT_EQ(Probe(*set_with_empty, array, /*negate=*/false), std::vector({1, 1, 0, 1})); + ASSERT_EQ(Probe(*set_with_empty, array, /*negate=*/true), std::vector({0, 0, 0, 0})); + + auto set_without_empty = LiteralSet::CreateOrNull(FieldType::STRING, {StringLiteral("a")}); + ASSERT_TRUE(set_without_empty); + ASSERT_EQ(Probe(*set_without_empty, array, /*negate=*/false), std::vector({0, 1, 0, 0})); + ASSERT_EQ(Probe(*set_without_empty, array, /*negate=*/true), std::vector({1, 0, 0, 1})); +} + TEST_F(LiteralSetTest, TestNullLiteralIgnoredForIn) { auto literal_set = LiteralSet::CreateOrNull( FieldType::BIGINT, {Literal(int64_t{1}), Literal(FieldType::BIGINT), Literal(int64_t{3})}); diff --git a/src/paimon/common/predicate/predicate_test.cpp b/src/paimon/common/predicate/predicate_test.cpp index 88b5cb3fc..d8b9ccaac 100644 --- a/src/paimon/common/predicate/predicate_test.cpp +++ b/src/paimon/common/predicate/predicate_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -1001,6 +1002,34 @@ TEST_F(PredicateTest, TestInAfterRebind) { ASSERT_FALSE(rebound->Test(arrow_schema, CreateBigIntRow({1, 2})).value()); } +TEST_F(PredicateTest, TestInt64BoundaryIn) { + // Building the lookup for the full int64 range used to crash with an out of bounds dense + // bitmap index; construction itself is part of what this test guards. + std::vector literals = {Literal(std::numeric_limits::min()), + Literal(std::numeric_limits::max())}; + auto in_base = + PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, literals); + auto in_predicate = std::dynamic_pointer_cast(in_base); + ASSERT_TRUE(in_predicate); + auto not_in_base = PredicateBuilder::NotIn(/*field_index=*/0, /*field_name=*/"f0", + FieldType::BIGINT, literals); + auto not_in_predicate = std::dynamic_pointer_cast(not_in_base); + ASSERT_TRUE(not_in_predicate); + + auto f0 = arrow::ipc::internal::json::ArrayFromJSON( + arrow::int64(), R"([-9223372036854775808, 0, 9223372036854775807, null])") + .ValueOrDie(); + std::shared_ptr src_type = + arrow::struct_({arrow::field("f0", arrow::int64())}); + std::shared_ptr struct_array = + arrow::StructArray::Make({f0}, src_type->fields()).ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(auto in_valid, in_predicate->Test(*struct_array)); + ASSERT_EQ(in_valid, std::vector({1, 0, 1, 0})); + ASSERT_OK_AND_ASSIGN(auto not_in_valid, not_in_predicate->Test(*struct_array)); + ASSERT_EQ(not_in_valid, std::vector({0, 1, 0, 0})); +} + TEST_F(PredicateTest, TestAnd) { auto bigint_type = arrow::int64(); ASSERT_OK_AND_ASSIGN(