From a920b43c8a8c91d09f5538da4f8da0baed223055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 9 Sep 2026 10:55:20 +0200 Subject: [PATCH 1/5] GH-51245: [C++][Gandiva] Add support for LLVM 23.1.0 --- cpp/CMakeLists.txt | 1 + cpp/src/gandiva/engine.cc | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index fd5eaf77f23b..9c81360a9571 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -179,6 +179,7 @@ set(ARROW_DOC_DIR "${CMAKE_INSTALL_DOCDIR}") set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") set(ARROW_LLVM_VERSIONS + "23.1" "22.1" "21.1" "20.1" diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index e6edb0042ce5..015de585282d 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -207,7 +207,12 @@ Status UseJITLinkIfEnabled(llvm::orc::LLJITBuilder& jit_builder) { static auto maybe_use_jit_link = ::arrow::internal::GetEnvVar("GANDIVA_USE_JIT_LINK"); if (maybe_use_jit_link.ok()) { ARROW_ASSIGN_OR_RAISE(static auto memory_manager, CreateMemmoryManager()); -# if LLVM_VERSION_MAJOR >= 21 +# if LLVM_VERSION_MAJOR >= 23 + jit_builder.setObjectLinkingLayerCreator( + [&](llvm::orc::ExecutionSession& ES, llvm::jitlink::JITLinkMemoryManager&) { + return std::make_unique(ES, *memory_manager); + }); +# elif LLVM_VERSION_MAJOR >= 21 jit_builder.setObjectLinkingLayerCreator([&](llvm::orc::ExecutionSession& ES) { return std::make_unique(ES, *memory_manager); }); From 5e73a384286c16b75458d1121413242e275ffe9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 9 Sep 2026 10:58:29 +0200 Subject: [PATCH 2/5] Minor commit to trigger rebuild --- ci/docker/conda-cpp.dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile index ef24197dfd3b..9cbef4bbc082 100644 --- a/ci/docker/conda-cpp.dockerfile +++ b/ci/docker/conda-cpp.dockerfile @@ -28,6 +28,7 @@ RUN /arrow/ci/scripts/install_minio.sh latest /opt/conda COPY ci/conda_env_cpp.txt \ ci/conda_env_gandiva.txt \ /arrow/ci/ +RUN echo "Force rebuild" RUN mamba install -q -y \ --file arrow/ci/conda_env_cpp.txt \ --file arrow/ci/conda_env_gandiva.txt \ From c25e5c4f1534672e34406a8f1630bc9ff2d9f0cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 9 Sep 2026 11:53:12 +0200 Subject: [PATCH 3/5] Unpin llvm<23 --- ci/conda_env_gandiva.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/conda_env_gandiva.txt b/ci/conda_env_gandiva.txt index 7127dc905586..edc49b692b84 100644 --- a/ci/conda_env_gandiva.txt +++ b/ci/conda_env_gandiva.txt @@ -16,4 +16,4 @@ # under the License. clang -llvmdev<23 +llvmdev From 10e83b4a140b85da4ce82986d92810b81df2a9cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 9 Sep 2026 12:22:36 +0200 Subject: [PATCH 4/5] Throw away commit for some debugging --- cpp/src/gandiva/tests/projector_test.cc | 71 +++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index 6e9450e1c650..5e20193c4b5d 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -2091,7 +2091,11 @@ TEST_F(TestProjector, TestCastVarbinaryFunction) { EXPECT_ARROW_ARRAY_EQUALS(out_float8, outputs.at(3)); } -TEST_F(TestProjector, TestToDate) { +// GH-51245: the body is shared so the same projection can be run under +// different Configurations, to tell an LLVM 23 optimizer/codegen regression +// apart from a plain to_date bug. +static void CheckToDate(const std::shared_ptr& config, + arrow::MemoryPool* pool) { // schema for input fields auto field0 = field("f0", arrow::utf8()); auto field_node = std::make_shared(field0); @@ -2110,7 +2114,7 @@ TEST_F(TestProjector, TestToDate) { // Build a projector for the expressions. std::shared_ptr projector; - auto status = Projector::Make(schema, {expr}, TestConfiguration(), &projector); + auto status = Projector::Make(schema, {expr}, config, &projector); EXPECT_TRUE(status.ok()); // Create a row-batch with some sample data @@ -2125,13 +2129,74 @@ TEST_F(TestProjector, TestToDate) { // Evaluate expression arrow::ArrayVector outputs; - status = projector->Evaluate(*in_batch, pool_, &outputs); + status = projector->Evaluate(*in_batch, pool, &outputs); EXPECT_TRUE(status.ok()); // Validate results EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0)); } +TEST_F(TestProjector, TestToDate) { CheckToDate(TestConfiguration(), pool_); } + +// If this passes while TestToDate fails, the regression is in the O3 pipeline +// rather than in the base IR that LLVMGenerator emits. +TEST_F(TestProjector, TestToDateNoOptimize) { + CheckToDate(ConfigurationBuilder().build(/*optimize=*/false), pool_); +} + +// If this passes while TestToDate fails, the regression is host-CPU specific +// (CI detects znver3 and enables AVX2/AVX512), i.e. most likely vectorization. +TEST_F(TestProjector, TestToDateNoHostCpu) { + auto config = ConfigurationBuilder().build(); + config->target_host_cpu(false); + CheckToDate(config, pool_); +} + +// Does the wrong result follow the value or the row index? Both inputs are +// 10 chars in the same format, so nothing distinguishes them except position. +TEST_F(TestProjector, TestToDateRowOrder) { + auto field0 = field("f0", arrow::utf8()); + auto field_node = std::make_shared(field0); + auto schema = arrow::schema({field0}); + auto field_result = field("res", arrow::date64()); + + auto pattern_node = std::make_shared( + arrow::utf8(), LiteralHolder(std::string("YYYY-MM-DD")), false); + auto fn_node = TreeExprBuilder::MakeFunction("to_date", {field_node, pattern_node}, + arrow::date64()); + auto expr = TreeExprBuilder::MakeExpression(fn_node, field_result); + + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, TestConfiguration(), &projector)); + + // A single record holding only the value that comes back null in the 3-row + // batch. Failing here means row 0 is mishandled regardless of batch size. + { + auto array0 = MakeArrowArrayUtf8({"1986-12-01"}, {true}); + auto exp = MakeArrowArrayDate64({533779200000}, {true}); + auto in_batch = arrow::RecordBatch::Make(schema, 1, {array0}); + + arrow::ArrayVector outputs; + ASSERT_OK(projector->Evaluate(*in_batch, pool_, &outputs)); + EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0)); + } + + // The original batch with the two valid dates swapped. If the null moves to + // 2012-12-01 the bug tracks the row index; if it stays on 1986-12-01 it + // tracks the value and the parser is back in scope. + { + auto array0 = + MakeArrowArrayUtf8({"2012-12-01", "1986-12-01", "invalid"}, {true, true, false}); + auto exp = + MakeArrowArrayDate64({1354320000000, 533779200000, 0}, {true, true, false}); + auto in_batch = arrow::RecordBatch::Make(schema, 3, {array0}); + + arrow::ArrayVector outputs; + ASSERT_OK(projector->Evaluate(*in_batch, pool_, &outputs)); + EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0)); + } +} + // ARROW-11617 TEST_F(TestProjector, TestIfElseOpt) { // schema for input From e32cc10f20ce8188d1afd812fbd2d379a7691137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 9 Sep 2026 12:47:56 +0200 Subject: [PATCH 5/5] Another throw away commit for diagnosing --- cpp/src/gandiva/engine.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index 015de585282d..6de0ee4d150a 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -508,8 +508,10 @@ static void OptimizeModuleWithNewPassManager(llvm::Module& module, function_pm.addPass(llvm::GVNPass()); function_pm.addPass(llvm::NewGVNPass()); function_pm.addPass(llvm::SimplifyCFGPass()); - function_pm.addPass(llvm::LoopVectorizePass()); - function_pm.addPass(llvm::SLPVectorizerPass()); + // GH-51245 probe: do the gandiva-injected vectorizers cause the LLVM 23 + // to_date miscompile? + // function_pm.addPass(llvm::LoopVectorizePass()); + // function_pm.addPass(llvm::SLPVectorizerPass()); module_pm.addPass(llvm::createModuleToFunctionPassAdaptor(std::move(function_pm))); module_pm.addPass(llvm::GlobalOptPass());