diff --git a/ci/conda_env_gandiva.txt b/ci/conda_env_gandiva.txt index 7127dc905586..edc49b692b84 100644 --- a/ci/conda_env_gandiva.txt +++ b/ci/conda_env_gandiva.txt @@ -16,4 +16,4 @@ # under the License. clang -llvmdev<23 +llvmdev diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile index ef24197dfd3b..9cbef4bbc082 100644 --- a/ci/docker/conda-cpp.dockerfile +++ b/ci/docker/conda-cpp.dockerfile @@ -28,6 +28,7 @@ RUN /arrow/ci/scripts/install_minio.sh latest /opt/conda COPY ci/conda_env_cpp.txt \ ci/conda_env_gandiva.txt \ /arrow/ci/ +RUN echo "Force rebuild" RUN mamba install -q -y \ --file arrow/ci/conda_env_cpp.txt \ --file arrow/ci/conda_env_gandiva.txt \ diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index fd5eaf77f23b..9c81360a9571 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -179,6 +179,7 @@ set(ARROW_DOC_DIR "${CMAKE_INSTALL_DOCDIR}") set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") set(ARROW_LLVM_VERSIONS + "23.1" "22.1" "21.1" "20.1" diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index e6edb0042ce5..6de0ee4d150a 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -207,7 +207,12 @@ Status UseJITLinkIfEnabled(llvm::orc::LLJITBuilder& jit_builder) { static auto maybe_use_jit_link = ::arrow::internal::GetEnvVar("GANDIVA_USE_JIT_LINK"); if (maybe_use_jit_link.ok()) { ARROW_ASSIGN_OR_RAISE(static auto memory_manager, CreateMemmoryManager()); -# if LLVM_VERSION_MAJOR >= 21 +# if LLVM_VERSION_MAJOR >= 23 + jit_builder.setObjectLinkingLayerCreator( + [&](llvm::orc::ExecutionSession& ES, llvm::jitlink::JITLinkMemoryManager&) { + return std::make_unique(ES, *memory_manager); + }); +# elif LLVM_VERSION_MAJOR >= 21 jit_builder.setObjectLinkingLayerCreator([&](llvm::orc::ExecutionSession& ES) { return std::make_unique(ES, *memory_manager); }); @@ -503,8 +508,10 @@ static void OptimizeModuleWithNewPassManager(llvm::Module& module, function_pm.addPass(llvm::GVNPass()); function_pm.addPass(llvm::NewGVNPass()); function_pm.addPass(llvm::SimplifyCFGPass()); - function_pm.addPass(llvm::LoopVectorizePass()); - function_pm.addPass(llvm::SLPVectorizerPass()); + // GH-51245 probe: do the gandiva-injected vectorizers cause the LLVM 23 + // to_date miscompile? + // function_pm.addPass(llvm::LoopVectorizePass()); + // function_pm.addPass(llvm::SLPVectorizerPass()); module_pm.addPass(llvm::createModuleToFunctionPassAdaptor(std::move(function_pm))); module_pm.addPass(llvm::GlobalOptPass()); diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index 6e9450e1c650..5e20193c4b5d 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -2091,7 +2091,11 @@ TEST_F(TestProjector, TestCastVarbinaryFunction) { EXPECT_ARROW_ARRAY_EQUALS(out_float8, outputs.at(3)); } -TEST_F(TestProjector, TestToDate) { +// GH-51245: the body is shared so the same projection can be run under +// different Configurations, to tell an LLVM 23 optimizer/codegen regression +// apart from a plain to_date bug. +static void CheckToDate(const std::shared_ptr& config, + arrow::MemoryPool* pool) { // schema for input fields auto field0 = field("f0", arrow::utf8()); auto field_node = std::make_shared(field0); @@ -2110,7 +2114,7 @@ TEST_F(TestProjector, TestToDate) { // Build a projector for the expressions. std::shared_ptr projector; - auto status = Projector::Make(schema, {expr}, TestConfiguration(), &projector); + auto status = Projector::Make(schema, {expr}, config, &projector); EXPECT_TRUE(status.ok()); // Create a row-batch with some sample data @@ -2125,13 +2129,74 @@ TEST_F(TestProjector, TestToDate) { // Evaluate expression arrow::ArrayVector outputs; - status = projector->Evaluate(*in_batch, pool_, &outputs); + status = projector->Evaluate(*in_batch, pool, &outputs); EXPECT_TRUE(status.ok()); // Validate results EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0)); } +TEST_F(TestProjector, TestToDate) { CheckToDate(TestConfiguration(), pool_); } + +// If this passes while TestToDate fails, the regression is in the O3 pipeline +// rather than in the base IR that LLVMGenerator emits. +TEST_F(TestProjector, TestToDateNoOptimize) { + CheckToDate(ConfigurationBuilder().build(/*optimize=*/false), pool_); +} + +// If this passes while TestToDate fails, the regression is host-CPU specific +// (CI detects znver3 and enables AVX2/AVX512), i.e. most likely vectorization. +TEST_F(TestProjector, TestToDateNoHostCpu) { + auto config = ConfigurationBuilder().build(); + config->target_host_cpu(false); + CheckToDate(config, pool_); +} + +// Does the wrong result follow the value or the row index? Both inputs are +// 10 chars in the same format, so nothing distinguishes them except position. +TEST_F(TestProjector, TestToDateRowOrder) { + auto field0 = field("f0", arrow::utf8()); + auto field_node = std::make_shared(field0); + auto schema = arrow::schema({field0}); + auto field_result = field("res", arrow::date64()); + + auto pattern_node = std::make_shared( + arrow::utf8(), LiteralHolder(std::string("YYYY-MM-DD")), false); + auto fn_node = TreeExprBuilder::MakeFunction("to_date", {field_node, pattern_node}, + arrow::date64()); + auto expr = TreeExprBuilder::MakeExpression(fn_node, field_result); + + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, TestConfiguration(), &projector)); + + // A single record holding only the value that comes back null in the 3-row + // batch. Failing here means row 0 is mishandled regardless of batch size. + { + auto array0 = MakeArrowArrayUtf8({"1986-12-01"}, {true}); + auto exp = MakeArrowArrayDate64({533779200000}, {true}); + auto in_batch = arrow::RecordBatch::Make(schema, 1, {array0}); + + arrow::ArrayVector outputs; + ASSERT_OK(projector->Evaluate(*in_batch, pool_, &outputs)); + EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0)); + } + + // The original batch with the two valid dates swapped. If the null moves to + // 2012-12-01 the bug tracks the row index; if it stays on 1986-12-01 it + // tracks the value and the parser is back in scope. + { + auto array0 = + MakeArrowArrayUtf8({"2012-12-01", "1986-12-01", "invalid"}, {true, true, false}); + auto exp = + MakeArrowArrayDate64({1354320000000, 533779200000, 0}, {true, true, false}); + auto in_batch = arrow::RecordBatch::Make(schema, 3, {array0}); + + arrow::ArrayVector outputs; + ASSERT_OK(projector->Evaluate(*in_batch, pool_, &outputs)); + EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0)); + } +} + // ARROW-11617 TEST_F(TestProjector, TestIfElseOpt) { // schema for input