GH-51245: [C++][Gandiva] Add support for LLVM 23.1.0

ci/conda_env_gandiva.txt

@@ -16,4 +16,4 @@
 # under the License.
 
 clang
-llvmdev<23
+llvmdev

ci/docker/conda-cpp.dockerfile

@@ -28,6 +28,7 @@ RUN /arrow/ci/scripts/install_minio.sh latest /opt/conda
 COPY ci/conda_env_cpp.txt \
      ci/conda_env_gandiva.txt \
      /arrow/ci/
+RUN echo "Force rebuild"
 RUN mamba install -q -y \
         --file arrow/ci/conda_env_cpp.txt \
         --file arrow/ci/conda_env_gandiva.txt \

cpp/CMakeLists.txt

@@ -179,6 +179,7 @@ set(ARROW_DOC_DIR "${CMAKE_INSTALL_DOCDIR}")
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 
 set(ARROW_LLVM_VERSIONS
+    "23.1"
     "22.1"
     "21.1"
     "20.1"

cpp/src/gandiva/engine.cc

@@ -207,7 +207,12 @@ Status UseJITLinkIfEnabled(llvm::orc::LLJITBuilder& jit_builder) {
   static auto maybe_use_jit_link = ::arrow::internal::GetEnvVar("GANDIVA_USE_JIT_LINK");
   if (maybe_use_jit_link.ok()) {
     ARROW_ASSIGN_OR_RAISE(static auto memory_manager, CreateMemmoryManager());
-#  if LLVM_VERSION_MAJOR >= 21
+#  if LLVM_VERSION_MAJOR >= 23
+    jit_builder.setObjectLinkingLayerCreator(
+        [&](llvm::orc::ExecutionSession& ES, llvm::jitlink::JITLinkMemoryManager&) {
+          return std::make_unique<llvm::orc::ObjectLinkingLayer>(ES, *memory_manager);
+        });
+#  elif LLVM_VERSION_MAJOR >= 21
     jit_builder.setObjectLinkingLayerCreator([&](llvm::orc::ExecutionSession& ES) {
       return std::make_unique<llvm::orc::ObjectLinkingLayer>(ES, *memory_manager);
     });
@@ -503,8 +508,10 @@ static void OptimizeModuleWithNewPassManager(llvm::Module& module,
     function_pm.addPass(llvm::GVNPass());
     function_pm.addPass(llvm::NewGVNPass());
     function_pm.addPass(llvm::SimplifyCFGPass());
-    function_pm.addPass(llvm::LoopVectorizePass());
-    function_pm.addPass(llvm::SLPVectorizerPass());
+    // GH-51245 probe: do the gandiva-injected vectorizers cause the LLVM 23
+    // to_date miscompile?
+    // function_pm.addPass(llvm::LoopVectorizePass());
+    // function_pm.addPass(llvm::SLPVectorizerPass());
     module_pm.addPass(llvm::createModuleToFunctionPassAdaptor(std::move(function_pm)));
 
     module_pm.addPass(llvm::GlobalOptPass());

cpp/src/gandiva/tests/projector_test.cc

@@ -2091,7 +2091,11 @@ TEST_F(TestProjector, TestCastVarbinaryFunction) {
   EXPECT_ARROW_ARRAY_EQUALS(out_float8, outputs.at(3));
 }
 
-TEST_F(TestProjector, TestToDate) {
+// GH-51245: the body is shared so the same projection can be run under
+// different Configurations, to tell an LLVM 23 optimizer/codegen regression
+// apart from a plain to_date bug.
+static void CheckToDate(const std::shared_ptr<Configuration>& config,
+                        arrow::MemoryPool* pool) {
   // schema for input fields
   auto field0 = field("f0", arrow::utf8());
   auto field_node = std::make_shared<FieldNode>(field0);
@@ -2110,7 +2114,7 @@ TEST_F(TestProjector, TestToDate) {
 
   // Build a projector for the expressions.
   std::shared_ptr<Projector> projector;
-  auto status = Projector::Make(schema, {expr}, TestConfiguration(), &projector);
+  auto status = Projector::Make(schema, {expr}, config, &projector);
   EXPECT_TRUE(status.ok());
 
   // Create a row-batch with some sample data
@@ -2125,13 +2129,74 @@ TEST_F(TestProjector, TestToDate) {
 
   // Evaluate expression
   arrow::ArrayVector outputs;
-  status = projector->Evaluate(*in_batch, pool_, &outputs);
+  status = projector->Evaluate(*in_batch, pool, &outputs);
   EXPECT_TRUE(status.ok());
 
   // Validate results
   EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0));
 }
 
+TEST_F(TestProjector, TestToDate) { CheckToDate(TestConfiguration(), pool_); }
+
+// If this passes while TestToDate fails, the regression is in the O3 pipeline
+// rather than in the base IR that LLVMGenerator emits.
+TEST_F(TestProjector, TestToDateNoOptimize) {
+  CheckToDate(ConfigurationBuilder().build(/*optimize=*/false), pool_);
+}
+
+// If this passes while TestToDate fails, the regression is host-CPU specific
+// (CI detects znver3 and enables AVX2/AVX512), i.e. most likely vectorization.
+TEST_F(TestProjector, TestToDateNoHostCpu) {
+  auto config = ConfigurationBuilder().build();
+  config->target_host_cpu(false);
+  CheckToDate(config, pool_);
+}
+
+// Does the wrong result follow the value or the row index? Both inputs are
+// 10 chars in the same format, so nothing distinguishes them except position.
+TEST_F(TestProjector, TestToDateRowOrder) {
+  auto field0 = field("f0", arrow::utf8());
+  auto field_node = std::make_shared<FieldNode>(field0);
+  auto schema = arrow::schema({field0});
+  auto field_result = field("res", arrow::date64());
+
+  auto pattern_node = std::make_shared<LiteralNode>(
+      arrow::utf8(), LiteralHolder(std::string("YYYY-MM-DD")), false);
+  auto fn_node = TreeExprBuilder::MakeFunction("to_date", {field_node, pattern_node},
+                                               arrow::date64());
+  auto expr = TreeExprBuilder::MakeExpression(fn_node, field_result);
+
+  std::shared_ptr<Projector> projector;
+  ASSERT_OK(Projector::Make(schema, {expr}, TestConfiguration(), &projector));
+
+  // A single record holding only the value that comes back null in the 3-row
+  // batch. Failing here means row 0 is mishandled regardless of batch size.
+  {
+    auto array0 = MakeArrowArrayUtf8({"1986-12-01"}, {true});
+    auto exp = MakeArrowArrayDate64({533779200000}, {true});
+    auto in_batch = arrow::RecordBatch::Make(schema, 1, {array0});
+
+    arrow::ArrayVector outputs;
+    ASSERT_OK(projector->Evaluate(*in_batch, pool_, &outputs));
+    EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0));
+  }
+
+  // The original batch with the two valid dates swapped. If the null moves to
+  // 2012-12-01 the bug tracks the row index; if it stays on 1986-12-01 it
+  // tracks the value and the parser is back in scope.
+  {
+    auto array0 =
+        MakeArrowArrayUtf8({"2012-12-01", "1986-12-01", "invalid"}, {true, true, false});
+    auto exp =
+        MakeArrowArrayDate64({1354320000000, 533779200000, 0}, {true, true, false});
+    auto in_batch = arrow::RecordBatch::Make(schema, 3, {array0});
+
+    arrow::ArrayVector outputs;
+    ASSERT_OK(projector->Evaluate(*in_batch, pool_, &outputs));
+    EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0));
+  }
+}
+
 // ARROW-11617
 TEST_F(TestProjector, TestIfElseOpt) {
   // schema for input