ci: resolve Arrow and Parquet from conda-forge on Unix legs

.github/actions/setup-conda-arrow/action.yml

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Set up conda Arrow
+description: Install Arrow and Parquet from conda-forge for CMake builds.
+
+inputs:
+  extra-packages:
+    description: Extra conda packages to install, space separated.
+    required: false
+    default: ""
+
+runs:
+  using: composite
+  steps:
+    - name: Set up conda
+      uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1
+      with:
+        miniforge-version: latest
+        channels: conda-forge
+        channel-priority: strict
+        activate-environment: arrow
+    - name: Install Arrow and Parquet
+      shell: bash -el {0}
+      env:
+        EXTRA_PACKAGES: ${{ inputs.extra-packages }}
+      run: '"${{ github.action_path }}/install-arrow.sh"'

.github/actions/setup-conda-arrow/install-arrow.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Installs Arrow and Parquet into the active conda environment and exports the
+# paths CMake and the test binaries need. Run from the setup-conda-arrow action.
+
+set -euo pipefail
+
+toolchain="${GITHUB_WORKSPACE}/cmake_modules/IcebergThirdpartyToolchain.cmake"
+
+# Keep conda Arrow in sync with the vendored build.
+arrow_version=$(sed -nE 's/^set\(ICEBERG_ARROW_BUILD_VERSION "([0-9]+\.[0-9]+\.[0-9]+)"\)$/\1/p' "${toolchain}")
+if [[ -z "${arrow_version}" ]]; then
+    echo "::error::Could not read ICEBERG_ARROW_BUILD_VERSION from ${toolchain}"
+    exit 1
+fi
+
+packages=("libarrow==${arrow_version}" "libparquet==${arrow_version}")
+if [[ -n "${EXTRA_PACKAGES:-}" ]]; then
+    read -r -a extra_packages <<< "${EXTRA_PACKAGES}"
+    packages+=("${extra_packages[@]}")
+fi
+mamba install -y "${packages[@]}"
+
+echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "${GITHUB_ENV}"
+# build_iceberg.sh reads this and fails if CMake configured vendored Arrow anyway.
+echo "ICEBERG_REQUIRE_SYSTEM_ARROW=ON" >> "${GITHUB_ENV}"
+
+if [[ "${RUNNER_OS}" == "macOS" ]]; then
+    echo "DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH:+${DYLD_FALLBACK_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "${GITHUB_ENV}"
+else
+    echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "${GITHUB_ENV}"
+fi

.github/workflows/aws_test.yml

@@ -117,6 +117,9 @@ jobs:
         if: ${{ matrix.s3 == 'ON' }}
         shell: bash
         run: bash ci/scripts/start_minio.sh
+      - name: Set up conda Arrow
+        if: ${{ matrix.bundle_awssdk == 'OFF' }}
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:

.github/workflows/sanitizer_test.yml

@@ -51,6 +51,8 @@ jobs:
       - name: Install dependencies
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: Set up conda Arrow
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
@@ -63,6 +65,10 @@ jobs:
           mkdir build && cd build
           cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON \
             -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+          if [[ -d _deps/vendoredarrow-src ]]; then
+            echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow"
+            exit 1
+          fi
           cmake --build . --verbose
       - name: Save sccache
         if: always()

.github/workflows/sql_catalog_test.yml

@@ -96,6 +96,9 @@ jobs:
         shell: pwsh
         run: |
           vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows
+      - name: Set up conda Arrow
+        if: ${{ !startsWith(matrix.runs-on, 'windows') }}
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
@@ -114,6 +117,10 @@ jobs:
             -DCMAKE_C_COMPILER_LAUNCHER=sccache \
             -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
             ${{ matrix.cmake_extra_args }}
+          if [[ "${RUNNER_OS}" != "Windows" && -d build/_deps/vendoredarrow-src ]]; then
+            echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow"
+            exit 1
+          fi
       - name: Build SQL catalog tests
         shell: bash
         run: cmake --build build --target sql_catalog_test

.github/workflows/test.yml

@@ -61,6 +61,8 @@ jobs:
       - name: Install dependencies
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: Set up conda Arrow
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
@@ -139,6 +141,8 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           persist-credentials: false
+      - name: Set up conda Arrow
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:

ci/scripts/build_iceberg.sh

@@ -88,6 +88,11 @@ fi
 
 cmake "${CMAKE_ARGS[@]}" ${source_dir}
 
+if [[ "${ICEBERG_REQUIRE_SYSTEM_ARROW:-OFF}" == "ON" && -d _deps/vendoredarrow-src ]]; then
+    echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow"
+    exit 1
+fi
+
 cmake --build . --target install
 if [[ "${run_tests}" == "ON" ]]; then
     ctest --output-on-failure