Faiss test

.github/workflows/publish-faiss-snapshot.yml

@@ -0,0 +1,304 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+name: Publish Faiss Snapshot
+
+on:
+  schedule:
+    # At the end of every day
+    - cron: '0 1 * * *'
+  workflow_dispatch:
+  push:
+    paths:
+      - 'paimon-faiss/**'
+      - 'paimon-faiss-jni/**'
+    branches:
+      - master
+
+env:
+  JDK_VERSION: 8
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  # Build native library for Linux AMD64
+  build-native-linux-amd64:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'temurin'
+
+      - name: Install native dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            build-essential \
+            libopenblas-dev \
+            liblapack-dev \
+            patchelf \
+            libgomp1 \
+            wget
+
+      - name: Install GCC 9
+        run: |
+          sudo apt-get install -y gcc-9 g++-9
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
+          gcc --version
+
+      - name: Install CMake 3.30.1
+        run: |
+          CMAKE_VERSION="3.30.1"
+          wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz
+          tar -xzf cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz
+          sudo mv cmake-${CMAKE_VERSION}-linux-x86_64 /opt/cmake
+          sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake
+          cmake --version
+
+      - name: Install FAISS
+        run: |
+          git clone --depth 1 --branch v1.7.4 https://github.com/facebookresearch/faiss.git /tmp/faiss
+          cd /tmp/faiss
+          cmake -B build \
+            -DFAISS_ENABLE_GPU=OFF \
+            -DFAISS_ENABLE_PYTHON=OFF \
+            -DBUILD_TESTING=OFF \
+            -DCMAKE_BUILD_TYPE=Release
+          cmake --build build -j $(nproc)
+          sudo cmake --install build
+
+      - name: Build native library
+        run: |
+          cd paimon-faiss-jni
+          ./scripts/build-native.sh --clean --fat-lib
+
+      - name: List built libraries
+        run: |
+          echo "=== Built libraries ==="
+          ls -la paimon-faiss-jni/src/main/resources/linux/amd64/
+          echo ""
+          echo "=== Library dependencies ==="
+          ldd paimon-faiss-jni/src/main/resources/linux/amd64/libpaimon_faiss_jni.so || true
+
+      - name: Upload native library
+        uses: actions/upload-artifact@v4
+        with:
+          name: native-linux-amd64
+          path: paimon-faiss-jni/src/main/resources/linux/amd64/
+          retention-days: 1
+
+  # Build native library for Linux AARCH64
+  build-native-linux-aarch64:
+    runs-on: ubuntu-24.04-arm
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'temurin'
+
+      - name: Install native dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            build-essential \
+            libopenblas-dev \
+            liblapack-dev \
+            patchelf \
+            libgomp1 \
+            wget
+
+      - name: Install GCC 9
+        run: |
+          sudo apt-get install -y gcc-9 g++-9 || sudo apt-get install -y gcc g++
+          if command -v gcc-9 &>/dev/null; then
+            sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
+            sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
+          fi
+          gcc --version
+
+      - name: Install CMake 3.30.1
+        run: |
+          CMAKE_VERSION="3.30.1"
+          wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-aarch64.tar.gz
+          tar -xzf cmake-${CMAKE_VERSION}-linux-aarch64.tar.gz
+          sudo mv cmake-${CMAKE_VERSION}-linux-aarch64 /opt/cmake
+          sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake
+          cmake --version
+
+      - name: Install FAISS
+        run: |
+          git clone --depth 1 --branch v1.7.4 https://github.com/facebookresearch/faiss.git /tmp/faiss
+          cd /tmp/faiss
+          cmake -B build \
+            -DFAISS_ENABLE_GPU=OFF \
+            -DFAISS_ENABLE_PYTHON=OFF \
+            -DBUILD_TESTING=OFF \
+            -DCMAKE_BUILD_TYPE=Release
+          cmake --build build -j $(nproc)
+          sudo cmake --install build
+
+      - name: Build native library
+        run: |
+          cd paimon-faiss-jni
+          ./scripts/build-native.sh --clean --fat-lib
+
+      - name: List built libraries
+        run: |
+          echo "=== Built libraries ==="
+          ls -la paimon-faiss-jni/src/main/resources/linux/aarch64/
+          echo ""
+          echo "=== Library dependencies ==="
+          ldd paimon-faiss-jni/src/main/resources/linux/aarch64/libpaimon_faiss_jni.so || true
+
+      - name: Upload native library
+        uses: actions/upload-artifact@v4
+        with:
+          name: native-linux-aarch64
+          path: paimon-faiss-jni/src/main/resources/linux/aarch64/
+          retention-days: 1
+
+  # Build native library for macOS ARM (Apple Silicon)
+  build-native-macos-arm:
+    runs-on: macos-14
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'temurin'
+
+      - name: Install dependencies
+        run: |
+          brew install cmake libomp openblas faiss
+
+      - name: Build native library
+        run: |
+          cd paimon-faiss-jni
+          ./scripts/build-native.sh --clean --fat-lib
+
+      - name: List built libraries
+        run: |
+          echo "=== Built libraries ==="
+          ls -la paimon-faiss-jni/src/main/resources/darwin/aarch64/
+          echo ""
+          echo "=== Library dependencies ==="
+          otool -L paimon-faiss-jni/src/main/resources/darwin/aarch64/libpaimon_faiss_jni.dylib || true
+
+      - name: Upload native library
+        uses: actions/upload-artifact@v4
+        with:
+          name: native-darwin-aarch64
+          path: paimon-faiss-jni/src/main/resources/darwin/aarch64/
+          retention-days: 1
+
+  # Package and publish
+  package-and-publish:
+    if: github.repository == 'apache/paimon'
+    needs: [build-native-linux-amd64, build-native-linux-aarch64, build-native-macos-arm]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'temurin'
+
+      - name: Download Linux AMD64 native library
+        uses: actions/download-artifact@v4
+        with:
+          name: native-linux-amd64
+          path: paimon-faiss-jni/src/main/resources/linux/amd64/
+
+      - name: Download Linux AARCH64 native library
+        uses: actions/download-artifact@v4
+        with:
+          name: native-linux-aarch64
+          path: paimon-faiss-jni/src/main/resources/linux/aarch64/
+
+      - name: Download macOS ARM native library
+        uses: actions/download-artifact@v4
+        with:
+          name: native-darwin-aarch64
+          path: paimon-faiss-jni/src/main/resources/darwin/aarch64/
+
+      - name: List all native libraries
+        run: |
+          echo "=== All native libraries ==="
+          find paimon-faiss-jni/src/main/resources -type f \( -name "*.so" -o -name "*.so.*" -o -name "*.dylib" \) -exec ls -la {} \;
+
+      - name: Cache local Maven repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: faiss-snapshot-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            faiss-snapshot-maven-
+
+      - name: Build and package paimon-faiss-jni
+        run: |
+          mvn -B clean install -pl paimon-faiss-jni -am -DskipTests -Ppaimon-faiss-vector -Drat.skip
+
+      - name: Build and package paimon-faiss
+        run: |
+          mvn -B clean install -pl paimon-faiss -am -DskipTests -Ppaimon-faiss-vector -Drat.skip
+
+      - name: Publish snapshot
+        env:
+          ASF_USERNAME: ${{ secrets.NEXUS_USER }}
+          ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
+          MAVEN_OPTS: -Xmx4096m
+        run: |
+          tmp_settings="tmp-settings.xml"
+          echo "<settings><servers><server>" > $tmp_settings
+          echo "<id>apache.snapshots.https</id><username>$ASF_USERNAME</username>" >> $tmp_settings
+          echo "<password>$ASF_PASSWORD</password>" >> $tmp_settings
+          echo "</server></servers></settings>" >> $tmp_settings
+
+          mvn --settings $tmp_settings deploy -pl paimon-faiss-jni,paimon-faiss -Dgpg.skip -Drat.skip -DskipTests -Ppaimon-faiss-vector
+
+          rm $tmp_settings
+
+      - name: Upload packaged JARs
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: faiss-jars
+          path: |
+            paimon-faiss-jni/target/*.jar
+            paimon-faiss/target/*.jar
+          retention-days: 7
+