ci: include embeddings runtime in build artifacts

.github/workflows/build_template.yml

@@ -44,8 +44,13 @@ on:
         type: string
         default: boost_nov22
       embeddings_artifact:
-        required: true
+        required: false
         type: string
+        default: ""
+      build_embeddings_locally:
+        required: false
+        type: boolean
+        default: false
       artifact_list:
         required: false
         type: string
@@ -124,6 +129,7 @@ jobs:
           cat CMakeLists.txt
 
       - name: Download embeddings lib
+        if: inputs.build_embeddings_locally != true
         uses: manticoresoftware/download_artifact_with_retries@v3
         continue-on-error: true
         with:
@@ -134,9 +140,42 @@ jobs:
         run: | # without adding the safe.directory cmake fails to do git log -1 --date=short --format=%cd
           bash /sysroot/root/entry_point.sh
           git config --global --add safe.directory /__w/columnar/columnar
-          mkdir -p embeddings/target/release
-          mv ./embeddings-lib/build/* embeddings/target/release/
-          rm -fr ./embeddings-lib
+          if [ "${{ inputs.build_embeddings_locally }}" = "true" ]; then
+            apt-get update -y
+            apt-get install -y curl ca-certificates pkg-config protobuf-compiler build-essential libssl-dev unzip
+            curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain stable
+            echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+
+            ort_version="1.24.2"
+            case "${{ inputs.arch }}" in
+              x86_64)
+                ort_asset="onnxruntime-linux-x64-static_lib-${ort_version}-glibc2_17.zip"
+                ort_dir="/tmp/ort/onnxruntime-linux-x64-static_lib-${ort_version}-glibc2_17"
+                ;;
+              aarch64)
+                ort_asset="onnxruntime-linux-aarch64-static_lib-${ort_version}-glibc2_17.zip"
+                ort_dir="/tmp/ort/onnxruntime-linux-aarch64-static_lib-${ort_version}-glibc2_17"
+                ;;
+              *)
+                echo "Unsupported arch for local embeddings build: ${{ inputs.arch }}"
+                exit 1
+                ;;
+            esac
+
+            rm -rf /tmp/ort
+            mkdir -p /tmp/ort
+            curl -fsSL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${ort_version}/${ort_asset}" -o /tmp/ort/ort.zip
+            unzip -q /tmp/ort/ort.zip -d /tmp/ort
+            echo "ORT_LIB_LOCATION=${ort_dir}/lib" >> "$GITHUB_ENV"
+          else
+            mkdir -p embeddings/target/release
+            mv ./embeddings-lib/build/* embeddings/target/release/
+            rm -fr ./embeddings-lib
+          fi
+
+      - name: Enable local embeddings build
+        if: inputs.build_embeddings_locally == true
+        run: echo "BUILD_EMBEDDINGS_LOCALLY=1" >> "$GITHUB_ENV"
 
       - name: Check out main cache before building
         uses: actions/cache@v4
@@ -159,6 +198,12 @@ jobs:
       - name: 🚀 Build
         run: ${{ inputs.cmake_command }}
 
+      - name: Sync locally built embeddings runtime
+        if: success() && inputs.build_embeddings_locally == true
+        run: |
+          mkdir -p embeddings/target/release
+          cp build/knn/embeddings/release/lib_manticore_knn_embeddings.* embeddings/target/release/
+
       - name: Move AVX2/AVX-512-optimized libraries
         if: success() && inputs.skip_avx2_move != true
         run: |
@@ -213,4 +258,4 @@ jobs:
         uses: manticoresoftware/upload_artifact_with_retries@v4
         with:
           name: ${{ steps.artifact_name.outputs.name }}
-          path: ${{ inputs.artifact_list }}
+          path: ${{ inputs.artifact_list }}

.github/workflows/embedding_build_template.yml

@@ -233,10 +233,12 @@ jobs:
           # Set Docker image based on architecture
           if [[ "${{ inputs.arch }}" == "aarch64" ]]; then
             docker_image="ghcr.io/manticoresoftware/rust-min-libc:aarch64-rust1.94.1-glibc2.27-openssl1.1.1k"
-            extra=""
+            ort_asset="onnxruntime-linux-aarch64-static_lib-1.24.2-glibc2_17.zip"
+            ort_dir="onnxruntime-linux-aarch64-static_lib-1.24.2-glibc2_17"
           else
             docker_image="ghcr.io/manticoresoftware/rust-min-libc:amd64-rust1.94.1-glibc2.27-openssl1.1.1k"
-            extra=""
+            ort_asset="onnxruntime-linux-x64-static_lib-1.24.2-glibc2_17.zip"
+            ort_dir="onnxruntime-linux-x64-static_lib-1.24.2-glibc2_17"
           fi
 
           docker run --rm  \
@@ -250,7 +252,13 @@ jobs:
             -e GIT_TIMESTAMP_ID="${{ steps.git_meta.outputs.timestamp }}" \
             --entrypoint bash \
             "${docker_image}" \
-            -c "apt-get update -y && apt-get install -y gcc g++ protobuf-compiler && ${extra} cargo build ${{ steps.params.outputs.args }}"
+            -c "set -euo pipefail && \
+              apt-get update -y && apt-get install -y gcc g++ protobuf-compiler curl unzip && \
+              rm -rf /tmp/ort && mkdir -p /tmp/ort && \
+              curl -fsSL https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.24.2/${ort_asset} -o /tmp/ort/ort.zip && \
+              unzip -q /tmp/ort/ort.zip -d /tmp/ort && \
+              export ORT_LIB_LOCATION=/tmp/ort/${ort_dir}/lib && \
+              cargo build ${{ steps.params.outputs.args }}"
           sudo chown -hR $(id -u):$(id -g) ${{ steps.vars.outputs.lib_dir }}
           sudo chown -hR $(id -u):$(id -g) .cargo-cache
 

.github/workflows/nightly_no_avx2.yml

@@ -78,26 +78,14 @@ jobs:
             echo "Set manticore_locator: (empty)"
           fi
 
-  embedding_linux_pack:
-    needs: commit_info
-    uses: ./.github/workflows/embedding_build_template.yml
-    strategy:
-      fail-fast: false
-      matrix:
-        distr: [linux]
-        arch: [x86_64]
-    name: ${{ matrix.distr }} ${{ matrix.arch }} embedding packing
-    with:
-      distr: ${{ matrix.distr }}
-      arch: ${{ matrix.arch }}
-
   build_linux_release:
     name: Linux release build
-    needs: [embedding_linux_pack, check_branch]
+    needs: [check_branch]
     uses: ./.github/workflows/build_template.yml
     with:
       MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }}
-      embeddings_artifact: embeddings_linux_x86_64
+      artifact_list: "build/columnar/lib_manticore_columnar_avx2.so build/columnar/lib_manticore_columnar_avx512.so build/columnar/lib_manticore_columnar.so build/secondary/lib_manticore_secondary_avx2.so build/secondary/lib_manticore_secondary_avx512.so build/secondary/lib_manticore_secondary.so build/knn/lib_manticore_knn.so build/knn/lib_manticore_knn_avx2.so build/knn/lib_manticore_knn_avx512.so embeddings/target/release/lib_manticore_knn_embeddings.so build/_deps/manticore-build/src/searchd build/_deps/manticore-build/src/indexer build/_deps/manticore-build/src/indextool build/_deps/manticore-build/config/*.c build/_deps/manticore-build/config/*.h"
+      build_embeddings_locally: true
 
   test_linux_release:
     name: Linux release mode tests

.github/workflows/test.yml

@@ -123,20 +123,22 @@ jobs:
 
   build_linux_release:
     name: Linux release build
-    needs: [embedding_linux_pack, check_branch]
+    needs: [check_branch]
     uses: ./.github/workflows/build_template.yml
     with:
       MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }}
-      embeddings_artifact: embeddings_linux_x86_64
+      artifact_list: "build/columnar/lib_manticore_columnar_avx2.so build/columnar/lib_manticore_columnar_avx512.so build/columnar/lib_manticore_columnar.so build/secondary/lib_manticore_secondary_avx2.so build/secondary/lib_manticore_secondary_avx512.so build/secondary/lib_manticore_secondary.so build/knn/lib_manticore_knn.so build/knn/lib_manticore_knn_avx2.so build/knn/lib_manticore_knn_avx512.so embeddings/target/release/lib_manticore_knn_embeddings.so build/_deps/manticore-build/src/searchd build/_deps/manticore-build/src/indexer build/_deps/manticore-build/src/indextool build/_deps/manticore-build/config/*.c build/_deps/manticore-build/config/*.h"
+      build_embeddings_locally: true
 
   build_linux_debug:
     name: Linux debug build
-    needs: [embedding_linux_pack, check_branch]
+    needs: [check_branch]
     uses: ./.github/workflows/build_template.yml
     with:
       CTEST_CONFIGURATION_TYPE: "Debug"
       MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }}
-      embeddings_artifact: embeddings_linux_x86_64
+      artifact_list: "build/columnar/lib_manticore_columnar_avx2.so build/columnar/lib_manticore_columnar_avx512.so build/columnar/lib_manticore_columnar.so build/secondary/lib_manticore_secondary_avx2.so build/secondary/lib_manticore_secondary_avx512.so build/secondary/lib_manticore_secondary.so build/knn/lib_manticore_knn.so build/knn/lib_manticore_knn_avx2.so build/knn/lib_manticore_knn_avx512.so embeddings/target/release/lib_manticore_knn_embeddings.so build/_deps/manticore-build/src/searchd build/_deps/manticore-build/src/indexer build/_deps/manticore-build/src/indextool build/_deps/manticore-build/config/*.c build/_deps/manticore-build/config/*.h"
+      build_embeddings_locally: true
 
   build_windows:
     name: Windows x64 build

.github/workflows/test_template.yml

@@ -104,19 +104,6 @@ jobs:
           name: ${{ steps.build_artifact.outputs.name }}
           path: .
 
-      - name: Download embeddings lib 
-        uses: manticoresoftware/download_artifact_with_retries@v4
-        continue-on-error: true
-        with:
-          name: embeddings_linux_x86_64
-          path: ./embeddings-lib/
-
-      - name: Initialization of embeddings lib
-        run: | 
-          mkdir -p embeddings/target/release
-          mv ./embeddings-lib/build/* embeddings/target/release/
-          rm -fr ./embeddings-lib
-
       - name: Check out main cache before building
         uses: actions/cache@v4
         with:

.github/workflows/win_test_template.yml

@@ -14,6 +14,10 @@ on:
       artifact_name:
         required: true
         type: string
+      embeddings_artifact:
+        required: false
+        type: string
+        default: embeddings_windows_x86_64
       MANTICORE_LOCATOR:
         required: false
         type: string
@@ -90,6 +94,17 @@ jobs:
         with:
           name: build_windows_Debug_x64
           path: .
+      - name: Download embeddings lib
+        uses: manticoresoftware/download_artifact_with_retries@v4
+        with:
+          name: ${{ inputs.embeddings_artifact }}
+          path: ./embeddings-lib/
+      - name: Prepare embeddings runtime path
+        shell: bash
+        run: |
+          mkdir -p embeddings/target/release
+          mv ./embeddings-lib/build/* embeddings/target/release/
+          rm -fr ./embeddings-lib
       - name: Cache Windows bundle
         id: bundle_cache
         uses: actions/cache@v4
@@ -146,20 +161,6 @@ jobs:
           php-version: '8.2'
           extensions: mysqli
 
-      - name: Download embeddings lib 
-        uses: manticoresoftware/download_artifact_with_retries@v4
-        continue-on-error: true
-        with:
-          name: embeddings_windows_x86_64
-          path: ./embeddings-lib/
-
-      - name: Initialization of embeddings lib
-        run: | 
-          mkdir embeddings\target
-          mkdir embeddings\target\release
-          mv .\embeddings-lib\build\* embeddings\target\release\
-          rm -r ./embeddings-lib
-
       - name: Replace standard libs to AVX2
         if: ${{ inputs.USE_AVX2 == true }}
         run: |

cmake/citest.cmake

@@ -14,6 +14,7 @@ set ( SEARCHD_CLI_EXTRA "$ENV{SEARCHD_CLI_EXTRA}" )
 set ( WITH_COVERAGE "$ENV{WITH_COVERAGE}" )
 set ( NO_TESTS "$ENV{NO_TESTS}" )
 set ( NO_BUILD "$ENV{NO_BUILD}" )
+set ( BUILD_EMBEDDINGS_LOCALLY "$ENV{BUILD_EMBEDDINGS_LOCALLY}" )
 set_property ( GLOBAL PROPERTY Label P$ENV{CI_PIPELINE_ID} J$ENV{CI_JOB_ID} )
 
 # how may times try the test before it is considered failed
@@ -49,6 +50,10 @@ if (SEARCHD_CLI_EXTRA)
 	list ( APPEND CONFIG_OPTIONS "SEARCHD_CLI_EXTRA=${SEARCHD_CLI_EXTRA}" )
 endif ()
 
+if (BUILD_EMBEDDINGS_LOCALLY)
+	list ( APPEND CONFIG_OPTIONS "BUILD_EMBEDDINGS_LOCALLY=ON" )
+endif ()
+
 set ( CTEST_START_WITH_EMPTY_BINARY_DIRECTORY TRUE )
 #ctest_empty_binary_directory(${CTEST_BINARY_DIRECTORY})
 

knn/CMakeLists.txt

@@ -50,7 +50,7 @@ if (PACK AND NOT EMBEDDINGS_LIB)
 endif()
 
 # Add embeddings library as a component to be included in packages
-add_library ( embeddings MODULE IMPORTED )
+add_library ( embeddings MODULE IMPORTED GLOBAL )
 set_target_properties ( embeddings PROPERTIES IMPORTED_LOCATION "${EMBEDDINGS_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/embeddings/")
 
 if ( NOT EXTERNAL_LIB ) # install only once, not from external_project build

testing.cmake

@@ -35,6 +35,7 @@ function ( special_ubertest_properties test )
 	set_property ( TEST "${test}" APPEND PROPERTY ENVIRONMENT "LIB_MANTICORE_COLUMNAR=$<TARGET_FILE:columnar_lib>" )
 	set_property ( TEST "${test}" APPEND PROPERTY ENVIRONMENT "LIB_MANTICORE_SECONDARY=$<TARGET_FILE:secondary_index>" )
 	set_property ( TEST "${test}" APPEND PROPERTY ENVIRONMENT "LIB_MANTICORE_KNN=$<TARGET_FILE:knn_lib>" )
+	set_property ( TEST "${test}" APPEND PROPERTY ENVIRONMENT "LIB_MANTICORE_KNN_EMBEDDINGS=$<TARGET_FILE:embeddings>" )
 endfunction ()
 
 # this will switch off pure manticore-specific tests: google, api, keyword consistency and benches (we don't need them here)
@@ -64,4 +65,4 @@ include ( "${columnar_BINARY_DIR}/manticore-get.cmake" )
 
 # add manticore sources to the tree. All testing will be done on manticore side; necessary additional tests/properties will
 # be set by cb functions defined above.
-FetchContent_MakeAvailable ( manticore )
+FetchContent_MakeAvailable ( manticore )