INTPYTHON-591 Add pymongo-voyageai to pipeline (#64)

Author: blink1073

.evergreen/config.yml

@@ -45,11 +45,16 @@ functions:
         binary: bash
         args: [.evergreen/fetch-repo.sh]
 
+  "assume role":
+    - command: ec2.assume_role
+      params:
+        role_arn: ${assume_role_arn}
+
   "execute tests":
     - command: subprocess.exec
       type: test
       params:
-        include_expansions_in_env: [DIR]
+        include_expansions_in_env: [DIR, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
         working_dir: "src"
         binary: bash
         args: [.evergreen/execute-tests.sh]
@@ -185,6 +190,22 @@ tasks:
       - func: "setup remote atlas"
       - func: "execute tests"
 
+  - name: test-pymongo-voyageai-local
+    tags: [local]
+    commands:
+      - func: "fetch repo"
+      - func: "assume role"
+      - func: "setup local atlas"
+      - func: "execute tests"
+
+  - name: test-pymongo-voyageai-remote
+    tags: [remote]
+    commands:
+      - func: "fetch repo"
+      - func: "assume role"
+      - func: "setup remote atlas"
+      - func: "execute tests"
+
   - name: test-haystack-embeddings-local
     tags: [local]
     commands:
@@ -298,6 +319,16 @@ buildvariants:
       - name: test-docarray-local
       - name: test-docarray-remote
 
+  - name: test-pymongo-voyageai-rhel
+    display_name: PyMongo-VoyageAI RHEL
+    expansions:
+      DIR: pymongo-voyageai
+    run_on:
+      - rhel87-small
+    tasks:
+      - name: test-pymongo-voyageai-local
+      - name: test-pymongo-voyageai-remote
+
   - name: test-haystack-embeddings-rhel
     display_name: Haystack Embeddings RHEL
     expansions:

.evergreen/fetch-repo.sh

@@ -3,7 +3,7 @@
 set -eu
 
 if [ ! -d "${DIR}" ]; then
-  echo '${REPO_NAME} could not be found' 1>&2
+  echo "${REPO_NAME} could not be found" 1>&2
   exit 1
 fi
 

.evergreen/provision-atlas.sh

@@ -17,5 +17,7 @@ scaffold_atlas
 source secrets-export.sh
 
 # Create the env file
+echo "export VOYAGEAI_S3_BUCKET=$VOYAGEAI_S3_BUCKET" >> env.sh
 echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> env.sh
 echo "export MONGODB_URI=$CONN_STRING" >> env.sh
+echo "export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY" >> env.sh

.evergreen/setup-remote.sh

@@ -38,14 +38,19 @@ case $DIR in
     haystack-embeddings | haystack-fulltext)
         MONGODB_URI=$HAYSTACK_MONGODB_URI
     ;;
+    pymongo-voyageai)
+        MONGODB_URI=$VOYAGEAI_MONGODB_URI
+    ;;
     *)
-        echo "Missing config in fetch-secrets.sh for DIR: $DIR"
+        echo "Missing config in setup-remote.sh for DIR: $DIR"
         exit 1
     ;;
 esac
 export MONGODB_URI
 
 # Create the env file
+echo "export VOYAGEAI_S3_BUCKET=$VOYAGEAI_S3_BUCKET" >> env.sh
+echo "export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY" >> env.sh
 echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> env.sh
 echo "export MONGODB_URI=$MONGODB_URI" >> env.sh
 

.gitignore

@@ -60,3 +60,4 @@ env.sh
 # Repo checkouts
 haystack-fulltext/haystack-core-integrations
 haystack-embeddings/haystack-core-integrations
+pymongo-voyageai/pymongo-voyageai

pymongo-voyageai/config.env

@@ -0,0 +1,3 @@
+REPO_NAME=pymongo-voyageai
+CLONE_URL="https://github.com/mongodb-labs/pymongo-voyageai.git"
+DATABASE="pymongo_voyageai_test_db"

pymongo-voyageai/run.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -eu
+
+# Get the MONGODB_URI and VOYAGEAI_API_KEY.
+SCRIPT_DIR=$(realpath "$(dirname ${BASH_SOURCE[0]})")
+ROOT_DIR=$(dirname $SCRIPT_DIR)
+. $ROOT_DIR/env.sh
+
+. $ROOT_DIR/.evergreen/utils.sh
+
+PYTHON_BINARY=$(find_python3)
+
+$PYTHON_BINARY -m venv venv_pipeline
+source venv_pipeline/bin/activate
+
+pip install uv rust-just
+
+just install
+
+export S3_BUCKET_NAME=$VOYAGEAI_S3_BUCKET
+export MONGODB_URI=$MONGODB_URI
+export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY
+
+just test