Syncing to main

.github/workflows/model_pipeline.yaml

@@ -1,61 +1,210 @@
-
-name: Model Pipeline v1
-
-on:
-  push:
-    branches:
-      - dheeraj_model
-  pull_request:
-    branches:
-      - dheeraj_model
-  workflow_dispatch:
-
-env:
-  GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} # Your GCP Project ID
-  GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} # The service account key stored as a secret
-  IMAGE_NAME: "hello-world-image"
-  REGISTRY: "us-central1-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/ml-models"
-
-jobs:
-  model_pipeline:
-    runs-on: ubuntu-latest
-
-    steps:
-      # Step 1: Checkout repository
-      - name: Checkout Repository
-        uses: actions/checkout@v2
-
-      # Step 2: Clean docker image
-      - name: Clean up Docker to free space
-        run: docker system prune -af --volumes
-
-      # Step 3: Set up Docker Buildx
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      # Step 4: Login to GCP Artifact Registry
-      - name: Login to GCP Artifact Registry
-        uses: docker/login-action@v1
-        with:
-          registry: us-central1-docker.pkg.dev
-          username: _json_key
-          password: ${{ secrets.GCP_SA_KEY }}
-
-      # Step 5: Build and Push Docker Image
-      - name: Build and Push Docker Image
-        run: |
-          docker build -f Dockerfilemodel -t $REGISTRY/$IMAGE_NAME:latest .
-          docker push $REGISTRY/$IMAGE_NAME:latest
-
-      # Step 6: Run Model with Raytuner from Docker Image
-      - name: Run Model development file
-        run: |
-          docker run --rm $REGISTRY/$IMAGE_NAME:latest python model/model_development_refined.py
-
-      - name: Notify on success
-        if: success()
-        run: echo "Unit tests passed successfully"
-
-      - name: Notify on failure
-        if: failure()
-        run: echo "Unit tests failed"
+name: Deploy model to Vertex AI
+on:
+  push:
+    branches:
+      - #dhanush_model
+  pull_request:
+    branches:
+      - #dhanush_model
+  workflow_dispatch:
+env:
+  GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+  GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+  IMAGE_NAME: "ml-model-image"
+  REGISTRY: "us-east1-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/ml-models-new"
+  GCS_BUCKET: "nih-dataset-mlops"
+
+jobs:
+  deploy-to-vertex-ai:
+    runs-on: ubuntu-latest
+
+    steps:
+    # Step 1: Check out the repository
+    - name: Checkout Code
+      uses: actions/checkout@v3
+
+
+    #Google Cloud Authentication
+    - id: 'auth'
+      name: 'Authenticate to Google Cloud'
+      uses: 'google-github-actions/auth@v1'
+      with:
+          credentials_json: "${{ secrets.GCP_SA_KEY }}"
+
+    - name: 'Set up Cloud SDK'
+      uses: 'google-github-actions/setup-gcloud@v1'      
+
+     - name: Create Temporary GCP Key File
+        run: |
+          mkdir -p app
+          cp -r model app/
+          echo '${{ secrets.GCP_SA_KEY }}' | jq '.' > app/gcp-key.json
+
+      - name: Build and Push Docker Image
+        run: |
+          echo "Building Docker image..."
+          docker build -f DockerfileRetraining -t $REGISTRY/$IMAGE_NAME:latest .
+          echo "Pushing image to Artifact Registry..."
+          docker push $REGISTRY/$IMAGE_NAME:latest
+
+      - name: Run Model Training Script
+        run: |
+          echo "Starting model training..."
+          mkdir -p model_output
+          docker run --rm \
+          -v $(pwd)/model_output:/app/model_output \
+          -v $(pwd)/app:/app \
+          -e GOOGLE_APPLICATION_CREDENTIALS=/app/gcp-key.json \
+          $REGISTRY/$IMAGE_NAME:latest \
+          python model/retraining.py
+          echo "Model training completed."
+
+      - name: Verify Model Output
+        run: |
+          echo "Checking contents of model_output directory:"
+          ls -l model_output
+          if [ ! -f model_output/model.mar ]; then
+            echo "❌ Model file model.mar not found"
+            exit 1
+          fi
+
+      - name: Verify Model pt Output
+        run: |
+          echo "Checking contents of model_output directory:"
+          ls -l model_output
+          if [ ! -f model_output/best_model.pt ]; then
+            echo "❌ Model file best_model.pt not found"
+            exit 1
+          fi
+
+      - name: Upload Model to GCS
+        run: |
+           echo "Starting GCS operations..."
+           # Define paths
+           GCS_LATEST_PATH="gs://nih-dataset-mlops/Models/Latest"
+           GCS_VERSIONED_PATH="gs://nih-dataset-mlops/Models/Versions"
+           # List files in the Latest folder
+           echo "Listing files in the Latest folder:"
+           gsutil ls "$GCS_LATEST_PATH/"
+           # Move all files from 'Latest' to 'Versioned' with their name + timestamp
+           echo "Moving files from Latest to Versions..."
+           for file in $(gsutil ls "$GCS_LATEST_PATH/"); do
+             # Get file's actual timestamp from metadata
+             FILE_TIMESTAMP=$(gsutil stat "$file" | grep "Update time:" | awk '{print $3"-"$4}' | sed 's/[:-]//g' | sed 's/\..*//')
+             BASENAME=$(basename "$file")
+
+             # Print file details before moving
+             echo "Processing file: $BASENAME with timestamp: $FILE_TIMESTAMP"
+
+             # Move the file to the 'Versioned' folder with the actual timestamp in its name
+             gsutil mv "$file" "$GCS_VERSIONED_PATH/${BASENAME}_${FILE_TIMESTAMP}"
+           done
+           echo "Uploading model to GCS..."
+           TIMESTAMP=$(date +%Y%m%d%H%M%S)
+           gsutil cp model_output/model.mar gs://$GCS_BUCKET/Models/Latest/model.mar
+           gsutil cp model_output/best_model.pt gs://$GCS_BUCKET/Models/Latest/best_model.pt
+           echo "Model uploaded successfully."
+
+      - name: Notify on Success
+        if: success()
+        run: |
+          echo "✅ Pipeline completed successfully:"
+          echo "- Model trained"
+          echo "- Model uploaded to GCS bucket: $GCS_BUCKET"
+          echo "- Docker image pushed to: $REGISTRY/$IMAGE_NAME:latest"
+
+      - name: Notify on Failure
+        if: failure()
+        run: |
+          echo "❌ Pipeline failed:"
+          echo "- Check the logs above for detailed error messages"
+          echo "- Verify GCP permissions and credentials"
+          echo "- Ensure all required services are enabled"
+
+
+
+    # Step 4: Create a model in Vertex AI Model Registry
+    - name: Create or check for Existing Model Registry in Vertex AI
+      id: create-model
+      run: |
+        MODEL_NAME="gitmodel"
+        REGION="us-east1"
+        MODEL_URI="gs://nih-dataset-mlops/Testing"
+        echo "Checking if model exists in Vertex AI Model Registry..."
+        MODEL_ID=$(gcloud ai models list --region=$REGION --filter="display_name='gitmodel'" --format="value(name)")
+
+        if [ -z "$MODEL_ID" ]; then
+          echo "Model does not exist. Creating a new model..."
+          gcloud ai models upload \
+            --region=$REGION \
+            --display-name=$MODEL_NAME \
+            --artifact-uri=$MODEL_URI \
+            --container-image-uri=us-docker.pkg.dev/vertex-ai/prediction/pytorch-cpu.2-1:latest
+          MODEL_ID=$(gcloud ai models list --region=$REGION --filter="display_name='gitmodel'" --format="value(name)")
+        else
+          echo "Model already exists: $MODEL_ID"
+        fi
+        echo "MODEL_ID=$MODEL_ID"
+        echo "MODEL_ID=$MODEL_ID" >> $GITHUB_ENV
+
+      # Step 7: Create or Reuse an Endpoint
+    - name: Create or Reuse Endpoint
+      id: create_endpoint
+      run: |
+          echo "Checking for existing endpoint..."
+          REGION="us-east1"
+          ENDPOINT_NAME=$(gcloud ai endpoints list --region=$REGION --filter="display_name='gitmodelendpoint'" --format="value(name)")
+
+          if [ -z "$ENDPOINT_NAME" ]; then
+            echo "No existing endpoint found. Creating a new one..."
+            ENDPOINT_NAME=$(gcloud ai endpoints create --region=$REGION --display-name=gitmodelendpoint --format="value(name)")
+          fi
+
+          if [ -z "$ENDPOINT_NAME" ]; then
+            echo "ERROR: Failed to create or retrieve endpoint. Exiting workflow."
+            exit 1
+          fi
+
+          echo "ENDPOINT_NAME=$ENDPOINT_NAME" 
+          echo "Retrieved or created ENDPOINT_NAME: $ENDPOINT_NAME"
+          echo "ENDPOINT_NAME=$ENDPOINT_NAME" >> $GITHUB_ENV
+
+
+      # Step 8: Deploy Model to Endpoint 
+    - name: Deploy Model to Endpoint
+      run: |
+          echo "Deploying model $MODEL_ID to endpoint $ENDPOINT_NAME..."
+          gcloud ai endpoints deploy-model $ENDPOINT_NAME --region=us-east1 --model=$MODEL_ID --display-name=gitmodelendpoint-deployment --machine-type=n1-standard-2 --traffic-split=0=100   
+
+
+    # Step 9: Get Endpoint URI
+    - name: Get Endpoint URI
+      run: |
+        GCP_PROJECT_ID='812555529114'
+        REGION="us-east1"
+        ENDPOINT=$(gcloud ai endpoints list --region us-east1 --filter="displayName='gitmodelendpoint'" --format="value(name)")
+        echo "Endpoint URI: https://us-east1-aiplatform.googleapis.com/v1/projects/${GCP_PROJECT_ID}/locations/${REGION}/endpoints/$ENDPOINT_NAME:predict"
+
+    - name: Build Streamlit Docker Image
+      run: |
+          docker build \
+            -t ${{ env.REGISTRY }}/streamlit_app:${{ env.IMAGE_TAG }} \
+            -f Dockerfilestream .
+
+    - name: Push Streamlit Docker Image
+      run: |
+          docker push ${{ env.REGISTRY }}/streamlit_app:${{ env.IMAGE_TAG }}
+
+    - name: Deploy Streamlit App to Cloud Run
+      run: |
+          gcloud run deploy streamlit-app \
+            --image ${{ env.REGISTRY }}/streamlit_app:${{ env.IMAGE_TAG }} \
+            --platform managed \
+            --region us-east1 \
+            --allow-unauthenticated \
+            --port 8080 \
+            --cpu 1 \
+            --memory 512Mi \
+            --verbosity=debug \
+            --timeout 600s
+