Dheeraj model

.github/workflows/model_pipeline.yaml

@@ -1,126 +1,121 @@
-
-name: Model Pipeline v1
-
-on:
-  push:
-    branches:
-      - dheeraj_model
-  pull_request:
-    branches:
-      - dheeraj_model
-  workflow_dispatch:
-
-env:
-  GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} # Your GCP Project ID
-  GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} # The service account key stored as a secret
-  IMAGE_NAME: "hello-world-image"
-  REGISTRY: "us-central1-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/ml-models"
-
-jobs:
-  model_pipeline:
-    runs-on: ubuntu-latest
-
-    steps:
-      # Step 1: Checkout repository
-      - name: Checkout Repository
-        uses: actions/checkout@v2
-
-      # Step 2: Clean docker image
-      - name: Clean up Docker to free space
-        run: docker system prune -af --volumes
-
-      # Step 3: Set up Docker Buildx
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      # Step 4: Login to GCP Artifact Registry
-      - name: Login to GCP Artifact Registry
-        uses: docker/login-action@v1
-        with:
-          registry: us-central1-docker.pkg.dev
-          username: _json_key
-          password: ${{ secrets.GCP_SA_KEY }}
-
-      # Step 5: Build and Push Docker Image
-      - name: Build and Push Docker Image
-        run: |
-          docker build -f Dockerfilemodel -t $REGISTRY/$IMAGE_NAME:latest .
-          docker push $REGISTRY/$IMAGE_NAME:latest
-
-      # Step 6: Run Model with Raytuner from Docker Image
-      - name: Run Model development file
-        run: |
-          docker run --rm $REGISTRY/$IMAGE_NAME:latest python model/model_development_refined.py
-
-      - name: Notify on success
-        if: success()
-        run: echo "Unit tests passed successfully"
-
-      - name: Notify on failure
-        if: failure()
-        run: echo "Unit tests failed"
-
-      # - name: Set up Python
-      #   uses: actions/setup-python@v2
-      #   with:
-      #     python-version: "3.8"
-
-      # - name: Install Dependencies
-      #   run: |
-      #     python -m pip install --upgrade pip
-      #     pip install -r requirements.txt
-
-      # - name: Run Model Training Script
-      #   env:
-      #     GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }}
-      #   run: |
-      #     python train_model.py
-
-      # - name: Validate Model
-      #   run: |
-      #     python validate_model.py
-
-      # - name: Check Validation Results
-      #   run: |
-      #     ACCURACY=$(jq .accuracy validation_metrics.json)
-      #     if (( $(echo "$ACCURACY < 0.85" | bc -l) )); then
-      #       echo "Model validation failed!"
-      #       exit 1
-      #     fi
-
-      # - name: Run Bias Detection
-      #   run: |
-      #     python detect_bias.py
-
-      # - name: Check Bias Results
-      #   run: |
-      #     BIAS=$(jq .bias_score bias_metrics.json)
-      #     if (( $(echo "$BIAS > 0.2" | bc -l) )); then
-      #       echo "Significant bias detected!"
-      #       exit 1
-      #     fi
-
-      # - name: Push Model to Artifact Registry
-      #   run: |
-      #     gcloud auth activate-service-account --key-file=$GCP_CREDENTIALS
-      #     gsutil cp model.pkl gs://my-model-bucket/models/model-$(date +%Y%m%d%H%M%S).pkl
-
-      # - name: Notify on Slack
-      #   uses: slackapi/slack-github-action@v1.23
-      #   with:
-      #     payload: |
-      #       {
-      #         "channel": "#mlopsgrp10",
-      #         "text": "Model training pipeline completed with status ${{ job.status }}."
-      #       }
-      #   env:
-      #     SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
-
-      # - name: Compare with Previous Model
-      #   run: |
-      #     python compare_models.py
-
-      # - name: Rollback if New Model is Worse
-      #   if: failure()
-      #   run: |
-      #     gsutil cp gs://my-model-bucket/models/previous_model.pkl model.pkl
+# name: Deploy model to Vertex AI
+# on:
+#   push:
+#     branches:
+#       - dhanush_model
+#   pull_request:
+#     branches:
+#       - dhanush_model
+#   workflow_dispatch:
+# env:
+#   GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+#   GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+#   IMAGE_NAME: "ml-model-image"
+#   REGISTRY: "us-east1-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/ml-models-new"
+#   GCS_BUCKET: "nih-dataset-mlops"
+
+# jobs:
+#   deploy-to-vertex-ai:
+#     runs-on: ubuntu-latest
+
+#     steps:
+#     # Step 1: Check out the repository
+#     - name: Checkout Code
+#       uses: actions/checkout@v3
+
+
+#     #Google Cloud Authentication
+#     - id: 'auth'
+#       name: 'Authenticate to Google Cloud'
+#       uses: 'google-github-actions/auth@v1'
+#       with:
+#           credentials_json: "${{ secrets.GCP_SA_KEY }}"
+
+#     - name: 'Set up Cloud SDK'
+#       uses: 'google-github-actions/setup-gcloud@v1'      
+
+#     # Step 4: Create a model in Vertex AI Model Registry
+#     - name: Create or check for Existing Model Registry in Vertex AI
+#       id: create-model
+#       run: |
+#         MODEL_NAME="gitmodel"
+#         REGION="us-east1"
+#         MODEL_URI="gs://nih-dataset-mlops/Testing"
+#         echo "Checking if model exists in Vertex AI Model Registry..."
+#         MODEL_ID=$(gcloud ai models list --region=$REGION --filter="display_name='gitmodel'" --format="value(name)")
+
+#         if [ -z "$MODEL_ID" ]; then
+#           echo "Model does not exist. Creating a new model..."
+#           gcloud ai models upload \
+#             --region=$REGION \
+#             --display-name=$MODEL_NAME \
+#             --artifact-uri=$MODEL_URI \
+#             --container-image-uri=us-docker.pkg.dev/vertex-ai/prediction/pytorch-cpu.2-1:latest
+#           MODEL_ID=$(gcloud ai models list --region=$REGION --filter="display_name='gitmodel'" --format="value(name)")
+#         else
+#           echo "Model already exists: $MODEL_ID"
+#         fi
+#         echo "MODEL_ID=$MODEL_ID"
+#         echo "MODEL_ID=$MODEL_ID" >> $GITHUB_ENV
+
+#       # Step 7: Create or Reuse an Endpoint
+#     - name: Create or Reuse Endpoint
+#       id: create_endpoint
+#       run: |
+#           echo "Checking for existing endpoint..."
+#           REGION="us-east1"
+#           ENDPOINT_NAME=$(gcloud ai endpoints list --region=$REGION --filter="display_name='gitmodelendpoint'" --format="value(name)")
+
+#           if [ -z "$ENDPOINT_NAME" ]; then
+#             echo "No existing endpoint found. Creating a new one..."
+#             ENDPOINT_NAME=$(gcloud ai endpoints create --region=$REGION --display-name=gitmodelendpoint --format="value(name)")
+#           fi
+
+#           if [ -z "$ENDPOINT_NAME" ]; then
+#             echo "ERROR: Failed to create or retrieve endpoint. Exiting workflow."
+#             exit 1
+#           fi
+
+#           echo "ENDPOINT_NAME=$ENDPOINT_NAME" 
+#           echo "Retrieved or created ENDPOINT_NAME: $ENDPOINT_NAME"
+#           echo "ENDPOINT_NAME=$ENDPOINT_NAME" >> $GITHUB_ENV
+
+
+#       # Step 8: Deploy Model to Endpoint 
+#     - name: Deploy Model to Endpoint
+#       run: |
+#           echo "Deploying model $MODEL_ID to endpoint $ENDPOINT_NAME..."
+#           gcloud ai endpoints deploy-model $ENDPOINT_NAME --region=us-east1 --model=$MODEL_ID --display-name=gitmodelendpoint-deployment --machine-type=n1-standard-2 --traffic-split=0=100   
+
+
+#     # Step 9: Get Endpoint URI
+#     - name: Get Endpoint URI
+#       run: |
+#         GCP_PROJECT_ID='812555529114'
+#         REGION="us-east1"
+#         ENDPOINT=$(gcloud ai endpoints list --region us-east1 --filter="displayName='gitmodelendpoint'" --format="value(name)")
+#         echo "Endpoint URI: https://us-east1-aiplatform.googleapis.com/v1/projects/${GCP_PROJECT_ID}/locations/${REGION}/endpoints/$ENDPOINT_NAME:predict"
+
+#     - name: Build Streamlit Docker Image
+#       run: |
+#           docker build \
+#             -t ${{ env.REGISTRY }}/streamlit_app:${{ env.IMAGE_TAG }} \
+#             -f Dockerfilestream .
+
+#     - name: Push Streamlit Docker Image
+#       run: |
+#           docker push ${{ env.REGISTRY }}/streamlit_app:${{ env.IMAGE_TAG }}
+
+#     - name: Deploy Streamlit App to Cloud Run
+#       run: |
+#           gcloud run deploy streamlit-app \
+#             --image ${{ env.REGISTRY }}/streamlit_app:${{ env.IMAGE_TAG }} \
+#             --platform managed \
+#             --region us-east1 \
+#             --allow-unauthenticated \
+#             --port 8080 \
+#             --cpu 1 \
+#             --memory 512Mi \
+#             --verbosity=debug \
+#             --timeout 600s
+