build: add performance profiling pipeline

eng/ci/public-build.yml

@@ -86,6 +86,15 @@ extends:
           parameters:
             PROJECT_DIRECTORY: 'workers'
             PoolName: 1es-pool-azfunc-public
+
+    - stage: RunWorkerProfiling
+      dependsOn: CheckPythonWorkerDependencies
+      jobs:
+        - template: /eng/templates/jobs/ci-profiling.yml@self
+          parameters:
+            PROJECT_DIRECTORY: 'workers'
+            PoolName: 1es-pool-azfunc-public
+
     - stage: RunWorkerEmulatorTests
       dependsOn: CheckPythonWorkerDependencies
       jobs:

eng/scripts/download-baseline-profile.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Download baseline performance profile from dev branch builds
+# This script is used in CI to fetch baseline profiles for comparison
+
+set -e
+
+ORGANIZATION="${1:-}"
+PROJECT="${2:-}"
+PIPELINE_ID="${3:-}"
+ARTIFACT_NAME="${4:-}"
+OUTPUT_DIR="${5:-baseline_results}"
+ACCESS_TOKEN="${SYSTEM_ACCESSTOKEN:-}"
+
+if [ -z "$ORGANIZATION" ] || [ -z "$PROJECT" ] || [ -z "$PIPELINE_ID" ] || [ -z "$ARTIFACT_NAME" ]; then
+    echo "Usage: $0 <organization> <project> <pipeline_id> <artifact_name> [output_dir]"
+    echo "Example: $0 myorg myproject 123 performance-profile-python-3.13"
+    exit 1
+fi
+
+echo "Searching for baseline artifacts from dev branch..."
+echo "Organization: $ORGANIZATION"
+echo "Project: $PROJECT"
+echo "Pipeline ID: $PIPELINE_ID"
+echo "Artifact: $ARTIFACT_NAME"
+
+mkdir -p "$OUTPUT_DIR"
+
+# Azure DevOps REST API endpoint
+API_VERSION="7.1-preview.7"
+BUILDS_API="https://dev.azure.com/${ORGANIZATION}/${PROJECT}/_apis/build/builds"
+
+# Get the latest successful build from dev branch
+echo "Fetching latest successful build from dev branch..."
+
+BUILD_RESPONSE=$(curl -s -u ":${ACCESS_TOKEN}" \
+    "${BUILDS_API}?definitions=${PIPELINE_ID}&branchName=refs/heads/dev&statusFilter=completed&resultFilter=succeeded&\$top=1&api-version=${API_VERSION}")
+
+BUILD_ID=$(echo "$BUILD_RESPONSE" | grep -o '"id":[0-9]*' | head -1 | grep -o '[0-9]*')
+
+if [ -z "$BUILD_ID" ]; then
+    echo "⚠ No successful builds found on dev branch"
+    exit 1
+fi
+
+echo "✓ Found build ID: $BUILD_ID"
+
+# Get artifacts for this build
+echo "Fetching artifacts from build $BUILD_ID..."
+
+ARTIFACTS_API="${BUILDS_API}/${BUILD_ID}/artifacts"
+ARTIFACTS_RESPONSE=$(curl -s -u ":${ACCESS_TOKEN}" \
+    "${ARTIFACTS_API}?api-version=${API_VERSION}")
+
+# Check if artifact exists
+if echo "$ARTIFACTS_RESPONSE" | grep -q "\"name\":\"${ARTIFACT_NAME}\""; then
+    echo "✓ Found artifact: $ARTIFACT_NAME"
+
+    # Get download URL
+    DOWNLOAD_URL=$(echo "$ARTIFACTS_RESPONSE" | grep -A 10 "\"name\":\"${ARTIFACT_NAME}\"" | grep -o '"downloadUrl":"[^"]*"' | cut -d'"' -f4)
+
+    if [ -n "$DOWNLOAD_URL" ]; then
+        echo "Downloading artifact..."
+
+        # Download and extract artifact
+        curl -s -u ":${ACCESS_TOKEN}" -o "${OUTPUT_DIR}/artifact.zip" "$DOWNLOAD_URL"
+
+        if [ -f "${OUTPUT_DIR}/artifact.zip" ]; then
+            unzip -q -o "${OUTPUT_DIR}/artifact.zip" -d "$OUTPUT_DIR"
+            rm -f "${OUTPUT_DIR}/artifact.zip"
+
+            echo "✓ Baseline artifact downloaded to $OUTPUT_DIR"
+
+            # List downloaded files
+            echo "Downloaded files:"
+            ls -lh "$OUTPUT_DIR"
+
+            exit 0
+        else
+            echo "✗ Failed to download artifact"
+            exit 1
+        fi
+    else
+        echo "✗ Could not find download URL for artifact"
+        exit 1
+    fi
+else
+    echo "⚠ Artifact not found: $ARTIFACT_NAME"
+    echo "This might be the first profiling run on dev branch"
+    exit 1
+fi

eng/templates/jobs/ci-profiling.yml

@@ -0,0 +1,192 @@
+parameters:
+  PROJECT_DIRECTORY: 'workers'
+
+jobs:
+  - job: "ProfilingPython"
+    displayName: "Performance Profiling - Proxy Worker"
+
+    pool:
+      name: ${{ parameters.PoolName }}
+      image: 1es-ubuntu-22.04
+      os: linux
+
+    strategy:
+      matrix:
+        Python313:
+          PYTHON_VERSION: '3.13'
+        Python314:
+          PYTHON_VERSION: '3.14'
+
+    steps:
+      - bash: |
+          echo "Disk space before cleanup:"
+          df -h
+
+          # Clean up Docker resources
+          docker system prune -af --volumes || true
+
+          # Clean pip cache
+          pip cache purge || true
+
+          # Clean apt cache
+          sudo apt-get clean || true
+          sudo rm -rf /var/lib/apt/lists/* || true
+
+          echo "Disk space after cleanup:"
+          df -h
+        displayName: 'Free disk space'
+
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: $(PYTHON_VERSION)
+
+      - task: UseDotNet@2
+        displayName: 'Install .NET 8'
+        inputs:
+          version: 8.0.x
+
+      - bash: |
+          chmod +x eng/scripts/install-dependencies.sh
+          chmod +x eng/scripts/test-setup.sh
+
+          eng/scripts/install-dependencies.sh $(PYTHON_VERSION) ${{ parameters.PROJECT_DIRECTORY }}
+          eng/scripts/test-setup.sh
+        displayName: 'Install dependencies'
+        condition: and(eq(variables.isSdkRelease, false), eq(variables.isExtensionsRelease, false), eq(variables['USETESTPYTHONSDK'], false), eq(variables['USETESTPYTHONEXTENSIONS'], false))
+
+      - bash: |
+          echo "Running performance profiling for proxy_worker..."
+
+          # Create profile output directory
+          mkdir -p profile_results
+
+          # Run tests with profiling enabled
+          python -m pytest -v --profile-dispatcher \
+                --profile-output-dir=profile_results \
+                --profile-show-report \
+                tests/unittest_proxy/test_dispatcher.py
+
+          echo "Performance profiling completed"
+
+          # Display profile summary
+          if [ -f profile_results/dispatcher_session_profile.txt ]; then
+              echo "=== Performance Profile Summary ==="
+              head -n 50 profile_results/dispatcher_session_profile.txt
+          fi
+        displayName: "Run Performance Profiling"
+        env:
+          PYTHON_VERSION: $(PYTHON_VERSION)
+        workingDirectory: $(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}
+
+      - bash: |
+          if [ -f profile_results/dispatcher_session_profile.json ]; then
+              echo "Checking for baseline profile to compare against..."
+
+              # Check if this is a PR build
+              if [ -n "$(System.PullRequest.PullRequestId)" ]; then
+                  echo "=========================================="
+                  echo "PR Build Detected - Running Comparison"
+                  echo "PR #$(System.PullRequest.PullRequestId)"
+                  echo "=========================================="
+
+                  # Download baseline from dev branch
+                  chmod +x eng/scripts/download-baseline-profile.sh
+
+                  if eng/scripts/download-baseline-profile.sh \
+                      "$(System.TeamFoundationCollectionUri)" \
+                      "$(System.TeamProject)" \
+                      "$(System.DefinitionId)" \
+                      "performance-profile-python-$(PYTHON_VERSION)" \
+                      "baseline_results"; then
+
+                      echo ""
+                      echo "✓ Baseline downloaded successfully"
+
+                      # Find the baseline JSON file (might be in subdirectory)
+                      BASELINE_FILE=$(find baseline_results -name "dispatcher_session_profile.json" | head -1)
+
+                      if [ -n "$BASELINE_FILE" ]; then
+                          echo "Running performance comparison..."
+
+                          python tests/unittest_proxy/compare_profiles.py \
+                              "$BASELINE_FILE" \
+                              profile_results/dispatcher_session_profile.json \
+                              --output profile_results/comparison_report.txt \
+                              --show-all
+
+                          echo ""
+                          echo "╔════════════════════════════════════════════════════════════╗"
+                          echo "║     PERFORMANCE COMPARISON REPORT (vs dev branch)         ║"
+                          echo "╚════════════════════════════════════════════════════════════╝"
+                          cat profile_results/comparison_report.txt
+                          echo ""
+
+                          # Check for regressions and create warnings
+                          REGRESSION_COUNT=$(grep -c "REGRESSION" profile_results/comparison_report.txt || echo "0")
+
+                          if [ "$REGRESSION_COUNT" -gt 0 ]; then
+                              echo "##vso[task.logissue type=warning]⚠️  $REGRESSION_COUNT performance regression(s) detected!"
+                              echo "##vso[task.logissue type=warning]Review the comparison report in the artifacts for details."
+                          else
+                              echo "✓ No performance regressions detected"
+                          fi
+
+                          # Upload comparison to build summary
+                          echo "##vso[task.uploadsummary]$(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}/profile_results/comparison_report.txt"
+                      else
+                          echo "⚠ Could not find baseline JSON file in downloaded artifact"
+                      fi
+                  else
+                      echo ""
+                      echo "⚠ Could not download baseline - comparison skipped"
+                      echo "This might be the first profiling run, or the baseline is unavailable"
+                  fi
+              else
+                  echo "=========================================="
+                  echo "Main/Dev Branch Build"
+                  echo "=========================================="
+                  echo "This profile will serve as baseline for future PR comparisons"
+                  echo "Artifact: performance-profile-python-$(PYTHON_VERSION)"
+              fi
+          else
+              echo "Skipping baseline comparison (no profile results)"
+          fi
+        displayName: "Compare Performance Against Baseline"
+        condition: succeededOrFailed()
+        env:
+          PYTHON_VERSION: $(PYTHON_VERSION)
+          SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+        workingDirectory: $(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}
+
+      - bash: |
+          if [ -d profile_results ]; then
+              echo "Uploading performance profile artifact..."
+              echo "##vso[artifact.upload artifactname=performance-profile-python-$(PYTHON_VERSION)]$(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}/profile_results"
+              echo "Performance profile artifact uploaded successfully"
+          else
+              echo "Skipping artifact upload (no profile results)"
+          fi
+        displayName: "Upload Performance Profile Artifact"
+        condition: succeededOrFailed()
+        env:
+          PYTHON_VERSION: $(PYTHON_VERSION)
+        workingDirectory: $(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}
+
+      - bash: |
+          echo "Cleaning up after profiling..."
+          # Remove build artifacts and caches
+          rm -rf $(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}/build || true
+          rm -rf $(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}/.pytest_cache || true
+          rm -rf $(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}/profile_results || true
+          rm -rf $(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}/baseline_results || true
+
+          # Clean Docker resources
+          docker system prune -f || true
+
+          # Clean pip cache
+          pip cache purge || true
+
+          echo "Disk space after cleanup:"
+          df -h
+        displayName: 'Cleanup after profiling'
+        condition: always()