CHORE: Flaky Setup + Optimization - Parallelize setup steps and reduce macOS CI job time

eng/pipelines/pr-validation-pipeline.yml

@@ -461,9 +461,21 @@ jobs:
 
 - job: PytestOnMacOS
   displayName: 'macOS x86_64'
+  # Colima + SQL Server container setup averages ~12.5 min but has a long tail
+  # (setup has been observed at 17-39 min). The ADO default job timeout is 60 min,
+  # which the setup tail plus tests plus the 20-min benchmark step can exceed,
+  # getting the job killed mid-step. Give the job enough headroom for the tail.
+  timeoutInMinutes: 90
   pool:
     vmImage: 'macos-latest'
 
+  variables:
+    # pip's on-disk download/wheel cache. Persisted across runs by the Cache@2
+    # task below so `pip install -r requirements.txt` reuses wheels instead of
+    # re-downloading them on every run. Exposed to script steps as the
+    # $PIP_CACHE_DIR environment variable, which pip honors automatically.
+    PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip-cache
+
   strategy:
     matrix:
       SQL2022:
@@ -482,63 +494,97 @@ jobs:
       addToPath: true
     displayName: 'Use Python $(pythonVersion) on macOS'
 
-  - script: |
-      brew update
-      # Uninstall existing CMake to avoid tap conflicts
+  - task: Cache@2
+    inputs:
+      key: 'pip | "$(Agent.OS)" | "$(pythonVersion)" | requirements.txt'
+      restoreKeys: |
+        pip | "$(Agent.OS)" | "$(pythonVersion)"
+        pip | "$(Agent.OS)"
+      path: $(PIP_CACHE_DIR)
+    displayName: 'Cache pip packages'
+
+  - script: |
+      # Single serialized Homebrew transaction. brew holds a global lock, so
+      # installing cmake, docker, and colima together is both faster than three
+      # separate `brew install` calls and lets the next step overlap the Colima
+      # VM boot + SQL setup with the Python build.
+      # No `brew update` here: the hosted macOS runner ships an up-to-date
+      # Homebrew and a full formula re-sync is ~1-3 min of pure setup overhead.
+      # Uninstall the preinstalled CMake first to avoid tap conflicts.
       brew uninstall cmake --ignore-dependencies || echo "CMake not installed or already removed"
-      # Install CMake from homebrew/core
-      brew install cmake
-    displayName: 'Install CMake'
-
-  - script: |
-      brew update
-      brew install docker colima
-
-      # Start Colima with extra resources
-      colima start --cpu 4 --memory 8 --disk 50
+      brew install cmake docker colima
+    displayName: 'Install build & container tooling (Homebrew)'
+
+  - script: |
+      set -e
+      # Overlap the ENTIRE container-side setup (Colima VM boot -> image pull ->
+      # container start -> SQL readiness) with the CPU-bound Python setup (pip
+      # install + pybind build). The two chains are independent until pytest, so
+      # we run the container chain in the background and the build in the
+      # foreground, then wait for the container before continuing.
+      #
+      # On the Intel macOS-15 hosted runner the linux/amd64 SQL Server image runs
+      # natively (no emulation); the dominant container costs are the Colima VM
+      # boot (~2-4 min) and the ~2 GB image pull, both of which are hidden behind
+      # the build here.
+      #
+      # NOTE: this deliberately merges what used to be separate steps (start
+      # Colima, pull+start SQL, install Python deps, build pybind). Splitting them
+      # back out restores the previous per-step timing breakdown.
+      setup_sql() {
+        echo "[sql] Starting Colima VM..."
+        colima start --cpu 4 --memory 8 --disk 50
+        docker context use colima >/dev/null || true
+        docker version
+
+        echo "[sql] Pulling SQL Server image..."
+        docker pull "$(sqlServerImage)"
+
+        echo "[sql] Starting SQL Server container..."
+        docker run \
+          --name sqlserver \
+          -e ACCEPT_EULA=Y \
+          -e MSSQL_SA_PASSWORD="${DB_PASSWORD}" \
+          -p 1433:1433 \
+          -d "$(sqlServerImage)"
+
+        echo "[sql] Waiting for SQL Server to accept connections..."
+        for i in {1..30}; do
+          if docker exec sqlserver \
+            /opt/mssql-tools18/bin/sqlcmd \
+            -S localhost -U SA -P "$DB_PASSWORD" \
+            -C -Q "SELECT 1"; then
+            echo "[sql] SQL Server is ready."
+            return 0
+          fi
+          sleep 2
+        done
+        echo "[sql] SQL Server did not become ready in time." >&2
+        return 1
+      }
 
-      # Optional: set Docker context (usually automatic)
-      docker context use colima >/dev/null || true
+      echo "Starting container setup (Colima + SQL Server) in the background..."
+      setup_sql > /tmp/sql_setup.log 2>&1 &
+      SQL_PID=$!
 
-      # Confirm Docker is operational
-      docker version
-      docker ps
-    displayName: 'Install and start Colima-based Docker'
+      echo "Installing Python dependencies (overlapped with container setup)..."
+      python -m pip install --upgrade pip
+      pip install -r requirements.txt
 
-  - script: |
-      # Pull and run SQL Server container
-      docker pull $(sqlServerImage)
-      docker run \
-        --name sqlserver \
-        -e ACCEPT_EULA=Y \
-        -e MSSQL_SA_PASSWORD="${DB_PASSWORD}" \
-        -p 1433:1433 \
-        -d $(sqlServerImage)
+      echo "Building pybind bindings (.so) (overlapped with container setup)..."
+      ( cd mssql_python/pybind && ./build.sh )
 
-      # Starting SQL Server container…
-      for i in {1..30}; do
-        docker exec sqlserver \
-          /opt/mssql-tools18/bin/sqlcmd \
-          -S localhost \
-          -U SA \
-          -P "$DB_PASSWORD" \
-          -C -Q "SELECT 1" && break
-        sleep 2
-      done
-    displayName: 'Pull & start SQL Server (Docker)'
+      echo "Waiting for container setup (Colima + SQL Server) to finish..."
+      if ! wait "$SQL_PID"; then
+        echo "Container setup failed:"
+        cat /tmp/sql_setup.log
+        exit 1
+      fi
+      cat /tmp/sql_setup.log
+    displayName: 'Build + start SQL Server (Colima boot & SQL setup overlapped with build)'
     env:
       DB_PASSWORD: $(DB_PASSWORD)
 
-  - script: |
-      python -m pip install --upgrade pip
-      pip install -r requirements.txt
-    displayName: 'Install Python dependencies'
-
-  - script: |
-      cd mssql_python/pybind
-      ./build.sh
-    displayName: 'Build pybind bindings (.so)'
-
   - template: steps/install-mssql-py-core.yml
     parameters:
       platform: unix