diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 846249c2..47e68cf3 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -461,9 +461,21 @@ jobs: - job: PytestOnMacOS displayName: 'macOS x86_64' + # Colima + SQL Server container setup averages ~12.5 min but has a long tail + # (setup has been observed at 17-39 min). The ADO default job timeout is 60 min, + # which the setup tail plus tests plus the 20-min benchmark step can exceed, + # getting the job killed mid-step. Give the job enough headroom for the tail. + timeoutInMinutes: 90 pool: vmImage: 'macos-latest' + variables: + # pip's on-disk download/wheel cache. Persisted across runs by the Cache@2 + # task below so `pip install -r requirements.txt` reuses wheels instead of + # re-downloading them on every run. Exposed to script steps as the + # $PIP_CACHE_DIR environment variable, which pip honors automatically. + PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip-cache + strategy: matrix: SQL2022: @@ -482,63 +494,97 @@ jobs: addToPath: true displayName: 'Use Python $(pythonVersion) on macOS' - - script: | - brew update - # Uninstall existing CMake to avoid tap conflicts + - task: Cache@2 + inputs: + key: 'pip | "$(Agent.OS)" | "$(pythonVersion)" | requirements.txt' + restoreKeys: | + pip | "$(Agent.OS)" | "$(pythonVersion)" + pip | "$(Agent.OS)" + path: $(PIP_CACHE_DIR) + displayName: 'Cache pip packages' + + - script: | + # Single serialized Homebrew transaction. brew holds a global lock, so + # installing cmake, docker, and colima together is both faster than three + # separate `brew install` calls and lets the next step overlap the Colima + # VM boot + SQL setup with the Python build. + # No `brew update` here: the hosted macOS runner ships an up-to-date + # Homebrew and a full formula re-sync is ~1-3 min of pure setup overhead. + # Uninstall the preinstalled CMake first to avoid tap conflicts. brew uninstall cmake --ignore-dependencies || echo "CMake not installed or already removed" - # Install CMake from homebrew/core - brew install cmake - displayName: 'Install CMake' - - - script: | - brew update - brew install docker colima - - # Start Colima with extra resources - colima start --cpu 4 --memory 8 --disk 50 + brew install cmake docker colima + displayName: 'Install build & container tooling (Homebrew)' + + - script: | + set -e + # Overlap the ENTIRE container-side setup (Colima VM boot -> image pull -> + # container start -> SQL readiness) with the CPU-bound Python setup (pip + # install + pybind build). The two chains are independent until pytest, so + # we run the container chain in the background and the build in the + # foreground, then wait for the container before continuing. + # + # On the Intel macOS-15 hosted runner the linux/amd64 SQL Server image runs + # natively (no emulation); the dominant container costs are the Colima VM + # boot (~2-4 min) and the ~2 GB image pull, both of which are hidden behind + # the build here. + # + # NOTE: this deliberately merges what used to be separate steps (start + # Colima, pull+start SQL, install Python deps, build pybind). Splitting them + # back out restores the previous per-step timing breakdown. + setup_sql() { + echo "[sql] Starting Colima VM..." + colima start --cpu 4 --memory 8 --disk 50 + docker context use colima >/dev/null || true + docker version + + echo "[sql] Pulling SQL Server image..." + docker pull "$(sqlServerImage)" + + echo "[sql] Starting SQL Server container..." + docker run \ + --name sqlserver \ + -e ACCEPT_EULA=Y \ + -e MSSQL_SA_PASSWORD="${DB_PASSWORD}" \ + -p 1433:1433 \ + -d "$(sqlServerImage)" + + echo "[sql] Waiting for SQL Server to accept connections..." + for i in {1..30}; do + if docker exec sqlserver \ + /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U SA -P "$DB_PASSWORD" \ + -C -Q "SELECT 1"; then + echo "[sql] SQL Server is ready." + return 0 + fi + sleep 2 + done + echo "[sql] SQL Server did not become ready in time." >&2 + return 1 + } - # Optional: set Docker context (usually automatic) - docker context use colima >/dev/null || true + echo "Starting container setup (Colima + SQL Server) in the background..." + setup_sql > /tmp/sql_setup.log 2>&1 & + SQL_PID=$! - # Confirm Docker is operational - docker version - docker ps - displayName: 'Install and start Colima-based Docker' + echo "Installing Python dependencies (overlapped with container setup)..." + python -m pip install --upgrade pip + pip install -r requirements.txt - - script: | - # Pull and run SQL Server container - docker pull $(sqlServerImage) - docker run \ - --name sqlserver \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="${DB_PASSWORD}" \ - -p 1433:1433 \ - -d $(sqlServerImage) + echo "Building pybind bindings (.so) (overlapped with container setup)..." + ( cd mssql_python/pybind && ./build.sh ) - # Starting SQL Server container… - for i in {1..30}; do - docker exec sqlserver \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$DB_PASSWORD" \ - -C -Q "SELECT 1" && break - sleep 2 - done - displayName: 'Pull & start SQL Server (Docker)' + echo "Waiting for container setup (Colima + SQL Server) to finish..." + if ! wait "$SQL_PID"; then + echo "Container setup failed:" + cat /tmp/sql_setup.log + exit 1 + fi + cat /tmp/sql_setup.log + displayName: 'Build + start SQL Server (Colima boot & SQL setup overlapped with build)' env: DB_PASSWORD: $(DB_PASSWORD) - - script: | - python -m pip install --upgrade pip - pip install -r requirements.txt - displayName: 'Install Python dependencies' - - - script: | - cd mssql_python/pybind - ./build.sh - displayName: 'Build pybind bindings (.so)' - - template: steps/install-mssql-py-core.yml parameters: platform: unix