CI: cache compiled dependencies on Linux, macOS and Windows

Add GitHub Actions caching of compiled dependencies for Linux, macOS and
Windows wheel jobs. build.sh and build.ps1 now write a versions.txt stamp
and skip rebuilding when the cached dependencies match the requested
versions.

Assisted-by: Kimi Code

Author: jdavid

.github/workflows/wheels.yml

@@ -48,7 +48,7 @@ jobs:
           path: ReleaseNotes.md
 
   build_wheels:
-    name: Wheels for ${{ matrix.name }}
+    name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
     needs: sdist
     strategy:
@@ -66,17 +66,38 @@ jobs:
           os: macos-15
         - name: windows-x64
           os: windows-latest
+          libgit2_prefix: C:/libgit2_install_x86_64
         - name: windows-x86
           os: windows-latest
+          libgit2_prefix: C:/libgit2_install_x86
         - name: windows-arm64
           # https://github.com/actions/partner-runner-images#available-images
           os: windows-11-arm
+          libgit2_prefix: C:/libgit2_install_arm64
 
     steps:
       - uses: actions/setup-python@v7
         with:
           python-version: '3.13'
 
+      - name: Cache compiled dependencies
+        if: ${{ !startsWith(matrix.name, 'windows-') }}
+        uses: actions/cache@v4
+        with:
+          path: ci
+          key: deps-ci-${{ matrix.name }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: deps-ci-${{ matrix.name }}-
+
+      - name: Cache compiled dependencies (Windows)
+        if: ${{ startsWith(matrix.name, 'windows-') }}
+        uses: actions/cache@v4
+        with:
+          path: |
+            build/libgit2_src
+            ${{ matrix.libgit2_prefix }}
+          key: deps-ci-${{ matrix.name }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: deps-ci-${{ matrix.name }}-
+
       - name: Download sdist
         uses: actions/download-artifact@v8
         with:
@@ -98,7 +119,7 @@ jobs:
           path: ./wheelhouse/*.whl
 
   build_wheels_emulated_linux:
-    name: Wheels for ${{ matrix.name }}
+    name: ${{ matrix.name }}
     runs-on: ubuntu-24.04
     needs: sdist
     strategy:
@@ -121,6 +142,13 @@ jobs:
         with:
           platforms: ${{ matrix.qemu_platform }}
 
+      - name: Cache compiled dependencies
+        uses: actions/cache@v4
+        with:
+          path: ci
+          key: deps-ci-${{ matrix.name }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: deps-ci-${{ matrix.name }}-
+
       - name: Download sdist
         uses: actions/download-artifact@v8
         with:

AGENTS.md

@@ -242,7 +242,8 @@ GitHub Actions workflows live in `.github/workflows/`:
   `sh build.sh mypy`.
 - **`wheels.yml`** — Uses `cibuildwheel` (`~=3.3`) to build wheels for Linux
   (amd64, arm64, ppc64le and riscv64 via QEMU, musl), macOS (intel, arm64,
-  PyPy), and Windows (x64, x86, arm64). It also builds an sdist, runs a
+  PyPy), and Windows (x64, x86, arm64). Linux, macOS and Windows jobs cache
+  the compiled dependencies between runs. It also builds an sdist, runs a
   `twine check` (skipped on version tags), publishes to PyPI, and creates a
   GitHub Release on version tags (`v*`), with release notes parsed from
   `CHANGELOG.md` by `.github/workflows/parse_release_notes.py`.

CHANGELOG.md

@@ -1,5 +1,8 @@
 # 1.20.1 (UNRELEASED)
 
+- CI (Linux, macOS, Windows): cache compiled dependencies between wheel
+  builds, keyed by platform and dependency versions.
+
 - CI (macOS): build architecture-specific OpenSSL/libssh2/libgit2
   dependencies instead of universal binaries, producing smaller wheels.
 

build.ps1

@@ -1,23 +1,51 @@
 $ErrorActionPreference = 'Stop'
 
+$LIBGIT2_VERSION = $env:LIBGIT2_VERSION
+$LIBGIT2_SRC = $env:LIBGIT2_SRC
+if (-not $LIBGIT2_SRC) {
+    $LIBGIT2_SRC = "build/libgit2_src"
+}
+
+# Prefer CMAKE_INSTALL_PREFIX, then LIBGIT2, then the default Program Files location.
+$INSTALL_PREFIX = $env:CMAKE_INSTALL_PREFIX
+if (-not $INSTALL_PREFIX) {
+    $INSTALL_PREFIX = $env:LIBGIT2
+}
+if (-not $INSTALL_PREFIX) {
+    $INSTALL_PREFIX = "$env:ProgramFiles\libgit2"
+}
+
+# Use cached dependencies if they match the requested version.
+if ((Test-Path -Path "$INSTALL_PREFIX\versions.txt") -and $LIBGIT2_VERSION) {
+    $cached = Get-Content "$INSTALL_PREFIX\versions.txt"
+    $matches = $cached | Select-String "^LIBGIT2_VERSION=$LIBGIT2_VERSION$"
+    if ($matches) {
+        Write-Host "Using cached dependencies"
+        exit 0
+    }
+}
+
 if (!(Test-Path -Path "build")) {
     # in case the pygit2 package build/ workspace has not been created by cibuildwheel yet
     mkdir build
 }
-if (Test-Path -Path "$env:LIBGIT2_SRC") {
-    Set-Location "$env:LIBGIT2_SRC"
+if (Test-Path -Path "$LIBGIT2_SRC") {
+    Set-Location "$LIBGIT2_SRC"
     # for local runs, reuse build/libgit_src if it exists
     if (Test-Path -Path build) {
         # purge previous build env (likely for a different arch type)
         Remove-Item -Recurse -Force build
     }
     # ensure we are checked out to the right version
     git fetch --depth=1 --tags
-    git checkout "v$env:LIBGIT2_VERSION"
+    git checkout "v$LIBGIT2_VERSION"
 } else {
     # from a fresh run (like in CI)
-    git clone --depth=1 -b "v$env:LIBGIT2_VERSION" https://github.com/libgit2/libgit2.git $env:LIBGIT2_SRC
-    Set-Location "$env:LIBGIT2_SRC"
+    git clone --depth=1 -b "v$LIBGIT2_VERSION" https://github.com/libgit2/libgit2.git $LIBGIT2_SRC
+    Set-Location "$LIBGIT2_SRC"
 }
-cmake -B build -S . -DBUILD_TESTS=OFF
+cmake -B build -S . -DBUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX="$INSTALL_PREFIX"
 cmake --build build/ --config=Release --target install
+
+# Record version so the cache can be reused.
+"LIBGIT2_VERSION=$LIBGIT2_VERSION" | Set-Content "$INSTALL_PREFIX\versions.txt" -NoNewline

build.sh

@@ -82,6 +82,16 @@ if [ "$CIBUILDWHEEL" = "1" ]; then
             apk add --no-cache perl
         fi
     fi
+
+    # Use cached dependencies if they match the requested versions.
+    if [ -f ci/versions.txt ] && \
+       grep -q "^LIBGIT2_VERSION=$LIBGIT2_VERSION$" ci/versions.txt && \
+       grep -q "^LIBSSH2_VERSION=$LIBSSH2_VERSION$" ci/versions.txt && \
+       grep -q "^OPENSSL_VERSION=$OPENSSL_VERSION$" ci/versions.txt; then
+        echo "Using cached dependencies"
+        exit 0
+    fi
+
     rm -rf ci
     mkdir ci || true
     cd ci
@@ -199,6 +209,12 @@ if [ -n "$LIBGIT2_VERSION" ]; then
 fi
 
 if [ "$CIBUILDWHEEL" = "1" ]; then
+    # Record versions so the cache can be reused.
+    cat > $PREFIX/versions.txt <<EOF
+LIBGIT2_VERSION=$LIBGIT2_VERSION
+LIBSSH2_VERSION=$LIBSSH2_VERSION
+OPENSSL_VERSION=$OPENSSL_VERSION
+EOF
     if [ "$KERNEL" = "Darwin" ]; then
         echo "PREFIX        " $PREFIX
         echo "OPENSSL_PREFIX" $OPENSSL_PREFIX