Speed up integration test CI by packing the matrix and trimming per-job setup

.github/actions/go-lint-setup/action.yml

@@ -11,4 +11,7 @@ runs:
     # the Go cache to '/'; relocating it to /mnt here would force a cold vet.
     - name: Reclaim runner disk
       shell: bash
+      env:
+        # go vet over the full tree runs on '/' alongside the Go cache.
+        CI_FREE_DISK_MIN_GIB: 60
       run: bash ./.github/scripts/ci-free-disk.sh

.github/scripts/ci-free-disk.sh

@@ -14,6 +14,16 @@ echo "::group::Disk usage before reclaim"
 df -h / /mnt 2>/dev/null || df -h /
 echo "::endgroup::"
 
+# Removing the bundles below costs ~3.5 min of rm -rf. Skip it when the root
+# filesystem already has enough headroom (large runners ship with >80 GiB free).
+min_free_gib="${CI_FREE_DISK_MIN_GIB:-40}"
+avail_gib=$(df -BG --output=avail / | tail -1 | tr -dc '0-9')
+avail_gib=${avail_gib:-0}
+if [ "${avail_gib}" -ge "${min_free_gib}" ]; then
+  echo "Root filesystem has ${avail_gib}G free (>= ${min_free_gib}G); skipping reclaim."
+  exit 0
+fi
+
 # Large preinstalled bundles unused by Sei's Go/Docker/Node CI.
 junk=(
   /usr/share/dotnet

.github/workflows/execution-specs.yml

@@ -66,6 +66,9 @@ jobs:
       # See: https://github.com/actions/checkout/releases/tag/v7.0.0
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
       - name: Reclaim runner disk
+        env:
+          # Image build + seid build need more headroom than test shards.
+          CI_FREE_DISK_MIN_GIB: 60
         run: bash .github/scripts/ci-free-disk.sh
       - name: Relocate Docker data-root to /mnt
         run: bash .github/scripts/ci-relocate-docker-data-root.sh
@@ -151,6 +154,8 @@ jobs:
       # See: https://github.com/actions/checkout/releases/tag/v7.0.0
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
       - name: Reclaim runner disk
+        env:
+          CI_FREE_DISK_MIN_GIB: 40
         run: bash .github/scripts/ci-free-disk.sh
       - name: Relocate Docker data-root to /mnt
         run: bash .github/scripts/ci-relocate-docker-data-root.sh

.github/workflows/integration-test.yml

@@ -25,11 +25,40 @@ defaults:
   run:
     shell: bash
 
+env:
+  # Non-cone sparse-checkout pattern shared by every job that needs the source
+  # tree: everything except the darwin/aarch64 wasmvm archives, which CI
+  # (linux/amd64 only) never links and which make up ~500 MB of the tree.
+  SPARSE_CHECKOUT: |
+    /*
+    !/sei-wasmvm/internal/api/*darwin*
+    !/sei-wasmvm/internal/api/*aarch64*
+    !/sei-wasmvm/internal/api/*.dylib
+    !/sei-wasmd/x/wasm/artifacts/*/api/*darwin*
+    !/sei-wasmd/x/wasm/artifacts/*/api/*aarch64*
+    !/sei-wasmd/x/wasm/artifacts/*/api/*.dylib
+
 jobs:
   # Generate the integration-test matrix from integration-test-matrix.json.
   # Runs the full matrix on every event (pull_request, push, merge_group) — all
   # Autobahn coverage runs on every PR. The matrix is built here, in a job
   # output, because the `matrix` context is NOT available to a job-level `if:`.
+  #
+  # Packing policy for integration-test-matrix.json. Every row is its own job
+  # and all rows start at once, so wall-clock = prepare-cluster + the slowest
+  # row, while each extra row costs its ~2 min of fixed setup in runner-minutes.
+  # A row is a list of suites that share one cluster boot (same `env`) and run
+  # back-to-back against the same chain state. When adding a suite:
+  #   - Keep every row under ~8 min of test time. Join an existing row with a
+  #     matching `env` that has room; otherwise add a new row.
+  #   - Suites that assert absolute chain state (total supply, wasm code
+  #     counts, ...) run first in their row, and the row's `_comment` names the
+  #     assertion so a later suite is not appended ahead of it.
+  #   - Destructive suites stay alone: anything that kills, wipes, or
+  #     state-syncs a validator, disables a module, or runs a chain upgrade
+  #     leaves the cluster in a state no other suite should inherit.
+  #   - A suite that boots its own cluster (Autobahn Basic) is a separate job,
+  #     not a matrix row.
   set-matrix:
     name: Resolve integration test matrix
     runs-on: ubuntu-latest
@@ -39,6 +68,8 @@ jobs:
     steps:
       # See: https://github.com/actions/checkout/releases/tag/v7.0.0
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        with:
+          sparse-checkout: .github/workflows
       - name: Build matrix
         id: build
         run: |
@@ -67,10 +98,18 @@ jobs:
     steps:
       # See: https://github.com/actions/checkout/releases/tag/v7.0.0
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        with:
+          # CI only builds linux/amd64; leave the darwin/aarch64 wasmvm
+          # archives (~500 MB of the tree) out of the fetch.
+          sparse-checkout: ${{ env.SPARSE_CHECKOUT }}
+          sparse-checkout-cone-mode: false
       # Reclaim disk and push Docker + Go caches onto /mnt before the
       # image builds so the runner's small root filesystem does not fill up and
       # crash the runner mid-job.
       - name: Reclaim runner disk
+        env:
+          # Image build + seid build need more headroom than test jobs.
+          CI_FREE_DISK_MIN_GIB: 60
         run: bash .github/scripts/ci-free-disk.sh
       - name: Relocate Docker data-root to /mnt
         run: bash .github/scripts/ci-relocate-docker-data-root.sh
@@ -110,25 +149,6 @@ jobs:
           labels: sei-chain.ci-run-id=${{ github.run_id }}
           cache-from: type=registry,ref=${{ env.GHCR_LOCALNODE }}:cache
           cache-to: ${{ github.event_name == 'push' && format('type=registry,ref={0}:cache,mode=max', env.GHCR_LOCALNODE) || '' }}
-      - name: Build rpcnode image (cached)
-        uses: docker/build-push-action@v6
-        with:
-          context: docker/rpcnode
-          file: docker/rpcnode/Dockerfile
-          platforms: linux/amd64
-          push: false
-          load: true
-          tags: sei-chain/rpcnode
-          labels: sei-chain.ci-run-id=${{ github.run_id }}
-          cache-from: type=registry,ref=${{ env.GHCR_RPCNODE }}:cache
-          cache-to: ${{ github.event_name == 'push' && format('type=registry,ref={0}:cache,mode=max', env.GHCR_RPCNODE) || '' }}
-      # rpcnode is independent of the seid build, so push it now: push failures
-      # (retried below) surface before the expensive seid build rather than after.
-      - name: Push rpcnode image to GHCR for test jobs
-        run: |
-          source .github/scripts/docker-registry-retry.sh
-          docker tag sei-chain/rpcnode "${GHCR_RPCNODE}:${{ github.run_id }}"
-          push_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}"
       - uses: actions/setup-go@v6
         with:
           go-version: '1.25.6'
@@ -150,16 +170,75 @@ jobs:
           path: integration-build.tar.gz
           retention-days: 1
 
+  # rpcnode does not depend on seid, so it is built and pushed in parallel with
+  # prepare-cluster instead of on its critical path.
+  prepare-rpcnode:
+    name: Prepare integration test rpcnode image
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      packages: write
+      contents: read
+    env:
+      GHCR_RPCNODE: ghcr.io/sei-protocol/sei-chain-integration-test-rpcnode
+    steps:
+      # See: https://github.com/actions/checkout/releases/tag/v7.0.0
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        with:
+          sparse-checkout: |
+            .github
+            docker/rpcnode
+      - name: Login to Docker Hub
+        # See: https://github.com/docker/login-action/releases/tag/v4.2.0
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
+        if: env.DOCKERHUB_USERNAME != ''
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GitHub Container Registry
+        # See: https://github.com/docker/login-action/releases/tag/v4.2.0
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Docker Buildx
+        # See: https://github.com/docker/setup-buildx-action/releases/tag/v4.1.0
+        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
+      - name: Build rpcnode image (cached)
+        uses: docker/build-push-action@v6
+        with:
+          context: docker/rpcnode
+          file: docker/rpcnode/Dockerfile
+          platforms: linux/amd64
+          push: false
+          load: true
+          tags: sei-chain/rpcnode
+          labels: sei-chain.ci-run-id=${{ github.run_id }}
+          cache-from: type=registry,ref=${{ env.GHCR_RPCNODE }}:cache
+          cache-to: ${{ github.event_name == 'push' && format('type=registry,ref={0}:cache,mode=max', env.GHCR_RPCNODE) || '' }}
+      - name: Push rpcnode image to GHCR for test jobs
+        run: |
+          source .github/scripts/docker-registry-retry.sh
+          docker tag sei-chain/rpcnode "${GHCR_RPCNODE}:${{ github.run_id }}"
+          push_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}"
+
   integration-tests:
     name: Integration Test (${{ matrix.test.name }})
-    needs: [ set-matrix, prepare-cluster ]
+    needs: [ set-matrix, prepare-cluster, prepare-rpcnode ]
     runs-on: ubuntu-large
     timeout-minutes: 30
     permissions:
       packages: read
       contents: read
     env:
-      JOB_TIMEOUT: 28m
+      # Wall-clock budget for a whole row's scripts. Each `go test` invocation
+      # gets `-timeout $JOB_TIMEOUT` set to whatever is left of this budget, so
+      # a hang anywhere in the row hits Go's timeout (goroutine dump) before
+      # timeout-minutes cancels the job.
+      ROW_BUDGET_SECONDS: 1560
       GHCR_LOCALNODE: ghcr.io/sei-protocol/sei-chain-integration-test-localnode
       GHCR_RPCNODE: ghcr.io/sei-protocol/sei-chain-integration-test-rpcnode
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -175,6 +254,9 @@ jobs:
     steps:
       # See: https://github.com/actions/checkout/releases/tag/v7.0.0
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        with:
+          sparse-checkout: ${{ env.SPARSE_CHECKOUT }}
+          sparse-checkout-cone-mode: false
       - name: Reclaim runner disk
         run: bash .github/scripts/ci-free-disk.sh
       - name: Relocate Docker data-root to /mnt
@@ -184,9 +266,15 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: '22'
+          cache: npm
+          cache-dependency-path: |
+            contracts/package-lock.json
+            integration_test/dapp_tests/package-lock.json
+            integration_test/rpc_tests/package-lock.json
+            integration_test/precompile_tests/package-lock.json
 
       - name: Install jq
-        run: sudo apt-get install -y jq
+        run: command -v jq >/dev/null || sudo apt-get install -y jq
 
       - name: Set up Go
         uses: actions/setup-go@v6
@@ -210,8 +298,10 @@ jobs:
           # tarball so its copy does not sit on disk for the rest of the job.
           rm -f integration-build.tar.gz
           source .github/scripts/docker-registry-retry.sh
-          pull_with_retry "${GHCR_LOCALNODE}:${{ github.run_id }}"
-          pull_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}"
+          pull_with_retry "${GHCR_LOCALNODE}:${{ github.run_id }}" & localnode_pull=$!
+          pull_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}" & rpcnode_pull=$!
+          wait "${localnode_pull}"
+          wait "${rpcnode_pull}"
           docker tag "${GHCR_LOCALNODE}:${{ github.run_id }}" sei-chain/localnode
           docker tag "${GHCR_RPCNODE}:${{ github.run_id }}" sei-chain/rpcnode
           # Drop the redundant run-id-tagged references now that the images are
@@ -227,11 +317,11 @@ jobs:
 
       - name: Wait for docker cluster to start
         run: |
-          until [ $(cat build/generated/launch.complete |wc -l) = 4 ]
+          until [ $(cat build/generated/launch.complete 2>/dev/null |wc -l) = 4 ]
           do
-            sleep 10
+            sleep 2
           done
-          sleep 10
+          sleep 5
 
       - name: Start rpc node
         if: ${{ matrix.test.cluster != 'frozen-rpc-router' }}
@@ -244,11 +334,20 @@ jobs:
       - name: ${{ matrix.test.name }}
         run: |
           scripts=$(echo '${{ toJson(matrix.test.scripts) }}' | jq -r '.[]')
+          row_start=$SECONDS
           IFS=$'\n'   # change the internal field separator to newline
-          echo $scripts
           for script in $scripts
           do
-            bash -c "${script}"
+            remaining=$(( ROW_BUDGET_SECONDS - (SECONDS - row_start) ))
+            if [ "${remaining}" -lt 60 ]; then remaining=60; fi
+            export JOB_TIMEOUT="${remaining}s"
+            echo "::group::${script} (JOB_TIMEOUT=${JOB_TIMEOUT})"
+            if ! bash -c "${script}"; then
+              echo "::endgroup::"
+              echo "::error::suite failed: ${script}"
+              exit 1
+            fi
+            echo "::endgroup::"
           done
           unset IFS  # revert the internal field separator back to default
 
@@ -323,7 +422,7 @@ jobs:
     name: Integration Test (Autobahn Basic)
     runs-on: ubuntu-large
     timeout-minutes: 45
-    needs: prepare-cluster
+    needs: [ prepare-cluster, prepare-rpcnode ]
     permissions:
       packages: read
       contents: read
@@ -333,6 +432,9 @@ jobs:
     steps:
       # See: https://github.com/actions/checkout/releases/tag/v7.0.0
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        with:
+          sparse-checkout: ${{ env.SPARSE_CHECKOUT }}
+          sparse-checkout-cone-mode: false
       - name: Reclaim runner disk
         run: bash .github/scripts/ci-free-disk.sh
       - name: Relocate Docker data-root to /mnt
@@ -343,7 +445,7 @@ jobs:
         with:
           go-version: '1.25.6'
       - name: Install jq
-        run: sudo apt-get install -y jq
+        run: command -v jq >/dev/null || sudo apt-get install -y jq
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         if: env.DOCKERHUB_USERNAME != ''
@@ -369,8 +471,10 @@ jobs:
           # tarball so its copy does not sit on disk for the rest of the job.
           rm -f integration-build.tar.gz
           source .github/scripts/docker-registry-retry.sh
-          pull_with_retry "${GHCR_LOCALNODE}:${{ github.run_id }}"
-          pull_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}"
+          pull_with_retry "${GHCR_LOCALNODE}:${{ github.run_id }}" & localnode_pull=$!
+          pull_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}" & rpcnode_pull=$!
+          wait "${localnode_pull}"
+          wait "${rpcnode_pull}"
           docker tag "${GHCR_LOCALNODE}:${{ github.run_id }}" sei-chain/localnode
           docker tag "${GHCR_RPCNODE}:${{ github.run_id }}" sei-chain/rpcnode
           # Drop the redundant run-id-tagged references now that the images are
@@ -404,7 +508,7 @@ jobs:
   integration-test-check:
     name: Integration Test Check
     runs-on: ubuntu-latest
-    needs: [ prepare-cluster, integration-tests, autobahn-integration-tests ]
+    needs: [ prepare-cluster, prepare-rpcnode, integration-tests, autobahn-integration-tests ]
     if: always()
     steps:
       - name: Verify prepare and test jobs succeeded
@@ -413,6 +517,10 @@ jobs:
             echo "prepare-cluster job did not succeed (${{ needs.prepare-cluster.result }})"
             exit 1
           fi
+          if [[ "${{ needs.prepare-rpcnode.result }}" != "success" ]]; then
+            echo "prepare-rpcnode job did not succeed (${{ needs.prepare-rpcnode.result }})"
+            exit 1
+          fi
           if [[ "${{ needs.integration-tests.result }}" != "success" ]]; then
             echo "integration-tests job did not succeed (${{ needs.integration-tests.result }})"
             exit 1

.github/workflows/release-upgrade-test.yml

@@ -65,6 +65,9 @@ jobs:
           fetch-depth: 0
 
       - name: Reclaim runner disk
+        env:
+          # Builds two seid trees from a full-history checkout.
+          CI_FREE_DISK_MIN_GIB: 60
         run: bash .github/scripts/ci-free-disk.sh
 
       - name: Relocate Docker data-root to /mnt

Makefile

@@ -260,9 +260,12 @@ build-rpc-node:
 .PHONY: build-rpc-node
 
 # Integration-test CI: verify images pulled from GHCR by the matrix job.
-ensure-integration-ci-images:
+ensure-integration-ci-localnode-image:
 	@docker image inspect sei-chain/localnode >/dev/null 2>&1 || (echo "sei-chain/localnode image missing; pull from GHCR (see prepare-cluster job)" && exit 1)
-	@docker image inspect sei-chain/rpcnode >/dev/null 2>&1 || (echo "sei-chain/rpcnode image missing; pull from GHCR (see prepare-cluster job)" && exit 1)
+.PHONY: ensure-integration-ci-localnode-image
+
+ensure-integration-ci-images: ensure-integration-ci-localnode-image
+	@docker image inspect sei-chain/rpcnode >/dev/null 2>&1 || (echo "sei-chain/rpcnode image missing; pull from GHCR (see prepare-rpcnode job)" && exit 1)
 .PHONY: ensure-integration-ci-images
 
 # Build seid once inside the localnode image (integration-test prepare job).
@@ -281,7 +284,7 @@ build-seid-in-localnode: build-docker-node
 .PHONY: build-seid-in-localnode
 
 # CI variant: assumes localnode image already built by Buildx in prepare-cluster (skips docker build).
-build-seid-in-localnode-ci: ensure-integration-ci-images
+build-seid-in-localnode-ci: ensure-integration-ci-localnode-image
 	@mkdir -p build $(shell go env GOMODCACHE) $(shell go env GOCACHE)
 	@docker run --rm \
 		--user="$(shell id -u):$(shell id -g)" \

integration_test/precompile_tests/README.md

@@ -122,7 +122,8 @@ npm run precompile:ci              # == bash scripts/run-ci.sh
 ```
 
 `scripts/run-ci.sh` is the single orchestrator, used both locally and by the
-`EVM Precompiles` matrix entry in `.github/workflows/integration-test.yml`. It
+`EVM Module (Precompile & Endpoints) & Precompiles` matrix row in
+`.github/workflows/integration-test-matrix.json`. It
 assumes a Sei EVM RPC is already reachable (the workflow boots the 4-node
 cluster; locally you start it yourself), then end to end: