diff --git a/.github/actions/go-lint-setup/action.yml b/.github/actions/go-lint-setup/action.yml index 2dd40272ce..ed4747bd36 100644 --- a/.github/actions/go-lint-setup/action.yml +++ b/.github/actions/go-lint-setup/action.yml @@ -11,4 +11,7 @@ runs: # the Go cache to '/'; relocating it to /mnt here would force a cold vet. - name: Reclaim runner disk shell: bash + env: + # go vet over the full tree runs on '/' alongside the Go cache. + CI_FREE_DISK_MIN_GIB: 60 run: bash ./.github/scripts/ci-free-disk.sh diff --git a/.github/scripts/ci-free-disk.sh b/.github/scripts/ci-free-disk.sh index 09eea1fa01..817e638bb3 100755 --- a/.github/scripts/ci-free-disk.sh +++ b/.github/scripts/ci-free-disk.sh @@ -14,6 +14,16 @@ echo "::group::Disk usage before reclaim" df -h / /mnt 2>/dev/null || df -h / echo "::endgroup::" +# Removing the bundles below costs ~3.5 min of rm -rf. Skip it when the root +# filesystem already has enough headroom (large runners ship with >80 GiB free). +min_free_gib="${CI_FREE_DISK_MIN_GIB:-40}" +avail_gib=$(df -BG --output=avail / | tail -1 | tr -dc '0-9') +avail_gib=${avail_gib:-0} +if [ "${avail_gib}" -ge "${min_free_gib}" ]; then + echo "Root filesystem has ${avail_gib}G free (>= ${min_free_gib}G); skipping reclaim." + exit 0 +fi + # Large preinstalled bundles unused by Sei's Go/Docker/Node CI. junk=( /usr/share/dotnet diff --git a/.github/workflows/execution-specs.yml b/.github/workflows/execution-specs.yml index 5ea7d7693a..950ab65d53 100644 --- a/.github/workflows/execution-specs.yml +++ b/.github/workflows/execution-specs.yml @@ -66,6 +66,9 @@ jobs: # See: https://github.com/actions/checkout/releases/tag/v7.0.0 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - name: Reclaim runner disk + env: + # Image build + seid build need more headroom than test shards. + CI_FREE_DISK_MIN_GIB: 60 run: bash .github/scripts/ci-free-disk.sh - name: Relocate Docker data-root to /mnt run: bash .github/scripts/ci-relocate-docker-data-root.sh @@ -151,6 +154,8 @@ jobs: # See: https://github.com/actions/checkout/releases/tag/v7.0.0 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - name: Reclaim runner disk + env: + CI_FREE_DISK_MIN_GIB: 40 run: bash .github/scripts/ci-free-disk.sh - name: Relocate Docker data-root to /mnt run: bash .github/scripts/ci-relocate-docker-data-root.sh diff --git a/.github/workflows/integration-test-matrix.json b/.github/workflows/integration-test-matrix.json index bc44a7ad94..f52b55505f 100644 --- a/.github/workflows/integration-test-matrix.json +++ b/.github/workflows/integration-test-matrix.json @@ -1,205 +1,204 @@ [ { - "name": "Wasm Module", + "_comment": "Packing policy (which suites may share a row, ordering, isolation) is documented above the set-matrix job in integration-test.yml.", + "name": "EVM GIGA Mixed (Determinism)", + "env": "GIGA_STORAGE=true", "scripts": [ - "docker exec sei-node-0 integration_test/contracts/deploy_timelocked_token_contract.sh", - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestWasmModuleCore", - "docker exec sei-node-0 integration_test/contracts/deploy_timelocked_token_contract.sh", - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestWasmModuleEmergencyWithdraw" + "./integration_test/evm_module/scripts/evm_giga_mixed_tests.sh" ] }, { - "name": "Mint & Staking & Bank Module", + "_comment": "Suites: EVM Module (Compat), EVM RPC .io/.iox (spec fixtures).", + "name": "EVM Module (Compat) & RPC .io/.iox", + "env": "GIGA_STORAGE=true", "scripts": [ - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run \"TestMintModule|TestStakingModule|TestBankModule\"" + "./integration_test/evm_module/scripts/evm_compat_tests.sh", + "./integration_test/evm_module/scripts/evm_rpc_tests.sh" ] }, { - "name": "Autobahn Mint & Staking & Bank Module", - "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true", + "_comment": "Suites: EVM Module (Precompile & Endpoints), EVM Precompiles.", + "name": "EVM Module (Precompile & Endpoints) & Precompiles", + "env": "GIGA_STORAGE=true", "scripts": [ - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run \"TestMintModule|TestStakingModule|TestAutobahnBankModule\"" + "./integration_test/evm_module/scripts/evm_precompile_tests.sh", + "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh", + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestSeiDBFlatKV", + "docker exec sei-node-0 integration_test/contracts/verify_flatkv_evm_store.sh", + "./integration_test/precompile_tests/scripts/run-ci.sh" ] }, { - "name": "Gov & Oracle & Authz Module", + "_comment": "Suites: Autobahn EVM Module (Compat), Autobahn EVM GIGA Module.", + "name": "Autobahn EVM Module (Compat) & GIGA", + "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true", "scripts": [ - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run \"TestGovModule|TestOracleModule|TestAuthzModule\"" + "./integration_test/evm_module/scripts/evm_compat_tests.sh", + "./integration_test/evm_module/scripts/evm_giga_tests.sh" ] }, { - "name": "Autobahn Gov & Oracle & Authz Module", + "name": "Autobahn EVM Module (Precompile & Endpoints)", "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true", "scripts": [ - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run \"TestGovModule|TestOracleModule|TestAuthzModule\"" + "./integration_test/evm_module/scripts/evm_precompile_tests.sh", + "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh", + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestSeiDBFlatKV", + "docker exec sei-node-0 integration_test/contracts/verify_flatkv_evm_store.sh" ] }, { - "name": "Chain Operation Test", + "name": "EVM Interoperability (Pointer Tests)", "env": "GIGA_STORAGE=true", "scripts": [ - "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done", - "until [[ $(docker exec sei-rpc-node build/seid status |jq -M -r .SyncInfo.latest_block_height) -gt 10 ]]; do sleep 10; done", - "echo rpc node started", - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run \"TestChainOperation|TestStateSync\"", - "./integration_test/contracts/verify_statesync_flatkv_digest.sh", - "./integration_test/contracts/verify_cross_validator_flatkv_digest.sh" - ] - }, - { - "name": "Distribution Module", - "scripts": [ - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestDistributionModule" + "./integration_test/evm_module/scripts/evm_interoperability_pointer_tests.sh" ] }, { - "name": "Upgrade Module (Major)", - "env": "GIGA_STORAGE=true UPGRADE_VERSION_LIST=v1.0.0,v1.0.1,v1.0.2", + "name": "EVM Interoperability (Misc Tests)", + "env": "GIGA_STORAGE=true", "scripts": [ - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestUpgradeMajor" + "./integration_test/evm_module/scripts/evm_interoperability_misc_tests.sh" ] }, { - "name": "Autobahn Upgrade Module (Major)", - "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true UPGRADE_VERSION_LIST=v1.0.0,v1.0.1,v1.0.2", + "name": "EVM RPC Parity (geth reference)", + "env": "GIGA_STORAGE=true", "scripts": [ - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestUpgradeMajor" + "./integration_test/rpc_tests/scripts/run-ci.sh" ] }, { - "name": "Upgrade Module (Minor)", - "env": "GIGA_STORAGE=true UPGRADE_VERSION_LIST=v1.0.0,v1.0.1,v1.0.2", + "name": "Autobahn EVM Interoperability (Pointer Tests)", + "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true", "scripts": [ - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestUpgradeMinor" + "./integration_test/evm_module/scripts/evm_interoperability_pointer_tests.sh" ] }, { - "name": "Autobahn Upgrade Module (Minor)", - "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true UPGRADE_VERSION_LIST=v1.0.0,v1.0.1,v1.0.2", + "name": "Autobahn EVM Interoperability (Misc Tests)", + "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true", "scripts": [ - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestUpgradeMinor" + "./integration_test/evm_module/scripts/evm_interoperability_misc_tests.sh" ] }, { - "name": "SeiDB State Store", + "_comment": "Suites: SeiDB State Store, Mint & Staking & Bank, Distribution. SeiDB State Store runs first because it asserts absolute wasm code counts (genesis + its own fixtures). TestMintModule asserts an absolute usei total supply, so nothing upstream of it in this row may mint or burn usei (SeiDB SS is safe today: fees are not burned and tokenfactory creation does not burn usei).", + "name": "Cosmos Modules (SeiDB SS, Mint & Staking & Bank, Distribution)", "scripts": [ "docker exec sei-node-0 integration_test/contracts/deploy_wasm_contracts.sh", "docker exec sei-node-0 integration_test/contracts/create_tokenfactory_denoms.sh", - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestSeiDBStateStore" + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestSeiDBStateStore", + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run \"TestMintModule|TestStakingModule|TestBankModule\"", + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestDistributionModule" ] }, { - "name": "Autobahn SeiDB State Store", - "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true", + "_comment": "Suites: Gov & Oracle & Authz, Wasm Module. Gov runs first: it changes chain params (quorum, unbonding time, recheck_tx, NumKeysToMigratePerBlock) that the Wasm suite does not depend on; Wasm's undelegate/redelegate cases assert only tx code 0.", + "name": "Cosmos Modules (Gov & Oracle & Authz, Wasm)", "scripts": [ - "docker exec sei-node-0 integration_test/contracts/deploy_wasm_contracts.sh", - "docker exec sei-node-0 integration_test/contracts/create_tokenfactory_denoms.sh", - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestSeiDBStateStore" + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run \"TestGovModule|TestOracleModule|TestAuthzModule\"", + "docker exec sei-node-0 integration_test/contracts/deploy_timelocked_token_contract.sh", + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestWasmModuleCore", + "docker exec sei-node-0 integration_test/contracts/deploy_timelocked_token_contract.sh", + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestWasmModuleEmergencyWithdraw" ] }, { - "_comment": "Umbrella job for all FlatKV-specific docker-level coverage. Future FlatKV scenarios (additional bucket layouts, snapshot rotation, etc.) should be appended here rather than added as new matrix rows, so the CI surface stays one row per concern. The cluster boots with GIGA_STORAGE=true, so FlatKV is enabled from genesis (sc-write-mode = test_only_dual_write) and the FlatKV LtHash is folded into every block's AppHash from block 1. No mid-life migration is exercised here; an offline memiavl -> FlatKV import would change the AppHash trajectory at the import boundary and break tendermint replay. Step ordering: 1-3 deploy an EVM fixture, smoke-check EVM RPC reads, and verify the fixture round-tripped through dual_write into physical FlatKV; 4 SIGKILL one validator, restart, and assert 4-node FlatKV digest equality (WAL recovery on a live FlatKV cluster); 5-7 destructive disaster-recovery scenarios, run last because they wipe or damage one validator's local state.", - "name": "FlatKV Integration", - "env": "GIGA_STORAGE=true", + "_comment": "Stays alone: disables the wasm module on the cluster, so no other suite may share this row.", + "name": "Disable WASM Tests", "scripts": [ - "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh", - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestSeiDBFlatKV", - "docker exec sei-node-0 integration_test/contracts/verify_flatkv_evm_store.sh", - "./integration_test/contracts/verify_flatkv_crash_recovery.sh", - "./integration_test/contracts/verify_flatkv_statesync_crash_recovery.sh", - "./integration_test/contracts/verify_flatkv_total_loss_recovery.sh", - "./integration_test/contracts/verify_flatkv_partial_loss_fails_loudly.sh" + "./integration_test/evm_module/scripts/disable_wasm.sh" ] }, { - "_comment": "Post-migration FlatKV-only state-sync coverage. The cluster boots directly in sc-write-mode = flatkv_only, so memiavl is not allocated and all module state is served from FlatKV. The script kills and wipes one validator, configures it for state-sync from the remaining validators, and verifies the recovered FlatKV digest matches the donors at a shared height.", - "name": "FlatKV Only State Sync", - "env": "GIGA_FLATKV_ONLY=true", + "_comment": "Suites: Autobahn SeiDB State Store, Autobahn Mint & Staking & Bank, Autobahn Gov & Oracle & Authz. SeiDB State Store runs first because it asserts absolute wasm code counts. TestMintModule and TestGovModule (burn case) assert absolute usei total supply, so nothing upstream of them in this row may mint or burn usei (SeiDB SS, Staking and Bank are safe today: fees are not burned and tokenfactory creation does not burn usei).", + "name": "Autobahn Cosmos Modules", + "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true", "scripts": [ - "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh", - "./integration_test/contracts/verify_flatkv_only_statesync.sh" + "docker exec sei-node-0 integration_test/contracts/deploy_wasm_contracts.sh", + "docker exec sei-node-0 integration_test/contracts/create_tokenfactory_denoms.sh", + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestSeiDBStateStore", + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run \"TestMintModule|TestStakingModule|TestAutobahnBankModule\"", + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run \"TestGovModule|TestOracleModule|TestAuthzModule\"" ] }, { - "name": "EVM Module (Compat)", + "name": "dApp Tests (Uniswap)", "env": "GIGA_STORAGE=true", "scripts": [ - "./integration_test/evm_module/scripts/evm_compat_tests.sh" + "./integration_test/dapp_tests/dapp_uniswap_tests.sh" ] }, { - "name": "EVM Module (Precompile & Endpoints)", + "name": "dApp Tests (NFT & Steak)", "env": "GIGA_STORAGE=true", "scripts": [ - "./integration_test/evm_module/scripts/evm_precompile_tests.sh", - "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh", - "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestSeiDBFlatKV", - "docker exec sei-node-0 integration_test/contracts/verify_flatkv_evm_store.sh" - ] - }, - { - "name": "Autobahn EVM Module (Compat)", - "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true", - "scripts": [ - "./integration_test/evm_module/scripts/evm_compat_tests.sh" + "./integration_test/dapp_tests/dapp_nft_steak_tests.sh" ] }, { - "name": "Autobahn EVM Module (Precompile & Endpoints)", - "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true", + "_comment": "Umbrella row for all FlatKV-specific docker-level coverage; future FlatKV scenarios (additional bucket layouts, snapshot rotation, etc.) are appended here. Non-FlatKV suites must not join it: steps 5-7 are destructive, so per the packing policy the row stays alone. The cluster boots with GIGA_STORAGE=true, so FlatKV is enabled from genesis (sc-write-mode = test_only_dual_write) and the FlatKV LtHash is folded into every block's AppHash from block 1. No mid-life migration is exercised here; an offline memiavl -> FlatKV import would change the AppHash trajectory at the import boundary and break tendermint replay. Step ordering: 1-3 deploy an EVM fixture, smoke-check EVM RPC reads, and verify the fixture round-tripped through dual_write into physical FlatKV; 4 SIGKILL one validator, restart, and assert 4-node FlatKV digest equality (WAL recovery on a live FlatKV cluster); 5-7 destructive disaster-recovery scenarios, run last because they wipe or damage one validator's local state.", + "name": "FlatKV Integration", + "env": "GIGA_STORAGE=true", "scripts": [ - "./integration_test/evm_module/scripts/evm_precompile_tests.sh", "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh", "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestSeiDBFlatKV", - "docker exec sei-node-0 integration_test/contracts/verify_flatkv_evm_store.sh" + "docker exec sei-node-0 integration_test/contracts/verify_flatkv_evm_store.sh", + "./integration_test/contracts/verify_flatkv_crash_recovery.sh", + "./integration_test/contracts/verify_flatkv_statesync_crash_recovery.sh", + "./integration_test/contracts/verify_flatkv_total_loss_recovery.sh", + "./integration_test/contracts/verify_flatkv_partial_loss_fails_loudly.sh" ] }, { - "name": "EVM Interoperability (Pointer Tests)", - "env": "GIGA_STORAGE=true", + "_comment": "Post-migration FlatKV-only state-sync coverage; destructive (kills and wipes a validator), so the row stays alone. The cluster boots directly in sc-write-mode = flatkv_only, so memiavl is not allocated and all module state is served from FlatKV. The script kills and wipes one validator, configures it for state-sync from the remaining validators, and verifies the recovered FlatKV digest matches the donors at a shared height.", + "name": "FlatKV Only State Sync", + "env": "GIGA_FLATKV_ONLY=true", "scripts": [ - "./integration_test/evm_module/scripts/evm_interoperability_pointer_tests.sh" + "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh", + "./integration_test/contracts/verify_flatkv_only_statesync.sh" ] }, { - "name": "EVM Interoperability (Misc Tests)", + "_comment": "Stays alone: TestChainOperation/TestStateSync stop, wipe and state-sync validators.", + "name": "Chain Operation Test", "env": "GIGA_STORAGE=true", "scripts": [ - "./integration_test/evm_module/scripts/evm_interoperability_misc_tests.sh" - ] - }, - { - "name": "Autobahn EVM Interoperability (Pointer Tests)", - "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true", - "scripts": [ - "./integration_test/evm_module/scripts/evm_interoperability_pointer_tests.sh" + "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done", + "until [[ $(docker exec sei-rpc-node build/seid status |jq -M -r .SyncInfo.latest_block_height) -gt 10 ]]; do sleep 10; done", + "echo rpc node started", + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run \"TestChainOperation|TestStateSync\"", + "./integration_test/contracts/verify_statesync_flatkv_digest.sh", + "./integration_test/contracts/verify_cross_validator_flatkv_digest.sh" ] }, { - "name": "Autobahn EVM Interoperability (Misc Tests)", - "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true", + "name": "Upgrade Module (Major)", + "env": "GIGA_STORAGE=true UPGRADE_VERSION_LIST=v1.0.0,v1.0.1,v1.0.2", "scripts": [ - "./integration_test/evm_module/scripts/evm_interoperability_misc_tests.sh" + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestUpgradeMajor" ] }, { - "name": "Disable WASM Tests", + "name": "Autobahn Upgrade Module (Major)", + "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true UPGRADE_VERSION_LIST=v1.0.0,v1.0.1,v1.0.2", "scripts": [ - "./integration_test/evm_module/scripts/disable_wasm.sh" + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestUpgradeMajor" ] }, { - "name": "dApp Tests (Uniswap)", - "env": "GIGA_STORAGE=true", + "name": "Upgrade Module (Minor)", + "env": "GIGA_STORAGE=true UPGRADE_VERSION_LIST=v1.0.0,v1.0.1,v1.0.2", "scripts": [ - "./integration_test/dapp_tests/dapp_uniswap_tests.sh" + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestUpgradeMinor" ] }, { - "name": "dApp Tests (NFT & Steak)", - "env": "GIGA_STORAGE=true", + "name": "Autobahn Upgrade Module (Minor)", + "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true UPGRADE_VERSION_LIST=v1.0.0,v1.0.1,v1.0.2", "scripts": [ - "./integration_test/dapp_tests/dapp_nft_steak_tests.sh" + "go test -tags yaml_integration -v -timeout $JOB_TIMEOUT ./integration_test/runner/... -run TestUpgradeMinor" ] }, { @@ -209,27 +208,6 @@ "./integration_test/evm_module/scripts/evm_giga_tests.sh" ] }, - { - "name": "Autobahn EVM GIGA Module", - "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_STORAGE=true GIGA_OCC=true", - "scripts": [ - "./integration_test/evm_module/scripts/evm_giga_tests.sh" - ] - }, - { - "name": "EVM GIGA Mixed (Determinism)", - "env": "GIGA_STORAGE=true", - "scripts": [ - "./integration_test/evm_module/scripts/evm_giga_mixed_tests.sh" - ] - }, - { - "name": "EVM RPC .io/.iox (spec fixtures)", - "env": "GIGA_STORAGE=true", - "scripts": [ - "./integration_test/evm_module/scripts/evm_rpc_tests.sh" - ] - }, { "name": "Autobahn RPC .io/.iox (spec fixtures)", "env": "AUTOBAHN=true GIGA_EXECUTOR=true GIGA_OCC=false GIGA_STORAGE=true", @@ -243,19 +221,5 @@ "scripts": [ "go test -tags frozen_rpc_integration -v -count=1 -timeout 5m ./integration_test/frozen_rpc_router/..." ] - }, - { - "name": "EVM RPC Parity (geth reference)", - "env": "GIGA_STORAGE=true", - "scripts": [ - "./integration_test/rpc_tests/scripts/run-ci.sh" - ] - }, - { - "name": "EVM Precompiles", - "env": "GIGA_STORAGE=true", - "scripts": [ - "./integration_test/precompile_tests/scripts/run-ci.sh" - ] } ] diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 26d4bdc656..1f386d79d4 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -25,11 +25,40 @@ defaults: run: shell: bash +env: + # Non-cone sparse-checkout pattern shared by every job that needs the source + # tree: everything except the darwin/aarch64 wasmvm archives, which CI + # (linux/amd64 only) never links and which make up ~500 MB of the tree. + SPARSE_CHECKOUT: | + /* + !/sei-wasmvm/internal/api/*darwin* + !/sei-wasmvm/internal/api/*aarch64* + !/sei-wasmvm/internal/api/*.dylib + !/sei-wasmd/x/wasm/artifacts/*/api/*darwin* + !/sei-wasmd/x/wasm/artifacts/*/api/*aarch64* + !/sei-wasmd/x/wasm/artifacts/*/api/*.dylib + jobs: # Generate the integration-test matrix from integration-test-matrix.json. # Runs the full matrix on every event (pull_request, push, merge_group) — all # Autobahn coverage runs on every PR. The matrix is built here, in a job # output, because the `matrix` context is NOT available to a job-level `if:`. + # + # Packing policy for integration-test-matrix.json. Every row is its own job + # and all rows start at once, so wall-clock = prepare-cluster + the slowest + # row, while each extra row costs its ~2 min of fixed setup in runner-minutes. + # A row is a list of suites that share one cluster boot (same `env`) and run + # back-to-back against the same chain state. When adding a suite: + # - Keep every row under ~8 min of test time. Join an existing row with a + # matching `env` that has room; otherwise add a new row. + # - Suites that assert absolute chain state (total supply, wasm code + # counts, ...) run first in their row, and the row's `_comment` names the + # assertion so a later suite is not appended ahead of it. + # - Destructive suites stay alone: anything that kills, wipes, or + # state-syncs a validator, disables a module, or runs a chain upgrade + # leaves the cluster in a state no other suite should inherit. + # - A suite that boots its own cluster (Autobahn Basic) is a separate job, + # not a matrix row. set-matrix: name: Resolve integration test matrix runs-on: ubuntu-latest @@ -39,6 +68,8 @@ jobs: steps: # See: https://github.com/actions/checkout/releases/tag/v7.0.0 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + sparse-checkout: .github/workflows - name: Build matrix id: build run: | @@ -67,10 +98,18 @@ jobs: steps: # See: https://github.com/actions/checkout/releases/tag/v7.0.0 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + # CI only builds linux/amd64; leave the darwin/aarch64 wasmvm + # archives (~500 MB of the tree) out of the fetch. + sparse-checkout: ${{ env.SPARSE_CHECKOUT }} + sparse-checkout-cone-mode: false # Reclaim disk and push Docker + Go caches onto /mnt before the # image builds so the runner's small root filesystem does not fill up and # crash the runner mid-job. - name: Reclaim runner disk + env: + # Image build + seid build need more headroom than test jobs. + CI_FREE_DISK_MIN_GIB: 60 run: bash .github/scripts/ci-free-disk.sh - name: Relocate Docker data-root to /mnt run: bash .github/scripts/ci-relocate-docker-data-root.sh @@ -110,25 +149,6 @@ jobs: labels: sei-chain.ci-run-id=${{ github.run_id }} cache-from: type=registry,ref=${{ env.GHCR_LOCALNODE }}:cache cache-to: ${{ github.event_name == 'push' && format('type=registry,ref={0}:cache,mode=max', env.GHCR_LOCALNODE) || '' }} - - name: Build rpcnode image (cached) - uses: docker/build-push-action@v6 - with: - context: docker/rpcnode - file: docker/rpcnode/Dockerfile - platforms: linux/amd64 - push: false - load: true - tags: sei-chain/rpcnode - labels: sei-chain.ci-run-id=${{ github.run_id }} - cache-from: type=registry,ref=${{ env.GHCR_RPCNODE }}:cache - cache-to: ${{ github.event_name == 'push' && format('type=registry,ref={0}:cache,mode=max', env.GHCR_RPCNODE) || '' }} - # rpcnode is independent of the seid build, so push it now: push failures - # (retried below) surface before the expensive seid build rather than after. - - name: Push rpcnode image to GHCR for test jobs - run: | - source .github/scripts/docker-registry-retry.sh - docker tag sei-chain/rpcnode "${GHCR_RPCNODE}:${{ github.run_id }}" - push_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}" - uses: actions/setup-go@v6 with: go-version: '1.25.6' @@ -150,16 +170,75 @@ jobs: path: integration-build.tar.gz retention-days: 1 + # rpcnode does not depend on seid, so it is built and pushed in parallel with + # prepare-cluster instead of on its critical path. + prepare-rpcnode: + name: Prepare integration test rpcnode image + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + packages: write + contents: read + env: + GHCR_RPCNODE: ghcr.io/sei-protocol/sei-chain-integration-test-rpcnode + steps: + # See: https://github.com/actions/checkout/releases/tag/v7.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + sparse-checkout: | + .github + docker/rpcnode + - name: Login to Docker Hub + # See: https://github.com/docker/login-action/releases/tag/v4.2.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + if: env.DOCKERHUB_USERNAME != '' + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GitHub Container Registry + # See: https://github.com/docker/login-action/releases/tag/v4.2.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + # See: https://github.com/docker/setup-buildx-action/releases/tag/v4.1.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 + - name: Build rpcnode image (cached) + uses: docker/build-push-action@v6 + with: + context: docker/rpcnode + file: docker/rpcnode/Dockerfile + platforms: linux/amd64 + push: false + load: true + tags: sei-chain/rpcnode + labels: sei-chain.ci-run-id=${{ github.run_id }} + cache-from: type=registry,ref=${{ env.GHCR_RPCNODE }}:cache + cache-to: ${{ github.event_name == 'push' && format('type=registry,ref={0}:cache,mode=max', env.GHCR_RPCNODE) || '' }} + - name: Push rpcnode image to GHCR for test jobs + run: | + source .github/scripts/docker-registry-retry.sh + docker tag sei-chain/rpcnode "${GHCR_RPCNODE}:${{ github.run_id }}" + push_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}" + integration-tests: name: Integration Test (${{ matrix.test.name }}) - needs: [ set-matrix, prepare-cluster ] + needs: [ set-matrix, prepare-cluster, prepare-rpcnode ] runs-on: ubuntu-large timeout-minutes: 30 permissions: packages: read contents: read env: - JOB_TIMEOUT: 28m + # Wall-clock budget for a whole row's scripts. Each `go test` invocation + # gets `-timeout $JOB_TIMEOUT` set to whatever is left of this budget, so + # a hang anywhere in the row hits Go's timeout (goroutine dump) before + # timeout-minutes cancels the job. + ROW_BUDGET_SECONDS: 1560 GHCR_LOCALNODE: ghcr.io/sei-protocol/sei-chain-integration-test-localnode GHCR_RPCNODE: ghcr.io/sei-protocol/sei-chain-integration-test-rpcnode AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -175,6 +254,9 @@ jobs: steps: # See: https://github.com/actions/checkout/releases/tag/v7.0.0 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + sparse-checkout: ${{ env.SPARSE_CHECKOUT }} + sparse-checkout-cone-mode: false - name: Reclaim runner disk run: bash .github/scripts/ci-free-disk.sh - name: Relocate Docker data-root to /mnt @@ -184,9 +266,15 @@ jobs: - uses: actions/setup-node@v4 with: node-version: '22' + cache: npm + cache-dependency-path: | + contracts/package-lock.json + integration_test/dapp_tests/package-lock.json + integration_test/rpc_tests/package-lock.json + integration_test/precompile_tests/package-lock.json - name: Install jq - run: sudo apt-get install -y jq + run: command -v jq >/dev/null || sudo apt-get install -y jq - name: Set up Go uses: actions/setup-go@v6 @@ -210,8 +298,10 @@ jobs: # tarball so its copy does not sit on disk for the rest of the job. rm -f integration-build.tar.gz source .github/scripts/docker-registry-retry.sh - pull_with_retry "${GHCR_LOCALNODE}:${{ github.run_id }}" - pull_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}" + pull_with_retry "${GHCR_LOCALNODE}:${{ github.run_id }}" & localnode_pull=$! + pull_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}" & rpcnode_pull=$! + wait "${localnode_pull}" + wait "${rpcnode_pull}" docker tag "${GHCR_LOCALNODE}:${{ github.run_id }}" sei-chain/localnode docker tag "${GHCR_RPCNODE}:${{ github.run_id }}" sei-chain/rpcnode # Drop the redundant run-id-tagged references now that the images are @@ -227,11 +317,11 @@ jobs: - name: Wait for docker cluster to start run: | - until [ $(cat build/generated/launch.complete |wc -l) = 4 ] + until [ $(cat build/generated/launch.complete 2>/dev/null |wc -l) = 4 ] do - sleep 10 + sleep 2 done - sleep 10 + sleep 5 - name: Start rpc node if: ${{ matrix.test.cluster != 'frozen-rpc-router' }} @@ -244,11 +334,20 @@ jobs: - name: ${{ matrix.test.name }} run: | scripts=$(echo '${{ toJson(matrix.test.scripts) }}' | jq -r '.[]') + row_start=$SECONDS IFS=$'\n' # change the internal field separator to newline - echo $scripts for script in $scripts do - bash -c "${script}" + remaining=$(( ROW_BUDGET_SECONDS - (SECONDS - row_start) )) + if [ "${remaining}" -lt 60 ]; then remaining=60; fi + export JOB_TIMEOUT="${remaining}s" + echo "::group::${script} (JOB_TIMEOUT=${JOB_TIMEOUT})" + if ! bash -c "${script}"; then + echo "::endgroup::" + echo "::error::suite failed: ${script}" + exit 1 + fi + echo "::endgroup::" done unset IFS # revert the internal field separator back to default @@ -323,7 +422,7 @@ jobs: name: Integration Test (Autobahn Basic) runs-on: ubuntu-large timeout-minutes: 45 - needs: prepare-cluster + needs: [ prepare-cluster, prepare-rpcnode ] permissions: packages: read contents: read @@ -333,6 +432,9 @@ jobs: steps: # See: https://github.com/actions/checkout/releases/tag/v7.0.0 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + sparse-checkout: ${{ env.SPARSE_CHECKOUT }} + sparse-checkout-cone-mode: false - name: Reclaim runner disk run: bash .github/scripts/ci-free-disk.sh - name: Relocate Docker data-root to /mnt @@ -343,7 +445,7 @@ jobs: with: go-version: '1.25.6' - name: Install jq - run: sudo apt-get install -y jq + run: command -v jq >/dev/null || sudo apt-get install -y jq - name: Login to Docker Hub uses: docker/login-action@v3 if: env.DOCKERHUB_USERNAME != '' @@ -369,8 +471,10 @@ jobs: # tarball so its copy does not sit on disk for the rest of the job. rm -f integration-build.tar.gz source .github/scripts/docker-registry-retry.sh - pull_with_retry "${GHCR_LOCALNODE}:${{ github.run_id }}" - pull_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}" + pull_with_retry "${GHCR_LOCALNODE}:${{ github.run_id }}" & localnode_pull=$! + pull_with_retry "${GHCR_RPCNODE}:${{ github.run_id }}" & rpcnode_pull=$! + wait "${localnode_pull}" + wait "${rpcnode_pull}" docker tag "${GHCR_LOCALNODE}:${{ github.run_id }}" sei-chain/localnode docker tag "${GHCR_RPCNODE}:${{ github.run_id }}" sei-chain/rpcnode # Drop the redundant run-id-tagged references now that the images are @@ -404,7 +508,7 @@ jobs: integration-test-check: name: Integration Test Check runs-on: ubuntu-latest - needs: [ prepare-cluster, integration-tests, autobahn-integration-tests ] + needs: [ prepare-cluster, prepare-rpcnode, integration-tests, autobahn-integration-tests ] if: always() steps: - name: Verify prepare and test jobs succeeded @@ -413,6 +517,10 @@ jobs: echo "prepare-cluster job did not succeed (${{ needs.prepare-cluster.result }})" exit 1 fi + if [[ "${{ needs.prepare-rpcnode.result }}" != "success" ]]; then + echo "prepare-rpcnode job did not succeed (${{ needs.prepare-rpcnode.result }})" + exit 1 + fi if [[ "${{ needs.integration-tests.result }}" != "success" ]]; then echo "integration-tests job did not succeed (${{ needs.integration-tests.result }})" exit 1 diff --git a/.github/workflows/release-upgrade-test.yml b/.github/workflows/release-upgrade-test.yml index fb64fbfa6e..e5249a3167 100644 --- a/.github/workflows/release-upgrade-test.yml +++ b/.github/workflows/release-upgrade-test.yml @@ -65,6 +65,9 @@ jobs: fetch-depth: 0 - name: Reclaim runner disk + env: + # Builds two seid trees from a full-history checkout. + CI_FREE_DISK_MIN_GIB: 60 run: bash .github/scripts/ci-free-disk.sh - name: Relocate Docker data-root to /mnt diff --git a/Makefile b/Makefile index f4282f5784..1da7465302 100644 --- a/Makefile +++ b/Makefile @@ -260,9 +260,12 @@ build-rpc-node: .PHONY: build-rpc-node # Integration-test CI: verify images pulled from GHCR by the matrix job. -ensure-integration-ci-images: +ensure-integration-ci-localnode-image: @docker image inspect sei-chain/localnode >/dev/null 2>&1 || (echo "sei-chain/localnode image missing; pull from GHCR (see prepare-cluster job)" && exit 1) - @docker image inspect sei-chain/rpcnode >/dev/null 2>&1 || (echo "sei-chain/rpcnode image missing; pull from GHCR (see prepare-cluster job)" && exit 1) +.PHONY: ensure-integration-ci-localnode-image + +ensure-integration-ci-images: ensure-integration-ci-localnode-image + @docker image inspect sei-chain/rpcnode >/dev/null 2>&1 || (echo "sei-chain/rpcnode image missing; pull from GHCR (see prepare-rpcnode job)" && exit 1) .PHONY: ensure-integration-ci-images # Build seid once inside the localnode image (integration-test prepare job). @@ -281,7 +284,7 @@ build-seid-in-localnode: build-docker-node .PHONY: build-seid-in-localnode # CI variant: assumes localnode image already built by Buildx in prepare-cluster (skips docker build). -build-seid-in-localnode-ci: ensure-integration-ci-images +build-seid-in-localnode-ci: ensure-integration-ci-localnode-image @mkdir -p build $(shell go env GOMODCACHE) $(shell go env GOCACHE) @docker run --rm \ --user="$(shell id -u):$(shell id -g)" \ diff --git a/integration_test/precompile_tests/README.md b/integration_test/precompile_tests/README.md index 33eb4c7315..b5e1bb1a19 100644 --- a/integration_test/precompile_tests/README.md +++ b/integration_test/precompile_tests/README.md @@ -122,7 +122,8 @@ npm run precompile:ci # == bash scripts/run-ci.sh ``` `scripts/run-ci.sh` is the single orchestrator, used both locally and by the -`EVM Precompiles` matrix entry in `.github/workflows/integration-test.yml`. It +`EVM Module (Precompile & Endpoints) & Precompiles` matrix row in +`.github/workflows/integration-test-matrix.json`. It assumes a Sei EVM RPC is already reachable (the workflow boots the 4-node cluster; locally you start it yourself), then end to end: