From a337eec90dec02b691cad6f6b91482aa9537c2a6 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 1 Sep 2026 02:10:23 +0000 Subject: [PATCH] feat(ci): trigger downstream module integration tests on upstream component changes - Add get_upstream_modules and is_upstream_module_modified in .kokoro/common.sh to map intra-monorepo component dependencies: * java-bigquerystorage triggers java-bigquery and java-bigquery-jdbc * java-bigquery triggers java-bigquery-jdbc * grpc-gcp-java triggers java-spanner and java-spanner-jdbc * java-spanner triggers java-spanner-jdbc * java-storage triggers java-storage-nio * java-logging triggers java-logging-logback - Update should_test_all_modules in .kokoro/common.sh to run all integration tests if sdk-platform-java or google-auth-library-java is modified. - Update integration-single and graalvm-single in .kokoro/build.sh to run split integration and GraalVM tests when an upstream dependency is modified. - Include downstream modules in generate_modified_modules_list for batch runs. - Add comprehensive test coverage in .kokoro/common_test.sh for upstream triggers, auth, and sdk-platform-java changes. --- .kokoro/build.sh | 24 ++++---- .kokoro/common.sh | 76 ++++++++++++++++++++++-- .kokoro/common_test.sh | 131 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 210 insertions(+), 21 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 5c396a83889c..e8a31ffc502d 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -121,12 +121,14 @@ case ${JOB_TYPE} in integration-single) if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then echo "Not running integration checks -- this is Release Please SNAPSHOT pull request." - # Run tests if either global overrides require testing all modules (e.g. parent POM or - # shared dependencies) OR if this specific module was modified. Otherwise skip. - elif ! should_test_all_modules && ! is_module_modified "${BUILD_SUBDIR}"; then - echo "${BUILD_SUBDIR} not modified, skipping split integration test" + # Run tests if: + # 1. Global overrides require testing all modules (e.g. parent POM, sdk-platform-java, auth) + # 2. This specific module was modified + # 3. An upstream dependency of this module was modified (e.g. java-spanner or grpc-gcp-java for java-spanner-jdbc) + elif ! should_test_all_modules && ! is_module_modified "${BUILD_SUBDIR}" && ! is_upstream_module_modified "${BUILD_SUBDIR}"; then + echo "${BUILD_SUBDIR} not modified and no upstream dependencies modified, skipping split integration test" else - echo "${BUILD_SUBDIR} modified, running split integration test" + echo "${BUILD_SUBDIR} (or an upstream dependency) modified, running split integration test" echo "Compiling and building all modules for ${BUILD_SUBDIR}" install_modules "${BUILD_SUBDIR}" echo "Running in subdir: ${BUILD_SUBDIR}" @@ -191,12 +193,14 @@ case ${JOB_TYPE} in graalvm-single) if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then echo "Not running GraalVM checks -- this is Release Please SNAPSHOT pull request." - # Run tests if either global overrides require testing all modules (e.g. parent POM or - # shared dependencies) OR if this specific module was modified. Otherwise skip. - elif ! should_test_all_modules && ! is_module_modified "${BUILD_SUBDIR}"; then - echo "${BUILD_SUBDIR} not modified, skipping split GraalVM test" + # Run tests if: + # 1. Global overrides require testing all modules (e.g. parent POM, sdk-platform-java, auth) + # 2. This specific module was modified + # 3. An upstream dependency of this module was modified (e.g. java-spanner or grpc-gcp-java for java-spanner-jdbc) + elif ! should_test_all_modules && ! is_module_modified "${BUILD_SUBDIR}" && ! is_upstream_module_modified "${BUILD_SUBDIR}"; then + echo "${BUILD_SUBDIR} not modified and no upstream dependencies modified, skipping split GraalVM test" else - echo "${BUILD_SUBDIR} modified, running split GraalVM test" + echo "${BUILD_SUBDIR} (or an upstream dependency) modified, running split GraalVM test" echo "Compiling and building all modules for ${BUILD_SUBDIR}" install_modules "${BUILD_SUBDIR}" echo "Running in subdir: ${BUILD_SUBDIR}" diff --git a/.kokoro/common.sh b/.kokoro/common.sh index ee6770a9d6e1..5a967df4e3fa 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -250,12 +250,14 @@ function get_modified_files() { # Determines if the entire monorepo must be tested. # -# Monorepo-wide testing is triggered under three conditions: +# Monorepo-wide testing is triggered under four conditions: # 1. TEST_ALL_MODULES is set to "true" (used by nightly and scheduled CI builds). # 2. Root parent POMs (google-cloud-jar-parent or google-cloud-pom-parent) are modified, # as changes to parent POMs affect shared dependency versions and compiler/build plugins. -# 3. Core shared dependencies (sdk-platform-java/java-shared-dependencies) are modified, -# as gax, auth, and transport changes can break downstream client library integration tests. +# 3. Core SDK platform libraries (sdk-platform-java) are modified, as gax, generators, +# and core transport changes can break downstream client library integration tests. +# 4. Core authentication libraries (google-auth-library-java) are modified, as auth/credential +# changes affect all client libraries. function should_test_all_modules() { local files files=$(get_modified_files) @@ -264,7 +266,8 @@ function should_test_all_modules() { # stdin of grep, avoiding an external subshell pipeline (like 'echo "$var" | grep'). if [[ "${TEST_ALL_MODULES}" == "true" ]] || \ grep -q -E '^google-cloud-(pom|jar)-parent/pom.xml$' <<< "${files}" || \ - grep -q -E '^sdk-platform-java/java-shared-dependencies/' <<< "${files}"; then + grep -q -E '^sdk-platform-java/' <<< "${files}" || \ + grep -q -E '^google-auth-library-java/' <<< "${files}"; then return 0 fi return 1 @@ -317,6 +320,16 @@ function generate_modified_modules_list() { else echo "Found no changes in the java modules" fi + + # Also include downstream modules if any of their upstream dependencies were modified, + # ensuring batch integration tests cover dependent client libraries. + for module in "${maven_modules[@]}"; do + if is_upstream_module_modified "${module}"; then + if [[ ! " ${modified_module_list[*]} " =~ " ${module} " ]]; then + modified_module_list+=("${module}") + fi + fi + done fi } @@ -337,6 +350,61 @@ function is_module_modified() { grep -q -E "^${module}/" <<< "${files}" } +# Maps a module to its intra-monorepo upstream dependencies. +# +# Certain libraries in this repository directly depend on sibling modules (for example, +# java-bigquery depends on java-bigquerystorage, JDBC drivers wrap client SDKs, and +# java-spanner depends on grpc-gcp-java). Without this mapping, changes to an upstream +# dependency would not trigger integration tests for downstream consumers in PR CI. +# +# Returns space-separated module names that the given module depends on, or empty if none. +function get_upstream_modules() { + local module="$1" + case "${module}" in + java-bigquery) + echo "java-bigquerystorage" + ;; + java-bigquery-jdbc) + echo "java-bigquery java-bigquerystorage" + ;; + java-spanner) + echo "grpc-gcp-java" + ;; + java-spanner-jdbc) + echo "java-spanner grpc-gcp-java" + ;; + java-storage-nio) + echo "java-storage" + ;; + java-logging-logback) + echo "java-logging" + ;; + *) + ;; + esac +} + +# Checks if any upstream dependency of the given module was modified in the PR diff. +# +# Takes a module name (e.g. BUILD_SUBDIR), retrieves its upstream dependencies using +# get_upstream_modules, and checks if any of those upstream directories were touched. +# Returns 0 (true) if an upstream module was modified, triggering downstream tests; +# otherwise returns 1 (false). +function is_upstream_module_modified() { + local module="$1" + if [[ -z "${module}" ]]; then + return 1 + fi + + local upstream + for upstream in $(get_upstream_modules "${module}"); do + if is_module_modified "${upstream}"; then + return 0 + fi + done + return 1 +} + # Filters the modified_module_list to only include modules that contain # integration test files (matching IT*.java or *IT.java in src/test/java). # Not all modules will have ITs written and there is not need to test diff --git a/.kokoro/common_test.sh b/.kokoro/common_test.sh index 247a6a26cbfc..c16c939c468f 100755 --- a/.kokoro/common_test.sh +++ b/.kokoro/common_test.sh @@ -155,17 +155,31 @@ function test_should_test_all_modules { exit 1 fi - # Core shared dependencies - TEST_MODIFIED_FILES="sdk-platform-java/java-shared-dependencies/pom.xml" + # Core SDK platform + TEST_MODIFIED_FILES="sdk-platform-java/gapic-generator/src/main/Foo.java" if ! should_test_all_modules; then - echo "should_test_all_modules should return true for java-shared-dependencies change" + echo "should_test_all_modules should return true for sdk-platform-java change" exit 1 fi - # Prefix collision check: sibling paths starting with java-shared-dependencies must not match - TEST_MODIFIED_FILES="sdk-platform-java/java-shared-dependencies-bom/pom.xml" + # Prefix collision check: sibling paths starting with sdk-platform-java must not match + TEST_MODIFIED_FILES="sdk-platform-java-extra/pom.xml" if should_test_all_modules; then - echo "should_test_all_modules should return false for java-shared-dependencies prefix match" + echo "should_test_all_modules should return false for sdk-platform-java prefix match" + exit 1 + fi + + # Core auth library + TEST_MODIFIED_FILES="google-auth-library-java/oauth2_http/src/main/Foo.java" + if ! should_test_all_modules; then + echo "should_test_all_modules should return true for google-auth-library-java change" + exit 1 + fi + + # Prefix collision check: sibling paths starting with google-auth-library-java must not match + TEST_MODIFIED_FILES="google-auth-library-java-extra/pom.xml" + if should_test_all_modules; then + echo "should_test_all_modules should return false for google-auth-library-java prefix match" exit 1 fi @@ -179,6 +193,109 @@ function test_should_test_all_modules { unset TEST_MODIFIED_FILES } +# Test is_upstream_module_modified triggers correctly for downstream components. +function test_is_upstream_module_modified { + # Empty input should return false (1) + if is_upstream_module_modified ""; then + echo "is_upstream_module_modified should return false for empty input" + exit 1 + fi + + # 1. java-bigquerystorage modified: + # - java-bigquery should trigger + # - java-bigquery-jdbc should trigger + # - java-bigquerystorage itself should NOT have upstream trigger + TEST_MODIFIED_FILES="java-bigquerystorage/src/main/Foo.java" + if ! is_upstream_module_modified "java-bigquery"; then + echo "is_upstream_module_modified should return true for java-bigquery when java-bigquerystorage is modified" + exit 1 + fi + if ! is_upstream_module_modified "java-bigquery-jdbc"; then + echo "is_upstream_module_modified should return true for java-bigquery-jdbc when java-bigquerystorage is modified" + exit 1 + fi + if is_upstream_module_modified "java-bigquerystorage"; then + echo "is_upstream_module_modified should return false for java-bigquerystorage when java-bigquerystorage is modified" + exit 1 + fi + + # 2. java-bigquery modified: + # - java-bigquery-jdbc should trigger + # - java-bigquerystorage should NOT trigger + TEST_MODIFIED_FILES="java-bigquery/src/main/Foo.java" + if ! is_upstream_module_modified "java-bigquery-jdbc"; then + echo "is_upstream_module_modified should return true for java-bigquery-jdbc when java-bigquery is modified" + exit 1 + fi + if is_upstream_module_modified "java-bigquerystorage"; then + echo "is_upstream_module_modified should return false for java-bigquerystorage when java-bigquery is modified" + exit 1 + fi + + # 3. grpc-gcp-java modified: + # - java-spanner should trigger + # - java-spanner-jdbc should trigger + TEST_MODIFIED_FILES="grpc-gcp-java/src/main/Foo.java" + if ! is_upstream_module_modified "java-spanner"; then + echo "is_upstream_module_modified should return true for java-spanner when grpc-gcp-java is modified" + exit 1 + fi + if ! is_upstream_module_modified "java-spanner-jdbc"; then + echo "is_upstream_module_modified should return true for java-spanner-jdbc when grpc-gcp-java is modified" + exit 1 + fi + + # 4. java-spanner modified: + # - java-spanner-jdbc should trigger + # - java-spanner itself should NOT have upstream trigger + TEST_MODIFIED_FILES="java-spanner/src/main/Foo.java" + if ! is_upstream_module_modified "java-spanner-jdbc"; then + echo "is_upstream_module_modified should return true for java-spanner-jdbc when java-spanner is modified" + exit 1 + fi + if is_upstream_module_modified "java-spanner"; then + echo "is_upstream_module_modified should return false for java-spanner when java-spanner is modified" + exit 1 + fi + + # 5. java-storage modified: + # - java-storage-nio should trigger + # - java-storage itself should NOT have upstream trigger + TEST_MODIFIED_FILES="java-storage/src/main/Foo.java" + if ! is_upstream_module_modified "java-storage-nio"; then + echo "is_upstream_module_modified should return true for java-storage-nio when java-storage is modified" + exit 1 + fi + if is_upstream_module_modified "java-storage"; then + echo "is_upstream_module_modified should return false for java-storage when java-storage is modified" + exit 1 + fi + + # 6. java-logging modified: + # - java-logging-logback should trigger + # - java-logging itself should NOT have upstream trigger + TEST_MODIFIED_FILES="java-logging/src/main/Foo.java" + if ! is_upstream_module_modified "java-logging-logback"; then + echo "is_upstream_module_modified should return true for java-logging-logback when java-logging is modified" + exit 1 + fi + if is_upstream_module_modified "java-logging"; then + echo "is_upstream_module_modified should return false for java-logging when java-logging is modified" + exit 1 + fi + + # 7. Unrelated module modified: + TEST_MODIFIED_FILES="java-asset/src/main/Foo.java" + if is_upstream_module_modified "java-asset" || \ + is_upstream_module_modified "java-spanner" || \ + is_upstream_module_modified "java-bigquery"; then + echo "is_upstream_module_modified should return false when only an unrelated module is modified" + exit 1 + fi + + unset TEST_MODIFIED_FILES +} + # Test mock get_modified_files returns simulated files. function test_mock_get_modified_files { TEST_MODIFIED_FILES="dummy/file.txt" @@ -197,4 +314,4 @@ test_parse_pom_version test_mock_get_modified_files test_should_test_all_modules test_is_module_modified - +test_is_upstream_module_modified