From e4a886367824b4db4fa9246d23fe8c1f5c745545 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 31 Aug 2026 15:36:46 +0000 Subject: [PATCH] perf(ci): replace mvn help:evaluate with native bash and sed extraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In Kokoro CI and automation scripts, heavy Maven JVM invocations were used to parse POM values: - In generate_modified_modules_list (.kokoro/common.sh), evaluating project.modules launched a full JVM and evaluated the monorepo POMs, taking 20–30+ seconds on every single CI run. - In downstream-build.sh (.kokoro/presubmit/downstream-build.sh) and showcase-native.sh (sdk-platform-java/.kokoro/presubmit/showcase-native.sh), evaluating gapic-showcase.version launched Maven JVM processes. - In update_javadoc.sh (google-auth-library-java/scripts/update_javadoc.sh), maven-help-plugin:evaluate was called to obtain the project version. Optimizations: - Pure-Bash Module Extraction: Updated generate_modified_modules_list to use extract_pom_modules pom.xml, extracting all 270 modules in ~0.02s without JVM boot overhead. - Sed Showcase Version Extraction: Updated downstream-build.sh and showcase-native.sh to parse directly using sed with fail-fast validation, and added --fail to curl. - Sed Javadoc Version Extraction: Updated update_javadoc.sh to parse project version from pom.xml using sed, skipping parent POM declarations. - Unit Tests: Added automated unit tests in .kokoro/common_test.sh for extract_pom_modules and generate_modified_modules_list. - Refactoring & Best Practices: Scoped temporary variables as local, used pure-bash whitespace trimming and string manipulations, and anchored paths. --- .kokoro/common.sh | 96 +++++++++------- .kokoro/common_test.sh | 106 ++++++++++++++++++ .kokoro/presubmit/downstream-build.sh | 22 +++- .../scripts/update_javadoc.sh | 9 +- .../.kokoro/presubmit/showcase-native.sh | 17 ++- 5 files changed, 201 insertions(+), 49 deletions(-) diff --git a/.kokoro/common.sh b/.kokoro/common.sh index 5a967df4e3fa..69a4f04ebff1 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -90,53 +90,56 @@ function retry_with_backoff { # and naturally survives single-module components without throwing exit signals. function extract_pom_modules() { local pom_file="$1" - local modules_list="" + if [[ ! -f "${pom_file}" ]]; then + return 1 + fi + local line module local in_profiles=false local in_modules=false - - while IFS= read -r line || [ -n "$line" ]; do - if [[ "$line" == *""* ]]; then + local -a modules=() + + while IFS= read -r line || [[ -n "${line}" ]]; do + if [[ "${line}" == *""* ]]; then in_profiles=true - elif [[ "$line" == *""* ]]; then + elif [[ "${line}" == *""* ]]; then in_profiles=false - elif [[ "$line" == *""* ]] && [ "$in_profiles" = false ]; then + elif [[ "${line}" == *""* && "${in_profiles}" == "false" ]]; then in_modules=true - elif [[ "$line" == *""* ]] && [ "$in_profiles" = false ]; then + elif [[ "${line}" == *""* && "${in_profiles}" == "false" ]]; then in_modules=false break - elif [ "$in_modules" = true ] && [[ "$line" == *""* ]]; then + elif [[ "${in_modules}" == "true" && "${line}" == *""* ]]; then # Extract text between tags - local module="${line#*}" + module="${line#*}" module="${module%*}" - - # Trim whitespace natively + + # Trim leading/trailing whitespace without spawning external processes module="${module#"${module%%[![:space:]]*}"}" module="${module%"${module##*[![:space:]]}"}" - - if [ -z "$modules_list" ]; then - modules_list="$module" - else - modules_list="${modules_list} ${module}" + + if [[ -n "${module}" ]]; then + modules+=("${module}") fi fi - done < "$pom_file" - - echo "$modules_list" + done < "${pom_file}" + + echo "${modules[*]}" } # Given a folder containing a maven multi-module, assign the variable 'submodules' to a # comma-delimited list of /. function parse_submodules() { submodules_array=() - if [ -f "$1/pom.xml" ]; then + if [[ -f "$1/pom.xml" ]]; then local modules + local submodule # Use pure Bash extraction to find the modules in the aggregator pom file. # Faster than invoking mvn help:evaluate to list all the project modules, # cleanly ignores optional , and gracefully skips flat POMs. modules=$(extract_pom_modules "$1/pom.xml") - if [ -n "$modules" ]; then - for submodule in $modules; do + if [[ -n "${modules}" ]]; then + for submodule in ${modules}; do # Each entry = / submodules_array+=("$1/${submodule}") done @@ -285,39 +288,54 @@ function generate_modified_modules_list() { files=$(get_modified_files) printf "Modified files:\n%s\n" "${files}" - # Generate the list of valid maven modules - maven_modules_list=$(mvn help:evaluate -Dexpression=project.modules | grep '<.*>.*' | sed -e 's/<.*>\(.*\)<\/.*>/\1/g') + # Extract valid maven modules directly from pom.xml in pure Bash (~0.02s). + # This replaces 'mvn help:evaluate -Dexpression=project.modules' which previously + # spent 20-30+ seconds booting a JVM and evaluating the monorepo POMs on every run. + local root_pom="${commonScriptDir}/../pom.xml" + if [[ ! -f "${root_pom}" ]]; then + root_pom="pom.xml" + fi + local maven_modules_list + maven_modules_list=$(extract_pom_modules "${root_pom}") maven_modules=() - # If the first argument is "true" (default), then use the module exclusion list - use_exclusion_list=${1:-true} + # Positional parameter $1 specifies whether to apply the exclusion list (defaults to true). + local use_exclusion_list="${1:-true}" + local -a all_modules=() + read -r -a all_modules <<< "${maven_modules_list}" + + local module if [[ "${use_exclusion_list}" == "true" ]]; then echo "Excluding modules from the global exclusion list" - for module in $maven_modules_list; do - if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]]; then + for module in "${all_modules[@]}"; do + if [[ ! " ${excluded_modules[*]} " == *" ${module} "* ]]; then maven_modules+=("${module}") fi done else - maven_modules=(${maven_modules_list[*]}) + maven_modules=("${all_modules[@]}") fi modified_module_list=() # If either parent pom.xml or core shared dependency is touched, run ITs on all the modules if should_test_all_modules; then - modified_module_list=(${maven_modules[*]}) + # '("${maven_modules[@]}")' copies the array elements safely. + modified_module_list=("${maven_modules[@]}") echo "Testing the entire monorepo" else - modules=$(echo "${files}" | grep -E '(google-auth|java)-.*' || true) + # Extract the top-level directory from each modified file path: + # 'cut -d '/' -f1' takes the first path segment (e.g. 'java-bigquery/src/...' -> 'java-bigquery'). + # 'sort -u' sorts and deduplicates the candidate directory names. + local modules + modules=$(cut -d '/' -f1 <<< "${files}" | sort -u) printf "Files in java modules:\n%s\n" "${modules}" - if [[ -n $modules ]]; then - modules=$(echo "${modules}" | cut -d '/' -f1 | sort -u) - for module in $modules; do - if [[ " ${maven_modules[*]} " =~ " ${module} " ]]; then - modified_module_list+=("${module}") - fi - done - else + for module in ${modules}; do + # If this top-level directory is a recognized Maven module, add it to our list. + if [[ " ${maven_modules[*]} " == *" ${module} "* ]]; then + modified_module_list+=("${module}") + fi + done + if [[ ${#modified_module_list[@]} -eq 0 ]]; then echo "Found no changes in the java modules" fi diff --git a/.kokoro/common_test.sh b/.kokoro/common_test.sh index c16c939c468f..56e6189feef1 100755 --- a/.kokoro/common_test.sh +++ b/.kokoro/common_test.sh @@ -81,6 +81,35 @@ function test_parse_pom_version { popd } +# Test that extract_pom_modules correctly extracts modules from root pom.xml +# in pure Bash without launching Maven, and handles non-existent files safely. +function test_extract_pom_modules { + local -a modules + read -r -a modules <<< "$(extract_pom_modules "${scriptDir}/../pom.xml")" + if (( ${#modules[@]} < 200 )); then + echo "extract_pom_modules failed: expected at least 200 modules, got ${#modules[@]}" + exit 1 + fi + if [[ ! " ${modules[*]} " =~ " java-bigquery " ]]; then + echo "extract_pom_modules missing java-bigquery" + exit 1 + fi + if [[ ! " ${modules[*]} " =~ " java-bigquerystorage " ]]; then + echo "extract_pom_modules missing java-bigquerystorage" + exit 1 + fi + if [[ ! " ${modules[*]} " =~ " sdk-platform-java " ]]; then + echo "extract_pom_modules missing sdk-platform-java" + exit 1 + fi + + # Verify non-existent file returns 1 and empty output + if extract_pom_modules "non_existent_pom.xml" &>/dev/null; then + echo "extract_pom_modules should return non-zero for non-existent pom" + exit 1 + fi +} + # Tests that is_module_modified strictly matches the module directory prefix, # preventing prefix collisions (e.g. java-bigquery vs java-bigquerystorage). function test_is_module_modified { @@ -308,10 +337,87 @@ function test_mock_get_modified_files { unset TEST_MODIFIED_FILES } +# Test that parse_submodules expands multi-module directories to child modules +# and preserves single-module/flat components. +function test_parse_submodules { + pushd "${scriptDir}/.." >/dev/null + + parse_submodules "java-bigquery" + if [[ "${submodules}" != "java-bigquery/google-cloud-bigquery,java-bigquery/google-cloud-bigquery-bom" ]]; then + echo "parse_submodules failed for java-bigquery: got ${submodules}" + exit 1 + fi + + parse_submodules "google-cloud-jar-parent" + if [[ "${submodules}" != "google-cloud-jar-parent" ]]; then + echo "parse_submodules failed for google-cloud-jar-parent: got ${submodules}" + exit 1 + fi + + popd >/dev/null +} + +# Test that generate_modified_modules_list correctly maps modified files to +# their top-level module names without requiring mvn help:evaluate, and verifies +# both exclusion list enabled (default) and disabled modes. +function test_generate_modified_modules_list { + pushd "${scriptDir}/.." >/dev/null + TEST_MODIFIED_FILES="java-bigquery/google-cloud-bigquery/src/main/java/Foo.java +java-asset/google-cloud-asset/pom.xml" + generate_modified_modules_list false >/dev/null + + local has_bigquery="false" + local has_asset="false" + local has_bigquerystorage="false" + + if [[ " ${modified_module_list[*]} " =~ " java-bigquery " ]]; then + has_bigquery="true" + fi + if [[ " ${modified_module_list[*]} " =~ " java-asset " ]]; then + has_asset="true" + fi + if [[ " ${modified_module_list[*]} " =~ " java-bigquerystorage " ]]; then + has_bigquerystorage="true" + fi + + if [[ "${has_bigquery}" != "true" ]]; then + echo "generate_modified_modules_list missing java-bigquery" + exit 1 + fi + if [[ "${has_asset}" != "true" ]]; then + echo "generate_modified_modules_list missing java-asset" + exit 1 + fi + if [[ "${has_bigquerystorage}" == "true" ]]; then + echo "generate_modified_modules_list incorrectly included java-bigquerystorage" + exit 1 + fi + + # Test default exclusion list behavior: java-bigquery is excluded, while java-asset is included + TEST_MODIFIED_FILES="java-asset/google-cloud-asset/Foo.java +java-bigquery/google-cloud-bigquery/Bar.java" + generate_modified_modules_list true >/dev/null + + if [[ " ${modified_module_list[*]} " =~ " java-bigquery " ]]; then + echo "generate_modified_modules_list should exclude java-bigquery when exclusion list is true" + exit 1 + fi + if [[ ! " ${modified_module_list[*]} " =~ " java-asset " ]]; then + echo "generate_modified_modules_list missing java-asset when exclusion list is true" + exit 1 + fi + + popd >/dev/null + unset TEST_MODIFIED_FILES +} + test_find_all_poms_with_versioned_dependency test_update_pom_dependency test_parse_pom_version test_mock_get_modified_files +test_extract_pom_modules +test_parse_submodules test_should_test_all_modules test_is_module_modified test_is_upstream_module_modified +test_generate_modified_modules_list diff --git a/.kokoro/presubmit/downstream-build.sh b/.kokoro/presubmit/downstream-build.sh index aa6781b1ffee..78ac24390928 100755 --- a/.kokoro/presubmit/downstream-build.sh +++ b/.kokoro/presubmit/downstream-build.sh @@ -47,14 +47,26 @@ pushd java-showcase modify_shared_config popd -# Parse showcase version from the local directory -pushd java-showcase/gapic-showcase -SHOWCASE_VERSION=$(mvn help:evaluate -Dexpression=gapic-showcase.version -q -DforceStdout) -popd +# Extract the showcase version directly from pom.xml using sed: +# - 'sed -n': suppresses default line printing. +# - 's:...[[:space:]]*\([^<[:space:]]*\).*:\1:p': captures non-whitespace version text and prints it. +# - '/.../q': quits immediately on first match, avoiding trailing passes and external pipe utilities. +# This replaces 'mvn help:evaluate' which previously took 15+ seconds to boot Maven. +SHOWCASE_VERSION=$(sed -n 's:.*[[:space:]]*\([^<[:space:]]*\).*:\1:p; //q' java-showcase/gapic-showcase/pom.xml) + +# Fail fast with a clear error message if the version could not be parsed, +# preventing malformed curl URLs and ambiguous downstream failures. +if [[ -z "${SHOWCASE_VERSION}" ]]; then + echo "Error: Failed to parse gapic-showcase.version from java-showcase/gapic-showcase/pom.xml" >&2 + exit 1 +fi # Start showcase server mkdir -p /usr/src/showcase -curl --location https://github.com/googleapis/gapic-showcase/releases/download/v"${SHOWCASE_VERSION}"/gapic-showcase-"${SHOWCASE_VERSION}"-linux-amd64.tar.gz --output /usr/src/showcase/showcase-"${SHOWCASE_VERSION}"-linux-amd64.tar.gz +# Use '--fail' so curl exits with an error status on HTTP failures (e.g., 404/500). +# Without '--fail', curl writes the error response body (HTML) to the tar.gz file +# and returns exit code 0, which results in cryptic tar decompression failures. +curl --fail --location https://github.com/googleapis/gapic-showcase/releases/download/v"${SHOWCASE_VERSION}"/gapic-showcase-"${SHOWCASE_VERSION}"-linux-amd64.tar.gz --output /usr/src/showcase/showcase-"${SHOWCASE_VERSION}"-linux-amd64.tar.gz pushd /usr/src/showcase/ tar -xf showcase-* ./gapic-showcase run & diff --git a/google-auth-library-java/scripts/update_javadoc.sh b/google-auth-library-java/scripts/update_javadoc.sh index e5d73c2188b6..0b69b4e8c9fb 100755 --- a/google-auth-library-java/scripts/update_javadoc.sh +++ b/google-auth-library-java/scripts/update_javadoc.sh @@ -31,10 +31,15 @@ set -e -VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -Ev '(^\[|\w+:)') +# Extract the project's version from pom.xml: +# - '//,/<\/parent>/d' deletes the block so we do not mistakenly +# extract the parent POM's version (e.g. google-cloud-shared-config) if a +# block appears before or contains a declaration. +# - '//...q;' captures the project's own tag and exits immediately. +VERSION=$(sed -n '//,/<\/parent>/d; //{s:.*[[:space:]]*\([^<[:space:]]*\).*:\1:p; q;}' pom.xml) if [ -z "$VERSION" ]; then - echo "Error updating Javadoc: could not obtain version number from maven-help-plugin." + echo "Error updating Javadoc: could not obtain version number from pom.xml." exit 1 fi diff --git a/sdk-platform-java/.kokoro/presubmit/showcase-native.sh b/sdk-platform-java/.kokoro/presubmit/showcase-native.sh index ad85d8dc6a08..4a3b3f3875af 100644 --- a/sdk-platform-java/.kokoro/presubmit/showcase-native.sh +++ b/sdk-platform-java/.kokoro/presubmit/showcase-native.sh @@ -38,12 +38,23 @@ mvn install --projects '!gapic-generator-java' \ SHARED_DEPS_VERSION=$(parse_pom_version java-shared-dependencies) # Run showcase integration tests in GraalVM -pushd java-showcase/gapic-showcase -SHOWCASE_VERSION=$(mvn help:evaluate -Dexpression=gapic-showcase.version -q -DforceStdout) -popd +SHOWCASE_POM="java-showcase/gapic-showcase/pom.xml" +if [[ ! -f "${SHOWCASE_POM}" && -f "../java-showcase/gapic-showcase/pom.xml" ]]; then + SHOWCASE_POM="../java-showcase/gapic-showcase/pom.xml" +fi +SHOWCASE_VERSION=$(sed -n 's:.*[[:space:]]*\([^<[:space:]]*\).*:\1:p; //q' "${SHOWCASE_POM}") +if [[ -z "${SHOWCASE_VERSION}" ]]; then + echo "Error: Failed to parse gapic-showcase.version from ${SHOWCASE_POM}" >&2 + exit 1 +fi + # Start showcase server mkdir -p /usr/src/showcase +# Use '--fail' so curl exits with an error status on HTTP failures (e.g., 404/500). +# Without '--fail', curl writes the error response body (HTML) to the tar.gz file +# and returns exit code 0, which results in cryptic tar decompression failures. curl \ + --fail \ --location https://github.com/googleapis/gapic-showcase/releases/download/v"${SHOWCASE_VERSION}"/gapic-showcase-"${SHOWCASE_VERSION}"-linux-amd64.tar.gz \ --output /usr/src/showcase/showcase-"${SHOWCASE_VERSION}"-linux-amd64.tar.gz pushd /usr/src/showcase/