diff --git a/.kokoro/common.sh b/.kokoro/common.sh index ee6770a9d6e1..e1744f9b8b59 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -90,6 +90,9 @@ function retry_with_backoff { # and naturally survives single-module components without throwing exit signals. function extract_pom_modules() { local pom_file="$1" + if [[ ! -f "${pom_file}" ]]; then + return 1 + fi local modules_list="" local in_profiles=false local in_modules=false @@ -109,9 +112,8 @@ function extract_pom_modules() { local module="${line#*}" module="${module%*}" - # Trim whitespace natively - module="${module#"${module%%[![:space:]]*}"}" - module="${module%"${module##*[![:space:]]}"}" + # Trim leading/trailing whitespace without spawning external processes + read -r module <<< "${module}" if [ -z "$modules_list" ]; then modules_list="$module" @@ -282,39 +284,48 @@ function generate_modified_modules_list() { files=$(get_modified_files) printf "Modified files:\n%s\n" "${files}" - # Generate the list of valid maven modules - maven_modules_list=$(mvn help:evaluate -Dexpression=project.modules | grep '<.*>.*' | sed -e 's/<.*>\(.*\)<\/.*>/\1/g') + # Extract valid maven modules directly from pom.xml in pure Bash (~0.02s). + # This replaces 'mvn help:evaluate -Dexpression=project.modules' which previously + # spent 20-30+ seconds booting a JVM and evaluating the monorepo POMs on every run. + local maven_modules_list + maven_modules_list=$(extract_pom_modules pom.xml) maven_modules=() - # If the first argument is "true" (default), then use the module exclusion list - use_exclusion_list=${1:-true} + # Positional parameter $1 specifies whether to apply the exclusion list (defaults to true). + local use_exclusion_list="${1:-true}" + local module if [[ "${use_exclusion_list}" == "true" ]]; then echo "Excluding modules from the global exclusion list" - for module in $maven_modules_list; do + for module in ${maven_modules_list}; do if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]]; then maven_modules+=("${module}") fi done else - maven_modules=(${maven_modules_list[*]}) + # 'read -r -a ARRAY <<< STRING' parses a space-delimited string into a Bash array. + read -r -a maven_modules <<< "${maven_modules_list}" fi modified_module_list=() # If either parent pom.xml or core shared dependency is touched, run ITs on all the modules if should_test_all_modules; then - modified_module_list=(${maven_modules[*]}) + # '("${maven_modules[@]}")' copies the array elements safely. + modified_module_list=("${maven_modules[@]}") echo "Testing the entire monorepo" else - modules=$(echo "${files}" | grep -E '(google-auth|java)-.*' || true) + # Extract the top-level directory from each modified file path: + # 'cut -d '/' -f1' takes the first path segment (e.g. 'java-bigquery/src/...' -> 'java-bigquery'). + # 'sort -u' sorts and deduplicates the candidate directory names. + local modules + modules=$(cut -d '/' -f1 <<< "${files}" | sort -u) printf "Files in java modules:\n%s\n" "${modules}" - if [[ -n $modules ]]; then - modules=$(echo "${modules}" | cut -d '/' -f1 | sort -u) - for module in $modules; do - if [[ " ${maven_modules[*]} " =~ " ${module} " ]]; then - modified_module_list+=("${module}") - fi - done - else + for module in ${modules}; do + # If this top-level directory is a recognized Maven module, add it to our list. + if [[ " ${maven_modules[*]} " =~ " ${module} " ]]; then + modified_module_list+=("${module}") + fi + done + if [[ ${#modified_module_list[@]} -eq 0 ]]; then echo "Found no changes in the java modules" fi fi diff --git a/.kokoro/common_test.sh b/.kokoro/common_test.sh index 247a6a26cbfc..bc1b5368cb35 100755 --- a/.kokoro/common_test.sh +++ b/.kokoro/common_test.sh @@ -81,6 +81,35 @@ function test_parse_pom_version { popd } +# Test that extract_pom_modules correctly extracts modules from root pom.xml +# in pure Bash without launching Maven, and handles non-existent files safely. +function test_extract_pom_modules { + local -a modules + read -r -a modules <<< "$(extract_pom_modules "${scriptDir}/../pom.xml")" + if (( ${#modules[@]} < 200 )); then + echo "extract_pom_modules failed: expected at least 200 modules, got ${#modules[@]}" + exit 1 + fi + if [[ ! " ${modules[*]} " =~ " java-bigquery " ]]; then + echo "extract_pom_modules missing java-bigquery" + exit 1 + fi + if [[ ! " ${modules[*]} " =~ " java-bigquerystorage " ]]; then + echo "extract_pom_modules missing java-bigquerystorage" + exit 1 + fi + if [[ ! " ${modules[*]} " =~ " sdk-platform-java " ]]; then + echo "extract_pom_modules missing sdk-platform-java" + exit 1 + fi + + # Verify non-existent file returns 1 and empty output + if extract_pom_modules "non_existent_pom.xml" &>/dev/null; then + echo "extract_pom_modules should return non-zero for non-existent pom" + exit 1 + fi +} + # Tests that is_module_modified strictly matches the module directory prefix, # preventing prefix collisions (e.g. java-bigquery vs java-bigquerystorage). function test_is_module_modified { @@ -191,10 +220,51 @@ function test_mock_get_modified_files { unset TEST_MODIFIED_FILES } +# Test that generate_modified_modules_list correctly maps modified files to +# their top-level module names without requiring mvn help:evaluate. +function test_generate_modified_modules_list { + pushd "${scriptDir}/.." >/dev/null + TEST_MODIFIED_FILES="java-bigquery/google-cloud-bigquery/src/main/java/Foo.java +sdk-platform-java/gax-java/pom.xml" + generate_modified_modules_list false >/dev/null + + local has_bigquery="false" + local has_sdk_platform="false" + local has_bigquerystorage="false" + + if [[ " ${modified_module_list[*]} " =~ " java-bigquery " ]]; then + has_bigquery="true" + fi + if [[ " ${modified_module_list[*]} " =~ " sdk-platform-java " ]]; then + has_sdk_platform="true" + fi + if [[ " ${modified_module_list[*]} " =~ " java-bigquerystorage " ]]; then + has_bigquerystorage="true" + fi + + popd >/dev/null + unset TEST_MODIFIED_FILES + + if [[ "${has_bigquery}" != "true" ]]; then + echo "generate_modified_modules_list missing java-bigquery" + exit 1 + fi + if [[ "${has_sdk_platform}" != "true" ]]; then + echo "generate_modified_modules_list missing sdk-platform-java" + exit 1 + fi + if [[ "${has_bigquerystorage}" == "true" ]]; then + echo "generate_modified_modules_list incorrectly included java-bigquerystorage" + exit 1 + fi +} + test_find_all_poms_with_versioned_dependency test_update_pom_dependency test_parse_pom_version test_mock_get_modified_files +test_extract_pom_modules test_should_test_all_modules test_is_module_modified +test_generate_modified_modules_list diff --git a/.kokoro/presubmit/downstream-build.sh b/.kokoro/presubmit/downstream-build.sh index aa6781b1ffee..559ad83b5c6b 100755 --- a/.kokoro/presubmit/downstream-build.sh +++ b/.kokoro/presubmit/downstream-build.sh @@ -47,10 +47,21 @@ pushd java-showcase modify_shared_config popd -# Parse showcase version from the local directory -pushd java-showcase/gapic-showcase -SHOWCASE_VERSION=$(mvn help:evaluate -Dexpression=gapic-showcase.version -q -DforceStdout) -popd +# Extract the showcase version directly from pom.xml using sed: +# - 'sed -n': suppresses default line printing so only explicit matches are printed. +# - 's:...[[:space:]]*\(.*\)[[:space:]]*...:\1:p': captures the version text between the tags, +# stripping inner whitespace, and prints ('p') only matching lines. +# - 'head -n 1': guarantees a single line output if multiple matches exist. +# - 'xargs': trims any remaining leading or trailing whitespace. +# This replaces 'mvn help:evaluate' which previously took 15+ seconds to boot Maven. +SHOWCASE_VERSION=$(sed -n 's:.*[[:space:]]*\(.*\)[[:space:]]*.*:\1:p' java-showcase/gapic-showcase/pom.xml | head -n 1 | xargs) + +# Fail fast with a clear error message if the version could not be parsed, +# preventing malformed curl URLs and ambiguous downstream failures. +if [[ -z "${SHOWCASE_VERSION}" ]]; then + echo "Error: Failed to parse gapic-showcase.version from java-showcase/gapic-showcase/pom.xml" >&2 + exit 1 +fi # Start showcase server mkdir -p /usr/src/showcase