Skip to content

perf(ci): replace mvn help:evaluate with native bash and sed extraction #14218

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
lqiu96 wants to merge 1 commit into
base: fix-split-it-module-detection
Choose a base branch
from
+115 −23
Draft

perf(ci): replace mvn help:evaluate with native bash and sed extraction #14218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 30 additions & 19 deletions .kokoro/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ function retry_with_backoff {
# and naturally survives single-module components without throwing exit signals.
function extract_pom_modules() {
local pom_file="$1"
if [[ ! -f "${pom_file}" ]]; then
return 1
fi
local modules_list=""
local in_profiles=false
local in_modules=false
Expand All @@ -109,9 +112,8 @@ function extract_pom_modules() {
local module="${line#*<module>}"
module="${module%</module>*}"

# Trim whitespace natively
module="${module#"${module%%[![:space:]]*}"}"
module="${module%"${module##*[![:space:]]}"}"
# Trim leading/trailing whitespace without spawning external processes
read -r module <<< "${module}"

if [ -z "$modules_list" ]; then
modules_list="$module"
Expand Down Expand Up @@ -282,39 +284,48 @@ function generate_modified_modules_list() {
files=$(get_modified_files)
printf "Modified files:\n%s\n" "${files}"

# Generate the list of valid maven modules
maven_modules_list=$(mvn help:evaluate -Dexpression=project.modules | grep '<.*>.*</.*>' | sed -e 's/<.*>\(.*\)<\/.*>/\1/g')
# Extract valid maven modules directly from pom.xml in pure Bash (~0.02s).
# This replaces 'mvn help:evaluate -Dexpression=project.modules' which previously
# spent 20-30+ seconds booting a JVM and evaluating the monorepo POMs on every run.
local maven_modules_list
maven_modules_list=$(extract_pom_modules pom.xml)
maven_modules=()

# If the first argument is "true" (default), then use the module exclusion list
use_exclusion_list=${1:-true}
# Positional parameter $1 specifies whether to apply the exclusion list (defaults to true).
local use_exclusion_list="${1:-true}"
local module
if [[ "${use_exclusion_list}" == "true" ]]; then
echo "Excluding modules from the global exclusion list"
for module in $maven_modules_list; do
for module in ${maven_modules_list}; do
if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]]; then
maven_modules+=("${module}")
fi
done
else
maven_modules=(${maven_modules_list[*]})
# 'read -r -a ARRAY <<< STRING' parses a space-delimited string into a Bash array.
read -r -a maven_modules <<< "${maven_modules_list}"
fi

modified_module_list=()
# If either parent pom.xml or core shared dependency is touched, run ITs on all the modules
if should_test_all_modules; then
modified_module_list=(${maven_modules[*]})
# '("${maven_modules[@]}")' copies the array elements safely.
modified_module_list=("${maven_modules[@]}")
echo "Testing the entire monorepo"
else
modules=$(echo "${files}" | grep -E '(google-auth|java)-.*' || true)
# Extract the top-level directory from each modified file path:
# 'cut -d '/' -f1' takes the first path segment (e.g. 'java-bigquery/src/...' -> 'java-bigquery').
# 'sort -u' sorts and deduplicates the candidate directory names.
local modules
modules=$(cut -d '/' -f1 <<< "${files}" | sort -u)
printf "Files in java modules:\n%s\n" "${modules}"
if [[ -n $modules ]]; then
modules=$(echo "${modules}" | cut -d '/' -f1 | sort -u)
for module in $modules; do
if [[ " ${maven_modules[*]} " =~ " ${module} " ]]; then
modified_module_list+=("${module}")
fi
done
else
for module in ${modules}; do
# If this top-level directory is a recognized Maven module, add it to our list.
if [[ " ${maven_modules[*]} " =~ " ${module} " ]]; then
modified_module_list+=("${module}")
fi
done
if [[ ${#modified_module_list[@]} -eq 0 ]]; then
echo "Found no changes in the java modules"
fi
fi
Expand Down
70 changes: 70 additions & 0 deletions .kokoro/common_test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,35 @@ function test_parse_pom_version {
popd
}

# Test that extract_pom_modules correctly extracts modules from root pom.xml
# in pure Bash without launching Maven, and handles non-existent files safely.
function test_extract_pom_modules {
local -a modules
read -r -a modules <<< "$(extract_pom_modules "${scriptDir}/../pom.xml")"
if (( ${#modules[@]} < 200 )); then
echo "extract_pom_modules failed: expected at least 200 modules, got ${#modules[@]}"
exit 1
fi
if [[ ! " ${modules[*]} " =~ " java-bigquery " ]]; then
echo "extract_pom_modules missing java-bigquery"
exit 1
fi
if [[ ! " ${modules[*]} " =~ " java-bigquerystorage " ]]; then
echo "extract_pom_modules missing java-bigquerystorage"
exit 1
fi
if [[ ! " ${modules[*]} " =~ " sdk-platform-java " ]]; then
echo "extract_pom_modules missing sdk-platform-java"
exit 1
fi

# Verify non-existent file returns 1 and empty output
if extract_pom_modules "non_existent_pom.xml" &>/dev/null; then
echo "extract_pom_modules should return non-zero for non-existent pom"
exit 1
fi
}

# Tests that is_module_modified strictly matches the module directory prefix,
# preventing prefix collisions (e.g. java-bigquery vs java-bigquerystorage).
function test_is_module_modified {
Expand Down Expand Up @@ -191,10 +220,51 @@ function test_mock_get_modified_files {
unset TEST_MODIFIED_FILES
}

# Test that generate_modified_modules_list correctly maps modified files to
# their top-level module names without requiring mvn help:evaluate.
function test_generate_modified_modules_list {
pushd "${scriptDir}/.." >/dev/null
TEST_MODIFIED_FILES="java-bigquery/google-cloud-bigquery/src/main/java/Foo.java
sdk-platform-java/gax-java/pom.xml"
generate_modified_modules_list false >/dev/null

local has_bigquery="false"
local has_sdk_platform="false"
local has_bigquerystorage="false"

if [[ " ${modified_module_list[*]} " =~ " java-bigquery " ]]; then
has_bigquery="true"
fi
if [[ " ${modified_module_list[*]} " =~ " sdk-platform-java " ]]; then
has_sdk_platform="true"
fi
if [[ " ${modified_module_list[*]} " =~ " java-bigquerystorage " ]]; then
has_bigquerystorage="true"
fi

popd >/dev/null
unset TEST_MODIFIED_FILES

if [[ "${has_bigquery}" != "true" ]]; then
echo "generate_modified_modules_list missing java-bigquery"
exit 1
fi
if [[ "${has_sdk_platform}" != "true" ]]; then
echo "generate_modified_modules_list missing sdk-platform-java"
exit 1
fi
if [[ "${has_bigquerystorage}" == "true" ]]; then
echo "generate_modified_modules_list incorrectly included java-bigquerystorage"
exit 1
fi
}

test_find_all_poms_with_versioned_dependency
test_update_pom_dependency
test_parse_pom_version
test_mock_get_modified_files
test_extract_pom_modules
test_should_test_all_modules
test_is_module_modified
test_generate_modified_modules_list

19 changes: 15 additions & 4 deletions .kokoro/presubmit/downstream-build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,21 @@ pushd java-showcase
modify_shared_config
popd

# Parse showcase version from the local directory
pushd java-showcase/gapic-showcase
SHOWCASE_VERSION=$(mvn help:evaluate -Dexpression=gapic-showcase.version -q -DforceStdout)
popd
# Extract the showcase version directly from pom.xml using sed:
# - 'sed -n': suppresses default line printing so only explicit matches are printed.
# - 's:...[[:space:]]*\(.*\)[[:space:]]*...:\1:p': captures the version text between the tags,
# stripping inner whitespace, and prints ('p') only matching lines.
# - 'head -n 1': guarantees a single line output if multiple matches exist.
# - 'xargs': trims any remaining leading or trailing whitespace.
# This replaces 'mvn help:evaluate' which previously took 15+ seconds to boot Maven.
SHOWCASE_VERSION=$(sed -n 's:.*<gapic-showcase\.version>[[:space:]]*\(.*\)[[:space:]]*</gapic-showcase\.version>.*:\1:p' java-showcase/gapic-showcase/pom.xml | head -n 1 | xargs)

# Fail fast with a clear error message if the version could not be parsed,
# preventing malformed curl URLs and ambiguous downstream failures.
if [[ -z "${SHOWCASE_VERSION}" ]]; then
echo "Error: Failed to parse gapic-showcase.version from java-showcase/gapic-showcase/pom.xml" >&2
exit 1
fi

# Start showcase server
mkdir -p /usr/src/showcase
Expand Down
Loading