Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions .kokoro/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -121,12 +121,14 @@ case ${JOB_TYPE} in
integration-single)
if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then
echo "Not running integration checks -- this is Release Please SNAPSHOT pull request."
# Run tests if either global overrides require testing all modules (e.g. parent POM or
# shared dependencies) OR if this specific module was modified. Otherwise skip.
elif ! should_test_all_modules && ! is_module_modified "${BUILD_SUBDIR}"; then
echo "${BUILD_SUBDIR} not modified, skipping split integration test"
# Run tests if:
# 1. Global overrides require testing all modules (e.g. parent POM, sdk-platform-java, auth)
# 2. This specific module was modified
# 3. An upstream dependency of this module was modified (e.g. java-spanner or grpc-gcp-java for java-spanner-jdbc)
elif ! should_test_all_modules && ! is_module_modified "${BUILD_SUBDIR}" && ! is_upstream_module_modified "${BUILD_SUBDIR}"; then
echo "${BUILD_SUBDIR} not modified and no upstream dependencies modified, skipping split integration test"
else
echo "${BUILD_SUBDIR} modified, running split integration test"
echo "${BUILD_SUBDIR} (or an upstream dependency) modified, running split integration test"
echo "Compiling and building all modules for ${BUILD_SUBDIR}"
install_modules "${BUILD_SUBDIR}"
echo "Running in subdir: ${BUILD_SUBDIR}"
Expand Down Expand Up @@ -191,12 +193,14 @@ case ${JOB_TYPE} in
graalvm-single)
if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then
echo "Not running GraalVM checks -- this is Release Please SNAPSHOT pull request."
# Run tests if either global overrides require testing all modules (e.g. parent POM or
# shared dependencies) OR if this specific module was modified. Otherwise skip.
elif ! should_test_all_modules && ! is_module_modified "${BUILD_SUBDIR}"; then
echo "${BUILD_SUBDIR} not modified, skipping split GraalVM test"
# Run tests if:
# 1. Global overrides require testing all modules (e.g. parent POM, sdk-platform-java, auth)
# 2. This specific module was modified
# 3. An upstream dependency of this module was modified (e.g. java-spanner or grpc-gcp-java for java-spanner-jdbc)
elif ! should_test_all_modules && ! is_module_modified "${BUILD_SUBDIR}" && ! is_upstream_module_modified "${BUILD_SUBDIR}"; then
echo "${BUILD_SUBDIR} not modified and no upstream dependencies modified, skipping split GraalVM test"
else
echo "${BUILD_SUBDIR} modified, running split GraalVM test"
echo "${BUILD_SUBDIR} (or an upstream dependency) modified, running split GraalVM test"
echo "Compiling and building all modules for ${BUILD_SUBDIR}"
install_modules "${BUILD_SUBDIR}"
echo "Running in subdir: ${BUILD_SUBDIR}"
Expand Down
76 changes: 72 additions & 4 deletions .kokoro/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -250,12 +250,14 @@ function get_modified_files() {

# Determines if the entire monorepo must be tested.
#
# Monorepo-wide testing is triggered under three conditions:
# Monorepo-wide testing is triggered under four conditions:
# 1. TEST_ALL_MODULES is set to "true" (used by nightly and scheduled CI builds).
# 2. Root parent POMs (google-cloud-jar-parent or google-cloud-pom-parent) are modified,
# as changes to parent POMs affect shared dependency versions and compiler/build plugins.
# 3. Core shared dependencies (sdk-platform-java/java-shared-dependencies) are modified,
# as gax, auth, and transport changes can break downstream client library integration tests.
# 3. Core SDK platform libraries (sdk-platform-java) are modified, as gax, generators,
# and core transport changes can break downstream client library integration tests.
# 4. Core authentication libraries (google-auth-library-java) are modified, as auth/credential
# changes affect all client libraries.
function should_test_all_modules() {
local files
files=$(get_modified_files)
Expand All @@ -264,7 +266,8 @@ function should_test_all_modules() {
# stdin of grep, avoiding an external subshell pipeline (like 'echo "$var" | grep').
if [[ "${TEST_ALL_MODULES}" == "true" ]] || \
grep -q -E '^google-cloud-(pom|jar)-parent/pom.xml$' <<< "${files}" || \
grep -q -E '^sdk-platform-java/java-shared-dependencies/' <<< "${files}"; then
grep -q -E '^sdk-platform-java/' <<< "${files}" || \
grep -q -E '^google-auth-library-java/' <<< "${files}"; then
return 0
fi
return 1
Expand Down Expand Up @@ -317,6 +320,16 @@ function generate_modified_modules_list() {
else
echo "Found no changes in the java modules"
fi

# Also include downstream modules if any of their upstream dependencies were modified,
# ensuring batch integration tests cover dependent client libraries.
for module in "${maven_modules[@]}"; do
if is_upstream_module_modified "${module}"; then
if [[ ! " ${modified_module_list[*]} " =~ " ${module} " ]]; then
modified_module_list+=("${module}")
fi
fi
done
fi
}

Expand All @@ -337,6 +350,61 @@ function is_module_modified() {
grep -q -E "^${module}/" <<< "${files}"
}

# Maps a module to its intra-monorepo upstream dependencies.
#
# Certain libraries in this repository directly depend on sibling modules (for example,
# java-bigquery depends on java-bigquerystorage, JDBC drivers wrap client SDKs, and
# java-spanner depends on grpc-gcp-java). Without this mapping, changes to an upstream
# dependency would not trigger integration tests for downstream consumers in PR CI.
#
# Returns space-separated module names that the given module depends on, or empty if none.
function get_upstream_modules() {
local module="$1"
case "${module}" in
java-bigquery)
echo "java-bigquerystorage"
;;
java-bigquery-jdbc)
echo "java-bigquery java-bigquerystorage"
;;
java-spanner)
echo "grpc-gcp-java"
;;
java-spanner-jdbc)
echo "java-spanner grpc-gcp-java"
;;
java-storage-nio)
echo "java-storage"
;;
java-logging-logback)
echo "java-logging"
;;
*)
;;
esac
}

# Checks if any upstream dependency of the given module was modified in the PR diff.
#
# Takes a module name (e.g. BUILD_SUBDIR), retrieves its upstream dependencies using
# get_upstream_modules, and checks if any of those upstream directories were touched.
# Returns 0 (true) if an upstream module was modified, triggering downstream tests;
# otherwise returns 1 (false).
function is_upstream_module_modified() {
local module="$1"
if [[ -z "${module}" ]]; then
return 1
fi

local upstream
for upstream in $(get_upstream_modules "${module}"); do
if is_module_modified "${upstream}"; then
return 0
fi
done
return 1
}

# Filters the modified_module_list to only include modules that contain
# integration test files (matching IT*.java or *IT.java in src/test/java).
# Not all modules will have ITs written and there is not need to test
Expand Down
131 changes: 124 additions & 7 deletions .kokoro/common_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -155,17 +155,31 @@ function test_should_test_all_modules {
exit 1
fi

# Core shared dependencies
TEST_MODIFIED_FILES="sdk-platform-java/java-shared-dependencies/pom.xml"
# Core SDK platform
TEST_MODIFIED_FILES="sdk-platform-java/gapic-generator/src/main/Foo.java"
if ! should_test_all_modules; then
echo "should_test_all_modules should return true for java-shared-dependencies change"
echo "should_test_all_modules should return true for sdk-platform-java change"
exit 1
fi

# Prefix collision check: sibling paths starting with java-shared-dependencies must not match
TEST_MODIFIED_FILES="sdk-platform-java/java-shared-dependencies-bom/pom.xml"
# Prefix collision check: sibling paths starting with sdk-platform-java must not match
TEST_MODIFIED_FILES="sdk-platform-java-extra/pom.xml"
if should_test_all_modules; then
echo "should_test_all_modules should return false for java-shared-dependencies prefix match"
echo "should_test_all_modules should return false for sdk-platform-java prefix match"
exit 1
fi

# Core auth library
TEST_MODIFIED_FILES="google-auth-library-java/oauth2_http/src/main/Foo.java"
if ! should_test_all_modules; then
echo "should_test_all_modules should return true for google-auth-library-java change"
exit 1
fi

# Prefix collision check: sibling paths starting with google-auth-library-java must not match
TEST_MODIFIED_FILES="google-auth-library-java-extra/pom.xml"
if should_test_all_modules; then
echo "should_test_all_modules should return false for google-auth-library-java prefix match"
exit 1
fi

Expand All @@ -179,6 +193,109 @@ function test_should_test_all_modules {
unset TEST_MODIFIED_FILES
}

# Test is_upstream_module_modified triggers correctly for downstream components.
function test_is_upstream_module_modified {
# Empty input should return false (1)
if is_upstream_module_modified ""; then
echo "is_upstream_module_modified should return false for empty input"
exit 1
fi

# 1. java-bigquerystorage modified:
# - java-bigquery should trigger
# - java-bigquery-jdbc should trigger
# - java-bigquerystorage itself should NOT have upstream trigger
TEST_MODIFIED_FILES="java-bigquerystorage/src/main/Foo.java"
if ! is_upstream_module_modified "java-bigquery"; then
echo "is_upstream_module_modified should return true for java-bigquery when java-bigquerystorage is modified"
exit 1
fi
if ! is_upstream_module_modified "java-bigquery-jdbc"; then
echo "is_upstream_module_modified should return true for java-bigquery-jdbc when java-bigquerystorage is modified"
exit 1
fi
if is_upstream_module_modified "java-bigquerystorage"; then
echo "is_upstream_module_modified should return false for java-bigquerystorage when java-bigquerystorage is modified"
exit 1
fi

# 2. java-bigquery modified:
# - java-bigquery-jdbc should trigger
# - java-bigquerystorage should NOT trigger
TEST_MODIFIED_FILES="java-bigquery/src/main/Foo.java"
if ! is_upstream_module_modified "java-bigquery-jdbc"; then
echo "is_upstream_module_modified should return true for java-bigquery-jdbc when java-bigquery is modified"
exit 1
fi
if is_upstream_module_modified "java-bigquerystorage"; then
echo "is_upstream_module_modified should return false for java-bigquerystorage when java-bigquery is modified"
exit 1
fi

# 3. grpc-gcp-java modified:
# - java-spanner should trigger
# - java-spanner-jdbc should trigger
TEST_MODIFIED_FILES="grpc-gcp-java/src/main/Foo.java"
if ! is_upstream_module_modified "java-spanner"; then
echo "is_upstream_module_modified should return true for java-spanner when grpc-gcp-java is modified"
exit 1
fi
if ! is_upstream_module_modified "java-spanner-jdbc"; then
echo "is_upstream_module_modified should return true for java-spanner-jdbc when grpc-gcp-java is modified"
exit 1
fi

# 4. java-spanner modified:
# - java-spanner-jdbc should trigger
# - java-spanner itself should NOT have upstream trigger
TEST_MODIFIED_FILES="java-spanner/src/main/Foo.java"
if ! is_upstream_module_modified "java-spanner-jdbc"; then
echo "is_upstream_module_modified should return true for java-spanner-jdbc when java-spanner is modified"
exit 1
fi
if is_upstream_module_modified "java-spanner"; then
echo "is_upstream_module_modified should return false for java-spanner when java-spanner is modified"
exit 1
fi

# 5. java-storage modified:
# - java-storage-nio should trigger
# - java-storage itself should NOT have upstream trigger
TEST_MODIFIED_FILES="java-storage/src/main/Foo.java"
if ! is_upstream_module_modified "java-storage-nio"; then
echo "is_upstream_module_modified should return true for java-storage-nio when java-storage is modified"
exit 1
fi
if is_upstream_module_modified "java-storage"; then
echo "is_upstream_module_modified should return false for java-storage when java-storage is modified"
exit 1
fi

# 6. java-logging modified:
# - java-logging-logback should trigger
# - java-logging itself should NOT have upstream trigger
TEST_MODIFIED_FILES="java-logging/src/main/Foo.java"
if ! is_upstream_module_modified "java-logging-logback"; then
echo "is_upstream_module_modified should return true for java-logging-logback when java-logging is modified"
exit 1
fi
if is_upstream_module_modified "java-logging"; then
echo "is_upstream_module_modified should return false for java-logging when java-logging is modified"
exit 1
fi

# 7. Unrelated module modified:
TEST_MODIFIED_FILES="java-asset/src/main/Foo.java"
if is_upstream_module_modified "java-asset" || \
is_upstream_module_modified "java-spanner" || \
is_upstream_module_modified "java-bigquery"; then
echo "is_upstream_module_modified should return false when only an unrelated module is modified"
exit 1
fi

unset TEST_MODIFIED_FILES
}
Comment thread
lqiu96 marked this conversation as resolved.

# Test mock get_modified_files returns simulated files.
function test_mock_get_modified_files {
TEST_MODIFIED_FILES="dummy/file.txt"
Expand All @@ -197,4 +314,4 @@ test_parse_pom_version
test_mock_get_modified_files
test_should_test_all_modules
test_is_module_modified

test_is_upstream_module_modified
Loading