Skip to content

Commit 2145857

Browse files
burkedavisonburkedavison
authored
ci: install and test only necessary modules (#10803)
* ci: install and test only necessary modules * ci: debugging mvn install * ci: remove quotes from INTEGRATION_TEST_ARGS * ci: remove debugging lines
1 parent 6b1e9fa commit 2145857

5 files changed

Lines changed: 92 additions & 64 deletions

File tree

‎.cloud/setup.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,5 @@ popd >/dev/null
2929
# Ensure all SNAPSHOTs are available in the local mvn repository
3030
pushd "$scriptDir/.." >/dev/null
3131
source ./.kokoro/common.sh
32-
install_modules
32+
install_modules "$1"
3333
popd >/dev/null

‎.kokoro/build.sh‎

Lines changed: 14 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -38,16 +38,17 @@ RETURN_CODE=0
3838
case ${JOB_TYPE} in
3939
test)
4040
retry_with_backoff 3 10 \
41-
mvn -B -ntp \
42-
-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
43-
-Dclirr.skip=true \
44-
-Denforcer.skip=true \
45-
-Dcheckstyle.skip=true \
46-
-Dflatten.skip=true \
47-
-Danimal.sniffer.skip=true \
48-
-Dmaven.wagon.http.retryHandler.count=5 \
49-
-T 1C \
50-
test
41+
mvn test \
42+
-B -ntp \
43+
-Dorg.slf4j.simpleLogger.showDateTime=true \
44+
-Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
45+
-Dclirr.skip=true \
46+
-Denforcer.skip=true \
47+
-Dcheckstyle.skip=true \
48+
-Dflatten.skip=true \
49+
-Danimal.sniffer.skip=true \
50+
-Dmaven.wagon.http.retryHandler.count=5 \
51+
-T 1C
5152
RETURN_CODE=$?
5253
echo "Finished running unit tests"
5354
;;
@@ -59,7 +60,7 @@ case ${JOB_TYPE} in
5960
echo "${modified_module_list[*]}"
6061
)
6162
setup_cloud "$module_list"
62-
install_modules
63+
install_modules "$module_list"
6364
run_integration_tests "$module_list"
6465
else
6566
echo "No Integration Tests to run"
@@ -69,31 +70,20 @@ case ${JOB_TYPE} in
6970
generate_graalvm_presubmit_modules_list
7071
printf "Running GraalVM presubmit checks for:\n%s\n" "${module_list}"
7172
setup_cloud "$module_list"
72-
install_modules
73+
install_modules "$module_list"
7374
run_graalvm_tests "$module_list"
7475
;;
7576
graalvm)
7677
generate_graalvm_modules_list
7778
if [ ! -z "${module_list}" ]; then
7879
printf "Running GraalVM checks for:\n%s\n" "${module_list}"
7980
setup_cloud "$module_list"
80-
install_modules
81+
install_modules "$module_list"
8182
run_graalvm_tests "$module_list"
8283
else
8384
echo "Not running GraalVM checks -- No changes in relevant modules"
8485
fi
8586
;;
86-
graalvm17)
87-
generate_graalvm_modules_list
88-
if [ ! -z "${module_list}" ]; then
89-
printf "Running GraalVM 17 checks for:\n%s\n" "${module_list}"
90-
setup_cloud "$module_list"
91-
install_modules
92-
run_graalvm_tests "$module_list"
93-
else
94-
echo "Not running GraalVM 17 checks -- No changes in relevant modules"
95-
fi
96-
;;
9787
*) ;;
9888

9989
esac

‎.kokoro/common.sh‎

Lines changed: 75 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,7 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
# TODO: remove java-core once we figure out how setup_cloud understands Maven's
17-
# "--also-make-dependents" option. https://github.com/googleapis/google-cloud-java/issues/9088
18-
excluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent' 'java-core')
16+
excluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent')
1917

2018
function retry_with_backoff {
2119
attempts_left=$1
@@ -128,46 +126,48 @@ function generate_modified_modules_list() {
128126
function run_integration_tests() {
129127
printf "Running Integration Tests for:\n%s\n" "$1"
130128
# --also-make-dependents to run other modules that use the affected module
131-
mvn -B ${INTEGRATION_TEST_ARGS} \
132-
-pl "$1" \
133-
--also-make-dependents \
134-
-ntp \
135-
-Penable-integration-tests \
129+
mvn verify -Penable-integration-tests --projects "$1" \
130+
${INTEGRATION_TEST_ARGS} \
131+
-B -ntp -fae \
136132
-DtrimStackTrace=false \
137133
-Dclirr.skip=true \
138134
-Denforcer.skip=true \
139-
-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
135+
-Dorg.slf4j.simpleLogger.showDateTime=true \
136+
-Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
140137
-Dcheckstyle.skip=true \
141138
-Dflatten.skip=true \
142139
-Danimal.sniffer.skip=true \
143140
-Djacoco.skip=true \
144141
-DskipUnitTests=true \
145142
-Dmaven.wagon.http.retryHandler.count=5 \
146-
-fae \
147-
-T 1C \
148-
verify
143+
-T 1C
149144

150145
RETURN_CODE=$?
151146
printf "Finished Integration Tests for:\n%s\n" "$1"
152147
}
153148

154149
function run_graalvm_tests() {
155-
printf "Running GraalVM ITs on:\n%s\n" "$1"
150+
IFS=, read -ra individual_modules <<< "$1"
151+
projects_to_install=()
152+
for module in "${individual_modules[@]}"; do
153+
p=$(echo "$module" | cut -d '-' -f2-) # Convert from 'java-x-y-z' to 'x-y-z'
154+
projects_to_install+=("java-$p/google-cloud-$p"); # Create reference to GAPIC project
155+
done
156+
IFS=, gapic_projects="${projects_to_install[*]}" # Convert to comma-separated string
157+
158+
printf "Running GraalVM ITs on:\n%s\n" "$gapic_projects"
156159

157-
mvn -B ${INTEGRATION_TEST_ARGS} \
158-
-pl "$1" \
159-
--also-make-dependents \
160-
-ntp \
160+
mvn test -Pnative --projects "$gapic_projects" \
161+
${INTEGRATION_TEST_ARGS} \
162+
-B -ntp -fae \
161163
-DtrimStackTrace=false \
162164
-Dclirr.skip=true \
163165
-Denforcer.skip=true \
164-
-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
166+
-Dorg.slf4j.simpleLogger.showDateTime=true \
167+
-Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
165168
-Dcheckstyle.skip=true \
166169
-Dflatten.skip=true \
167-
-Danimal.sniffer.skip=true \
168-
-Pnative \
169-
-fae \
170-
test
170+
-Danimal.sniffer.skip=true
171171

172172
RETURN_CODE=$?
173173
printf "Finished Unit and Integration Tests for GraalVM:\n%s\n" "$1"
@@ -176,7 +176,7 @@ function run_graalvm_tests() {
176176
function generate_graalvm_presubmit_modules_list() {
177177
modules_assigned_list=()
178178
generate_modified_modules_list
179-
if [[ ${#modified_module_list[@]} -gt 0 && ${#modified_module_list[@]} -lt 10 ]]; then
179+
if [[ ${#modified_module_list[@]} -gt 0 && ${#modified_module_list[@]} -lt 5 ]]; then
180180
# If only a few modules have been modified, focus presubmit testing only on them.
181181
module_list=$(
182182
IFS=,
@@ -229,18 +229,56 @@ function generate_graalvm_modules_list() {
229229
}
230230

231231
function install_modules() {
232-
retry_with_backoff 3 10 \
233-
mvn -B \
234-
-ntp \
235-
-DtrimStackTrace=false \
236-
-Dclirr.skip=true \
237-
-Denforcer.skip=true \
238-
-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
239-
-Dcheckstyle.skip=true \
240-
-Dflatten.skip=true \
241-
-Danimal.sniffer.skip=true \
242-
-DskipTests=true \
243-
-Djacoco.skip=true \
244-
-T 1C \
245-
install
232+
if [ -z "$1" ]; then
233+
mvn install \
234+
-B -ntp \
235+
-DtrimStackTrace=false \
236+
-Dclirr.skip=true \
237+
-Denforcer.skip=true \
238+
-Dorg.slf4j.simpleLogger.showDateTime=true \
239+
-Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
240+
-Dcheckstyle.skip=true \
241+
-Dflatten.skip=true \
242+
-Danimal.sniffer.skip=true \
243+
-DskipTests=true \
244+
-Djacoco.skip=true \
245+
-T 1C
246+
else
247+
IFS=, read -ra individual_modules <<< "$1"
248+
projects_to_install=()
249+
for module in "${individual_modules[@]}"; do
250+
p=$(echo "$module" | cut -d '-' -f2-) # Convert from 'java-x-y-z' to 'x-y-z'
251+
projects_to_install+=("java-$p/google-cloud-$p"); # Create reference to GAPIC project
252+
done
253+
IFS=, gapic_projects="${projects_to_install[*]}" # Convert to comma-separated string
254+
255+
# When working with a maven multi-module project containing other multi-module projects,
256+
# to build a module with its dependencies and without building its dependents:
257+
# Perform the install command on a grandchild module with the --also-make flag.
258+
#
259+
# Examples:
260+
#
261+
# mvn install --projects java-asset --also-make
262+
# ! Does not work. Maven reactor will not build java-asset's child modules, such as the
263+
# gapic, proto, and grpc modules.
264+
#
265+
# mvn install --projects java-kms --also-make-dependents
266+
# ! Does not work. Maven reactor will include java-kmsinventory in its build.
267+
#
268+
# mvn install --projects java-kms/google-cloud-kms --also-make
269+
# Correctly builds dependencies without building dependents.
270+
mvn install --projects "$gapic_projects" --also-make \
271+
-B -ntp \
272+
-DtrimStackTrace=false \
273+
-Dclirr.skip=true \
274+
-Denforcer.skip=true \
275+
-Dorg.slf4j.simpleLogger.showDateTime=true \
276+
-Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
277+
-Dcheckstyle.skip=true \
278+
-Dflatten.skip=true \
279+
-Danimal.sniffer.skip=true \
280+
-DskipTests=true \
281+
-Djacoco.skip=true \
282+
-T 1C
283+
fi
246284
}

‎.kokoro/nightly/graalvm-native-17.cfg‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ env_vars: {
88

99
env_vars: {
1010
key: "JOB_TYPE"
11-
value: "graalvm17"
11+
value: "graalvm"
1212
}
1313

1414
# TODO: remove this after we've migrated all tests and scripts

‎.kokoro/nightly/graalvm-sub-jobs/native-17/common.cfg‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ env_vars: {
2929

3030
env_vars: {
3131
key: "JOB_TYPE"
32-
value: "graalvm17"
32+
value: "graalvm"
3333
}
3434

3535
# TODO: remove this after we've migrated all tests and scripts

0 commit comments

Comments
 (0)