Skip to content

Commit 92f300a

Browse files
chingor13chingor13
authored
build: use matrix for split unit tests (#12127)
Uses a GitHub actions matrix and the dorny/paths-filter action (already in use) to conditionally run split unit tests for a select number of modules. When running the bulk unit test suite, we enable use the `bulkTests` profile. This profile is configured to skip the tests for the ones that have split tests. Also fixes the showcase-clirr check by installing necessary dependencies without running the enforcer check. Fixes #12185
1 parent 30f8afb commit 92f300a

11 files changed

Lines changed: 180 additions & 155 deletions

File tree

‎.github/workflows/ci.yaml‎

Lines changed: 88 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,85 @@ jobs:
5353
id: date
5454
run: echo "::set-output name=week_of_year::$(date +'%W' --utc)"
5555
- uses: actions/checkout@v4
56+
- uses: actions/setup-java@v4
57+
with:
58+
java-version: 8
59+
distribution: temurin
60+
- name: "Set jvm system property environment variable for surefire plugin (unit tests)"
61+
# Maven surefire plugin (unit tests) allows us to specify JVM to run the tests.
62+
# https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#jvm
63+
run: echo "SUREFIRE_JVM_OPT=-Djvm=${JAVA_HOME}/bin/java" >> $GITHUB_ENV
64+
shell: bash
65+
- uses: actions/setup-java@v4
66+
with:
67+
java-version: 11
68+
distribution: temurin
69+
cache: maven
70+
- uses: actions/cache@v4
71+
id: mvn-cache
72+
with:
73+
path: ~/.m2/repository
74+
key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }}
75+
- run: .kokoro/build.sh
76+
shell: bash
77+
env:
78+
JOB_TYPE: test
79+
JOB_NAME: units-8-runtime-${{matrix.java}}
80+
# detect which libraries have changed
81+
changes:
82+
runs-on: ubuntu-latest
83+
permissions:
84+
pull-requests: read
85+
outputs:
86+
packages: ${{ steps.filter.outputs.changes }}
87+
steps:
88+
- uses: dorny/paths-filter@v4
89+
id: filter
90+
with:
91+
filters: |
92+
java-bigquery: java-bigquery/**
93+
java-bigquerystorage: java-bigquerystorage/**
94+
java-datastore: java-datastore/**
95+
java-logging: java-logging/**
96+
java-logging-logback: java-logging-logback/**
97+
java-spanner: java-spanner/**
98+
sdk-platform-java: sdk-platform-java/**
99+
split-units:
100+
runs-on: ubuntu-latest
101+
needs: changes
102+
strategy:
103+
fail-fast: false
104+
matrix:
105+
package: ${{ fromJSON(needs.changes.outputs.packages) }}
106+
java: [11, 17, 21, 25]
107+
steps:
108+
- name: Get current week within the year
109+
id: date
110+
run: echo "::set-output name=week_of_year::$(date +'%W' --utc)"
111+
- uses: actions/checkout@v4
112+
- uses: actions/setup-java@v4
113+
with:
114+
distribution: temurin
115+
java-version: ${{matrix.java}}
116+
- run: .kokoro/build.sh
117+
env:
118+
BUILD_SUBDIR: ${{matrix.package}}
119+
JOB_TYPE: test
120+
JOB_NAME: units-${{matrix.package}}-${{matrix.java}}
121+
split-units-8:
122+
runs-on: ubuntu-latest
123+
name: "split-units"
124+
needs: changes
125+
strategy:
126+
fail-fast: false
127+
matrix:
128+
package: ${{ fromJSON(needs.changes.outputs.packages) }}
129+
java: [8]
130+
steps:
131+
- name: Get current week within the year
132+
id: date
133+
run: echo "::set-output name=week_of_year::$(date +'%W' --utc)"
134+
- uses: actions/checkout@v4
56135
- uses: actions/setup-java@v4
57136
with:
58137
java-version: 11
@@ -66,15 +145,16 @@ jobs:
66145
key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }}
67146
- name: Install all modules using Java 11
68147
shell: bash
69-
run: |
70-
mvn -V -B -ntp clean install -DskipTests -Dfmt.skip -Dclirr.skip -Denforcer.skip -Dcheckstyle.skip -Dflatten.skip -Danimal-sniffer.skip -Djacoco.skip
71-
working-directory: sdk-platform-java
148+
run: .kokoro/build.sh
149+
env:
150+
BUILD_SUBDIR: ${{matrix.package}}
151+
JOB_TYPE: install
72152
- uses: actions/setup-java@v4
73153
with:
74-
java-version: 8
154+
java-version: ${{matrix.java}}
75155
distribution: temurin
76156
- run: java -version
77-
- name: Run tests in Java 8 with the source compiled in Java 11
157+
- name: Run tests in Java ${{matrix.java}} with the source compiled in Java 11
78158
run: |
79159
mvn test \
80160
-B -ntp \
@@ -86,11 +166,13 @@ jobs:
86166
-Dflatten.skip=true \
87167
-Danimal.sniffer.skip=true \
88168
-Dmaven.wagon.http.retryHandler.count=5 \
169+
--also-make \
89170
-T 1C
90171
env:
172+
BUILD_SUBDIR: ${{matrix.package}}
91173
JOB_TYPE: test
92174
JOB_NAME: units-8-runtime-${{matrix.java}}
93-
working-directory: sdk-platform-java
175+
working-directory: ${{matrix.package}}
94176
windows:
95177
runs-on: windows-latest
96178
steps:

‎.github/workflows/java-bigquery-ci.yaml‎

Lines changed: 0 additions & 143 deletions
This file was deleted.

‎.github/workflows/sdk-platform-java-ci.yaml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -430,8 +430,11 @@ jobs:
430430
- name: Checkout sdk-platform-java @ PR merge commit
431431
uses: actions/checkout@v3
432432
- name: Install sdk-platform-java @ PR merge commit
433-
run: mvn install -B -ntp -T 1C -DskipTests
434-
working-directory: sdk-platform-java
433+
shell: bash
434+
run: .kokoro/build.sh
435+
env:
436+
JOB_TYPE: install
437+
BUILD_SUBDIR: sdk-platform-java
435438
# Showcase golden test ensures that src changes are already reflected in the PR.
436439
- name: Clirr check
437440
working-directory: sdk-platform-java/java-showcase

‎.kokoro/build.sh‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,9 @@ case ${JOB_TYPE} in
4343
install_modules "${BUILD_SUBDIR}"
4444
echo "Running in subdir: ${BUILD_SUBDIR}"
4545
pushd "${BUILD_SUBDIR}"
46+
EXTRA_PROFILE_OPTS=()
4647
else
48+
EXTRA_PROFILE_OPTS=("-PbulkTests")
4749
install_modules "sdk-platform-java"
4850
fi
4951
echo "SUREFIRE_JVM_OPT: ${SUREFIRE_JVM_OPT}"
@@ -59,7 +61,7 @@ case ${JOB_TYPE} in
5961
-Danimal.sniffer.skip=true \
6062
-Dmaven.wagon.http.retryHandler.count=5 \
6163
--also-make \
62-
${SUREFIRE_JVM_OPT}
64+
${SUREFIRE_JVM_OPT} "${EXTRA_PROFILE_OPTS[@]}"
6365
RETURN_CODE=$?
6466

6567
if [[ -n "${BUILD_SUBDIR}" ]]
@@ -69,6 +71,28 @@ case ${JOB_TYPE} in
6971
fi
7072
echo "Finished running unit tests"
7173
;;
74+
install)
75+
if [[ -n "${BUILD_SUBDIR}" ]]
76+
then
77+
echo "Compiling and building all modules for ${BUILD_SUBDIR}"
78+
install_modules "${BUILD_SUBDIR}"
79+
else
80+
install_modules "sdk-platform-java"
81+
mvn install \
82+
-B -ntp \
83+
-Dorg.slf4j.simpleLogger.showDateTime=true \
84+
-Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
85+
-Dclirr.skip=true \
86+
-Denforcer.skip=true \
87+
-Dcheckstyle.skip=true \
88+
-Dflatten.skip=true \
89+
-Danimal.sniffer.skip=true \
90+
-Dmaven.wagon.http.retryHandler.count=5 \
91+
-DskipTests=true \
92+
--also-make \
93+
-T 1C
94+
fi
95+
;;
7296
integration)
7397
generate_modified_modules_list
7498
if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then

‎java-bigquery/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,5 +162,12 @@
162162
<module>benchmark</module>
163163
</modules>
164164
</profile>
165+
<!-- skip tests if bulkTests profile is enabled (when testing the rest of the monorepo) -->
166+
<profile>
167+
<id>bulkTests</id>
168+
<properties>
169+
<skipTests>true</skipTests>
170+
</properties>
171+
</profile>
165172
</profiles>
166173
</project>

‎java-bigquerystorage/pom.xml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,4 +202,14 @@
202202
<module>google-cloud-bigquerystorage-bom</module>
203203
</modules>
204204

205-
</project>
205+
<profiles>
206+
<!-- skip tests if bulkTests profile is enabled (when testing the rest of the monorepo) -->
207+
<profile>
208+
<id>bulkTests</id>
209+
<properties>
210+
<skipTests>true</skipTests>
211+
</properties>
212+
</profile>
213+
</profiles>
214+
215+
</project>

‎java-datastore/pom.xml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -232,4 +232,13 @@
232232
<module>google-cloud-datastore-utils</module>
233233
</modules>
234234

235-
</project>
235+
<profiles>
236+
<!-- skip tests if bulkTests profile is enabled (when testing the rest of the monorepo) -->
237+
<profile>
238+
<id>bulkTests</id>
239+
<properties>
240+
<skipTests>true</skipTests>
241+
</properties>
242+
</profile>
243+
</profiles>
244+
</project>

‎java-logging-logback/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,5 +168,12 @@
168168
<test>!LoggingAppenderTest</test>
169169
</properties>
170170
</profile>
171+
<!-- skip tests if bulkTests profile is enabled (when testing the rest of the monorepo) -->
172+
<profile>
173+
<id>bulkTests</id>
174+
<properties>
175+
<skipTests>true</skipTests>
176+
</properties>
177+
</profile>
171178
</profiles>
172179
</project>

0 commit comments

Comments
 (0)