Skip to content

Commit e537901

Browse files
coryancoryan
authored
ci: build script for developer workflows (#12804)
This script is expected to support /most/ developer workflows. We will be able to create a configuration file in $HOME or in the workspace to build only a subset of the code, with clang-tidy, and to run (or not run) the unit and integration tests associated with that code. I expect this script will grow and be more customizable over time, as the team adds features to match their workflow.
1 parent c618793 commit e537901

10 files changed

Lines changed: 149 additions & 0 deletions

File tree

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,7 @@ cmake-build-*/
2727
# Ignore Visual Studio Code files
2828
.vsbuild/
2929
.vscode/
30+
31+
# `google-cloud-cpp` developers use this file to configure the development
32+
# workflow build.
33+
.cloudcxxrc

‎ci/cloudbuild/build.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,11 @@ if [[ "${DOCKER_FLAG}" = "true" ]]; then
351351
# generator dir.
352352
"--env=GENERATE_GOLDEN_ONLY=${GENERATE_GOLDEN_ONLY-}"
353353
)
354+
if [[ -r "${HOME}/.cloudcxxrc" ]]; then
355+
run_flags+=(
356+
"--volume=${HOME}/.cloudcxxrc:/h/.cloudcxxrc:Z"
357+
)
358+
fi
354359
# All GOOGLE_CLOUD_* env vars will be passed to the docker container.
355360
for e in $(env); do
356361
if [[ "${e}" = GOOGLE_CLOUD_* ]]; then

‎ci/cloudbuild/builds/development.sh‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#!/bin/bash
2+
#
3+
# Copyright 2023 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -euo pipefail
18+
19+
source "$(dirname "$0")/../../lib/init.sh"
20+
source module ci/cloudbuild/builds/lib/cmake.sh
21+
source module ci/cloudbuild/builds/lib/features.sh
22+
source module ci/cloudbuild/builds/lib/integration.sh
23+
source module ci/lib/io.sh
24+
25+
export CC=clang
26+
export CXX=clang++
27+
export CTCACHE_DIR=~/.cache/ctcache
28+
29+
# Load the build configuration. Start with the default, then any settings for
30+
# all workspaces, then the settings for the current workspace.
31+
#
32+
# Developers only need to override what changes, keep a configuration file that
33+
# applies to all their builds, and have some workspaces with custom settings.
34+
source module ci/etc/cloudcxxrc
35+
if [[ -r "${HOME}/.cloudcxxrc" ]]; then
36+
source "${HOME}/.cloudcxxrc"
37+
fi
38+
if [[ -r "${PROJECT_ROOT}/.cloudcxxrc" ]]; then
39+
source "${PROJECT_ROOT}/.cloudcxxrc"
40+
fi
41+
42+
# Always remove the CMakeCache because it may be invalidated by `cloudcxxrc`
43+
# changes.
44+
if [[ "${ALWAYS_RESET_CMAKE_CACHE}" = "YES" ]]; then
45+
io::run rm -f cmake-out/CMakeCache.txt
46+
fi
47+
48+
# Note: we use C++14 for this build because we don't want tidy suggestions that
49+
# require a newer C++ standard.
50+
mapfile -t cmake_args < <(cmake::common_args)
51+
io::run cmake "${cmake_args[@]}" \
52+
-DCMAKE_CXX_CLANG_TIDY=/usr/local/bin/clang-tidy-wrapper \
53+
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
54+
-DCMAKE_CXX_STANDARD=14 \
55+
-DGOOGLE_CLOUD_CPP_ENABLE="${ENABLED_FEATURES}"
56+
io::run cmake --build cmake-out
57+
58+
if [[ "${RUN_CLANG_TIDY_FIX}" = "YES" ]]; then
59+
io::bold clang-tidy -p cmake-out -fix
60+
git_files -z -- '*.h' '*.cc' |
61+
xargs -r -P "$(nproc)" -n 1 -0 clang-tidy -p cmake-out -fix
62+
io::run cmake --build cmake-out
63+
fi
64+
65+
if [[ "${RUN_UNIT_TESTS}" = "YES" ]]; then
66+
mapfile -t ctest_args < <(ctest::common_args)
67+
io::run ctest --test-dir cmake-out "${ctest_args[@]}" -LE integration-test
68+
fi
69+
70+
if [[ "${RUN_INTEGRATION_TESTS}" = "YES" ]]; then
71+
integration::ctest_with_emulators "cmake-out"
72+
fi
73+
74+
# This build should fail if any of the above work generated code differences.
75+
# This may be updated `.bzl` files, or files updated by clang-tidy.
76+
io::log_h2 "Highlight generated code differences"
77+
git diff --exit-code

‎ci/cloudbuild/builds/lib/ctest.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,11 @@ function ctest::common_args() {
3434
)
3535
printf "%s\n" "${args[@]}"
3636
}
37+
38+
function ctest::has_no_tests() {
39+
local dir="$1"
40+
local prefix="$2"
41+
shift 2
42+
local ctest_args=("$@")
43+
ctest --test-dir "${dir}" --show-only -R "${prefix}" "${ctest_args[@]}" 2>&1 | grep -q 'Total Tests: 0'
44+
}

‎ci/etc/cloudcxxrc‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
#
3+
# Copyright 2023 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# Assume the CMakeCache remains valid between runs.
18+
ALWAYS_RESET_CMAKE_CACHE=NO
19+
20+
# By default compile most features.
21+
ENABLED_FEATURES=__ga_libraries__,__experimental_libraries__,opentelemetry,experimental-storage_grpc
22+
23+
# Set to `YES` to run the unit tests.
24+
RUN_UNIT_TESTS=YES
25+
26+
# Set to `YES` to run the clang-tidy fixes.
27+
RUN_CLANG_TIDY_FIX=YES
28+
29+
# Set to the list of integration tests to run.
30+
RUN_INTEGRATION_TESTS=YES

‎google/cloud/bigtable/ci/run_integration_tests_emulator_cmake.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
set -euo pipefail
1818

1919
source "$(dirname "$0")/../../../../ci/lib/init.sh"
20+
source module /ci/cloudbuild/builds/lib/ctest.sh
2021
source module /ci/etc/integration-tests-config.sh
2122

2223
if [[ $# -lt 1 ]]; then
@@ -32,6 +33,10 @@ readonly BINARY_DIR
3233
shift
3334
ctest_args=("$@")
3435

36+
if ctest::has_no_tests "${BINARY_DIR}" "^bigtable_" "${ctest_args[@]}"; then
37+
exit 0
38+
fi
39+
3540
# Configure run_emulators_utils.sh to find the instance admin emulator.
3641
export CBT_INSTANCE_ADMIN_EMULATOR_CMD="${BINARY_DIR}/google/cloud/bigtable/tests/instance_admin_emulator"
3742
source module /google/cloud/bigtable/tools/run_emulator_utils.sh

‎google/cloud/internal/ci/run_integration_tests_emulator_cmake.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
set -euo pipefail
1818

1919
source "$(dirname "$0")/../../../../ci/lib/init.sh"
20+
source module /ci/cloudbuild/builds/lib/ctest.sh
2021
source module /ci/etc/integration-tests-config.sh
2122
source module /ci/lib/run_gcs_httpbin_emulator_utils.sh
2223

@@ -33,6 +34,10 @@ readonly BINARY_DIR
3334
shift
3435
ctest_args=("$@")
3536

37+
if ctest::has_no_tests "${BINARY_DIR}" "^common_" "${ctest_args[@]}"; then
38+
exit 0
39+
fi
40+
3641
cd "${BINARY_DIR}"
3742
start_emulator
3843

‎google/cloud/pubsub/ci/run_integration_tests_emulator_cmake.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
set -euo pipefail
1818

1919
source "$(dirname "$0")/../../../../ci/lib/init.sh"
20+
source module /ci/cloudbuild/builds/lib/ctest.sh
2021
source module /ci/etc/integration-tests-config.sh
2122
source module /google/cloud/pubsub/ci/lib/pubsub_emulator.sh
2223

@@ -33,6 +34,10 @@ readonly BINARY_DIR
3334
shift
3435
ctest_args=("$@")
3536

37+
if ctest::has_no_tests "${BINARY_DIR}" "^pubsub_" "${ctest_args[@]}"; then
38+
exit 0
39+
fi
40+
3641
cd "${BINARY_DIR}"
3742
# Start the emulator and arranges to kill it
3843
pubsub_emulator::start

‎google/cloud/spanner/ci/run_integration_tests_emulator_cmake.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
set -euo pipefail
1818

1919
source "$(dirname "$0")/../../../../ci/lib/init.sh"
20+
source module /ci/cloudbuild/builds/lib/ctest.sh
2021
source module /ci/etc/integration-tests-config.sh
2122
source module /ci/lib/io.sh
2223
source module /google/cloud/spanner/ci/lib/spanner_emulator.sh
@@ -32,6 +33,10 @@ CMAKE_BINARY_DIR="$(realpath "${1}")"
3233
readonly CMAKE_BINARY_DIR
3334
shift
3435

36+
if ctest::has_no_tests "${CMAKE_BINARY_DIR}" "^spanner_" "${ctest_args[@]}"; then
37+
exit 0
38+
fi
39+
3540
# Any additional arguments for the ctest invocation.
3641
ctest_args=("$@")
3742

‎google/cloud/storage/ci/run_integration_tests_emulator_cmake.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
set -euo pipefail
1818

1919
source "$(dirname "$0")/../../../../ci/lib/init.sh"
20+
source module /ci/cloudbuild/builds/lib/ctest.sh
2021
source module /ci/etc/integration-tests-config.sh
2122
source module /ci/lib/run_gcs_httpbin_emulator_utils.sh
2223

@@ -33,6 +34,10 @@ readonly BINARY_DIR
3334
shift
3435
ctest_args=("$@")
3536

37+
if ctest::has_no_tests "${BINARY_DIR}" "^storage_" "${ctest_args[@]}"; then
38+
exit 0
39+
fi
40+
3641
cd "${BINARY_DIR}"
3742
start_emulator
3843

0 commit comments

Comments
 (0)