Skip to content

Commit a5fd6fa

Browse files
scotthartscotthart
authored
chore(compute): update_discovery_doc.sh now edits generator_config (#14623)
1 parent 8821125 commit a5fd6fa

3 files changed

Lines changed: 77 additions & 47 deletions

File tree

‎ci/cloudbuild/builds/generate-libraries.sh‎

Lines changed: 23 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,13 @@ source module ci/cloudbuild/builds/lib/git.sh
2222

2323
bazel_output_base="$(bazel info output_base)"
2424

25-
io::log_h2 "Removing previously generated golden files"
26-
git grep -l "Generated by the Codegen C++ plugin" generator/integration_tests/golden | xargs rm
27-
if [ -z "${GENERATE_GOLDEN_ONLY}" ]; then
28-
io::log_h2 "Removing previously generated library files"
29-
git grep -l "Generated by the Codegen C++ plugin" google | xargs rm
25+
if [ -z "${UPDATED_DISCOVERY_DOCUMENT}" ]; then
26+
io::log_h2 "Removing previously generated golden files"
27+
git grep -l "Generated by the Codegen C++ plugin" generator/integration_tests/golden | xargs rm
28+
if [ -z "${GENERATE_GOLDEN_ONLY}" ]; then
29+
io::log_h2 "Removing previously generated library files"
30+
git grep -l "Generated by the Codegen C++ plugin" google | xargs rm
31+
fi
3032
fi
3133

3234
io::log_h2 "Running the generator to update the golden files"
@@ -59,8 +61,7 @@ if [ -z "${GENERATE_GOLDEN_ONLY}" ]; then
5961
rm "${newer_tmp_file}"
6062

6163
if [ -n "${UPDATED_DISCOVERY_DOCUMENT}" ]; then
62-
io::log_h2 "Adding new ${UPDATED_DISCOVERY_DOCUMENT} internal protos"
63-
git add "${PROJECT_ROOT}/protos/google/cloud/${UPDATED_DISCOVERY_DOCUMENT}/v1/internal"
64+
exit 0
6465
fi
6566

6667
# TODO(#5821): The generator should run clang-format on its output files itself
@@ -85,15 +86,6 @@ if [ -z "${GENERATE_GOLDEN_ONLY}" ]; then
8586
--check_comment_substitutions=true \
8687
--config_file="${PROJECT_ROOT}/generator/generator_config.textproto"
8788

88-
if [ -n "${UPDATED_DISCOVERY_DOCUMENT}" ]; then
89-
io::log_h2 "Adding new ${UPDATED_DISCOVERY_DOCUMENT} generated service code"
90-
git add "${PROJECT_ROOT}/google/cloud/${UPDATED_DISCOVERY_DOCUMENT}"
91-
fi
92-
else
93-
io::log_red "Skipping update of generated libraries."
94-
fi
95-
96-
if [ -z "${GENERATE_GOLDEN_ONLY}" ]; then
9789
# TODO(#5821): The generator should run clang-format on its output files itself
9890
# so we don't need this extra step.
9991
io::log_h2 "Formatting generated code"
@@ -123,28 +115,19 @@ else
123115
io::log_yellow "Skipping update of protobuf lists/deps."
124116
fi
125117

126-
if [ -n "${UPDATED_DISCOVERY_DOCUMENT}" ]; then
127-
io::log_h2 "Adding new ${UPDATED_DISCOVERY_DOCUMENT} files post formatting"
128-
git add -u "${PROJECT_ROOT}/google/cloud/${UPDATED_DISCOVERY_DOCUMENT}" \
129-
"${PROJECT_ROOT}/protos/google/cloud/${UPDATED_DISCOVERY_DOCUMENT}" \
130-
"${PROJECT_ROOT}/ci/etc/expected_install_directories"
131-
else
132-
# If the Discovery Document is not being updated, then this build should fail
133-
# if any generated files differ from what was checked in.
134-
io::log_h2 "Highlight generated code differences"
135-
# We use `--compact-summary` because in almost all cases the delta is at
136-
# least hundreds of lines long, and often it is thousands of lines long. The
137-
# summary is all we need in the script. The developer can do specific diffs
138-
# if needed.
139-
#
140-
# We only compare a subset of the directories because we expect changes in
141-
# generator/... (but not in generator/integration_tests/golden) while making
142-
# generator changes.
143-
git diff --exit-code --compact-summary generator/integration_tests/golden/ google/ ci/
144-
145-
if [[ -n "$(git status --porcelain)" ]]; then
146-
io::log_red "New unmanaged files created by generator"
147-
git status
148-
exit 1
149-
fi
118+
io::log_h2 "Highlight generated code differences"
119+
# We use `--compact-summary` because in almost all cases the delta is at
120+
# least hundreds of lines long, and often it is thousands of lines long. The
121+
# summary is all we need in the script. The developer can do specific diffs
122+
# if needed.
123+
#
124+
# We only compare a subset of the directories because we expect changes in
125+
# generator/... (but not in generator/integration_tests/golden) while making
126+
# generator changes.
127+
git diff --exit-code --compact-summary generator/integration_tests/golden/ google/ ci/
128+
129+
if [[ -n "$(git status --porcelain)" ]]; then
130+
io::log_red "New unmanaged files created by generator"
131+
git status
132+
exit 1
150133
fi

‎generator/discovery/update_discovery_doc.sh‎

Lines changed: 52 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,10 @@ function print_service_textproto() {
2323
service_proto_path="${1#protos/}"
2424
product_path="${service_proto_path%/*.proto}"
2525
initial_copyright_year=$(date +"%Y")
26-
cat <<_EOF_
26+
27+
(
28+
sed -n '/# update_discovery_doc.sh additions/q;p' "${PROJECT_ROOT}/${GENERATOR_CONFIG_RELATIVE_PATH}"
29+
cat <<_EOF_
2730
rest_services {
2831
service_proto_path: "${service_proto_path}"
2932
product_path: "${product_path}"
@@ -32,14 +35,17 @@ function print_service_textproto() {
3235
generate_rest_transport: true
3336
generate_grpc_transport: false
3437
}
38+
3539
_EOF_
40+
sed -n '/# update_discovery_doc.sh additions/,$p' "${PROJECT_ROOT}/${GENERATOR_CONFIG_RELATIVE_PATH}"
41+
) | sponge "${PROJECT_ROOT}/${GENERATOR_CONFIG_RELATIVE_PATH}"
3642
}
3743

3844
function add_service_directory() {
3945
compute_proto_path="${1#protos/google/cloud/compute/}"
4046
service_dir="${compute_proto_path%/*.proto}/"
4147
echo " \"${service_dir}\""
42-
sed -i -f - "${PROJECT_ROOT}/google/cloud/compute/service_dirs.cmake" <<EOT
48+
sed -i -f - "${PROJECT_ROOT}/${COMPUTE_SERVICE_DIRS_CMAKE_RELATIVE_PATH}" <<EOT
4349
/^set(service_dirs$/ {
4450
n # skip "cmake-format: sort" line
4551
a\ "${service_dir}"
@@ -59,6 +65,9 @@ fi
5965

6066
readonly COMPUTE_DISCOVERY_DOCUMENT_URL="https://www.googleapis.com/discovery/v1/apis/compute/v1/rest"
6167
readonly COMPUTE_DISCOVERY_JSON_RELATIVE_PATH="generator/discovery/compute_public_google_rest_v1.json"
68+
readonly GENERATOR_CONFIG_RELATIVE_PATH="generator/generator_config.textproto"
69+
readonly COMPUTE_SERVICE_DIRS_CMAKE_RELATIVE_PATH="google/cloud/compute/service_dirs.cmake"
70+
readonly COMPUTE_SERVICE_DIRS_BZL_RELATIVE_PATH="google/cloud/compute/service_dirs.bzl"
6271

6372
io::log_h2 "Fetching discovery document from ${COMPUTE_DISCOVERY_DOCUMENT_URL}"
6473
curl "${COMPUTE_DISCOVERY_DOCUMENT_URL}" >"${PROJECT_ROOT}/${COMPUTE_DISCOVERY_JSON_RELATIVE_PATH}"
@@ -69,24 +78,60 @@ readonly REVISION
6978
io::run git checkout -B update_compute_discovery_circa_"${REVISION}"
7079

7180
io::log_h2 "Adding updated Discovery JSON ${COMPUTE_DISCOVERY_JSON_RELATIVE_PATH}"
72-
git add "${PROJECT_ROOT}/${COMPUTE_DISCOVERY_JSON_RELATIVE_PATH}"
81+
git commit -m"chore(compute): update discovery doc circa ${REVISION}" \
82+
"${PROJECT_ROOT}/${COMPUTE_DISCOVERY_JSON_RELATIVE_PATH}"
7383

7484
io::log_h2 "Running generate-libraries with UPDATED_DISCOVERY_DOCUMENT=compute"
7585
UPDATED_DISCOVERY_DOCUMENT=compute ci/cloudbuild/build.sh -t generate-libraries-pr
7686

87+
io::log_h2 "Adding new compute internal protos"
88+
git add "${PROJECT_ROOT}/protos/google/cloud/compute/v1/internal"
89+
7790
NEW_FILES=$(git ls-files --others --exclude-standard protos/)
7891
if [[ -n "${NEW_FILES}" ]]; then
92+
git add protos/google/cloud/compute
93+
94+
io::log_h2 "Formatting generated protos"
95+
git ls-files -z -- '*.proto' |
96+
xargs -P "$(nproc)" -n 50 -0 clang-format -i
97+
git ls-files -z -- '*.proto' |
98+
xargs -r -P "$(nproc)" -n 50 -0 sed -i 's/[[:blank:]]\+$//'
99+
79100
io::log_red "New resources defined in Discovery Document created new protos:"
80101
echo "${NEW_FILES}"
81102
mapfile -t proto_array < <(echo "${NEW_FILES}")
82-
io::log_red "Add rest_services definitions to the generator_config.textproto and re-run this script."
103+
io::log_red "Adding rest_services definitions to the generator_config.textproto."
83104
for i in "${proto_array[@]}"; do
105+
echo "${i}"
84106
print_service_textproto "${i}"
85107
done
86-
git add protos
87-
io::log_yellow "Adding new directories to google/cloud/compute/service_dirs.cmake"
108+
fi
109+
110+
io::log_h2 "Running generate-libraries with updated generator_config.textproto"
111+
ci/cloudbuild/build.sh -t generate-libraries-pr || true
112+
113+
if [[ -n "${NEW_FILES}" ]]; then
114+
io::log_h2 "Adding new compute generated service code"
115+
git add google/cloud/compute
116+
117+
io::log_h2 "Formatting generated code"
118+
git ls-files -z -- '*.h' '*.cc' |
119+
xargs -P "$(nproc)" -n 50 -0 clang-format -i
120+
git ls-files -z -- '*.h' '*.cc' |
121+
xargs -r -P "$(nproc)" -n 50 -0 sed -i 's/[[:blank:]]\+$//'
122+
123+
io::log_yellow "Adding new directories to ${COMPUTE_SERVICE_DIRS_CMAKE_RELATIVE_PATH}"
88124
for i in "${proto_array[@]}"; do
89125
add_service_directory "${i}"
90126
done
91-
exit 1
127+
128+
io::log_yellow "Adding new directories to ${COMPUTE_SERVICE_DIRS_BZL_RELATIVE_PATH}"
129+
cmake -DGOOGLE_CLOUD_CPP_ENABLE=compute -S . -B ./cmake-build-debug
130+
131+
git commit -m"Update generator_config.textproto and service_dirs files" \
132+
"${PROJECT_ROOT}/${GENERATOR_CONFIG_RELATIVE_PATH}" \
133+
"${PROJECT_ROOT}/${COMPUTE_SERVICE_DIRS_CMAKE_RELATIVE_PATH}" \
134+
"${PROJECT_ROOT}/${COMPUTE_SERVICE_DIRS_BZL_RELATIVE_PATH}"
92135
fi
136+
137+
git commit -m"Update generated code" .

‎generator/generator_config.textproto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1628,6 +1628,8 @@ discovery_products {
16281628
generate_rest_transport: true
16291629
generate_grpc_transport: false
16301630
}
1631+
1632+
# update_discovery_doc.sh additions
16311633
}
16321634
16331635
# Confidential Computing

0 commit comments

Comments
 (0)