cleanup: speed up library generation (#12000)

coryan · web-flow · commit e2542b35f63f · 2023-06-30T13:01:32.000-04:00
`generate-libraries.sh` is used to:

- drive a CI build to verify the generated libraries and support files
  are up to date
- update the libraries after the googleapis SHA changes
- update the libraries after (and during) changes to the generator

The first two cases need to run `update_libraries.sh`. The last case is
probably the most common case, and the slowest part was running the
`update_libraries.sh`.

I took the opportunity to make the output more readable when making
large updates.
diff --git a/ci/cloudbuild/builds/generate-libraries.sh b/ci/cloudbuild/builds/generate-libraries.sh
@@ -85,20 +85,25 @@ else
     xargs -r -P "$(nproc)" -n 50 -0 sed -i 's/[[:blank:]]\+$//'
 fi
 
-if [ -z "${GENERATE_GOLDEN_ONLY}" ]; then
+if [[ "${TRIGGER_TYPE:-}" != "manual" ]]; then
   io::log_h2 "Updating protobuf lists/deps"
   external/googleapis/update_libraries.sh
 else
-  io::log_red "Skipping update of protobuf lists/deps."
+  io::log_yellow "Skipping update of protobuf lists/deps."
 fi
 
-# This build should fail if any generated files differ from what was checked
-# in. We only look in the google/ directory so that the build doesn't fail if
-# it's run while editing files in generator/...
+# This build should fail if any generated files differ from what was checked in.
 io::log_h2 "Highlight generated code differences"
-git diff --exit-code generator/integration_tests/golden/ google/ ci/
+# We use `--compact-summary` because in almost all cases the delta is at
+# least hundreds of lines long, and often it is thousands of lines long. The
+# summary is all we need in the script.  The developer can do specific diffs
+# if needed.
+#
+# We only compare a subset of the directories because we expect changes in
+# generator/... (but not in generator/integration_tests/golden) while making
+# generator changes.
+git diff --exit-code --compact-summary generator/integration_tests/golden/ google/ ci/
 
-io::log_h2 "Highlight new files"
 if [[ -n "$(git status --porcelain)" ]]; then
   io::log_red "New unmanaged files created by generator"
   git status
diff --git a/doc/contributor/howto-guide-update-googleapis-sha.md b/doc/contributor/howto-guide-update-googleapis-sha.md
@@ -42,6 +42,7 @@ git commit -m"chore: update googleapis SHA circa $(date +%Y-%m-%d)" bazel cmake
 ## Update the generated libraries
 
 ```shell
+external/googleapis/update_libraries.sh
 ci/cloudbuild/build.sh -t generate-libraries-pr
 ```
 
