Skip to content

Commit 16a443d

Browse files
coryancoryan
authored
ci(bazel): dynamically add dependency cache (#14465)
Use Bazel's downloader config to dynamically rewrite the URLs that need downloading. That means less hacking of our `MODULE.bazel`, and `*.bzl` files. Change the script that maintains the dependency cache to find all the `bzlmod` dependencies via `bazel mod deps` and some Python scripting.
1 parent e7e9f7c commit 16a443d

9 files changed

Lines changed: 81 additions & 24 deletions

File tree

‎.bazelrc‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,11 @@ build:macos --host_cxxopt=-std=c++14
3535
# the project separately.
3636
build --experimental_convenience_symlinks=ignore
3737

38+
# We mirror critical tarballs from several sources in case the canonical source
39+
# is temporarily unavailable, e.g., github.com being down. This option and flag
40+
# automatically rewrites the URLs.
41+
build --experimental_downloader_config=bazel/downloader.cfg
42+
3843
# It is frustrating when long-running builds/tests fail, but it is even more
3944
# frustrating when they fail and don't give any output. So, remove the limit.
4045
build --experimental_ui_max_stdouterr_bytes=-1

‎MODULE.bazel‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ archive_override(
5959
patches = ["//bazel:googleapis.modules.patch"],
6060
strip_prefix = "googleapis-622e10a1e8b2b6908e0ac7448d347a0c1b4130de",
6161
urls = [
62-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_google_googleapis/622e10a1e8b2b6908e0ac7448d347a0c1b4130de.tar.gz",
6362
"https://github.com/googleapis/googleapis/archive/622e10a1e8b2b6908e0ac7448d347a0c1b4130de.tar.gz",
6463
],
6564
)

‎bazel/bzlmod0.bzl‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ def gl_cpp_bzlmod0(name = None):
3434
http_archive,
3535
name = "com_github_curl_curl",
3636
urls = [
37-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_github_curl_curl/curl-7.69.1.tar.gz",
3837
"https://curl.haxx.se/download/curl-7.69.1.tar.gz",
3938
],
4039
sha256 = "01ae0c123dee45b01bbaef94c0bc00ed2aec89cb2ee0fd598e0d302a6b5e0a98",

‎bazel/deps-cache.py‎

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,11 @@
1717
"""Verify and (optionally) populate the GCS cache of Bazel dependencies."""
1818

1919
import argparse
20+
import base64
2021
import filecmp
2122
import glob
2223
import hashlib
24+
import json
2325
import os.path
2426
import subprocess
2527
import sys
@@ -66,9 +68,31 @@
6668
archives = []
6769

6870

69-
def http_archive(**kwargs):
71+
def http_archive(name, urls, integrity, **kwargs):
7072
"""The repo_rule we expect to see in maybe()."""
71-
pass
73+
filtered_urls = [
74+
url
75+
for url in urls
76+
if not any(url.startswith(prefix) for prefix in mirror_prefixes)
77+
]
78+
if len(filtered_urls) != 1:
79+
sys.exit(f"{name}: Ambiguous source URL {filtered_urls}")
80+
if not integrity.startswith("sha256-"):
81+
sys.exit(f"unknown integrity format in integrity = {integrity}")
82+
url = filtered_urls[0]
83+
url = url.removeprefix("https://")
84+
url = url.removeprefix("http://")
85+
path = os.path.join(bucket, url)
86+
sha256 = base64.b64decode(integrity.removeprefix("sha256-")).hex()
87+
archives.append(
88+
{
89+
"name": name,
90+
"source": filtered_urls[0],
91+
"sha256": sha256,
92+
"cache": f"https://storage.googleapis.com/{path}",
93+
"upload": f"gs://{path}",
94+
}
95+
)
7296

7397

7498
def maybe(repo_rule, name, urls, sha256, **kwargs):
@@ -82,7 +106,10 @@ def maybe(repo_rule, name, urls, sha256, **kwargs):
82106
]
83107
if len(filtered_urls) != 1:
84108
sys.exit(f"{name}: Ambiguous source URL {filtered_urls}")
85-
path = os.path.join(bucket, name, os.path.basename(filtered_urls[0]))
109+
url = filtered_urls[0]
110+
url = url.removeprefix("https://")
111+
url = url.removeprefix("http://")
112+
path = os.path.join(bucket, url)
86113
archives.append(
87114
{
88115
"name": name,
@@ -142,6 +169,18 @@ def verify(tmpdir, name, source, cache, upload, **kwargs):
142169
print("")
143170

144171

172+
def bzlmod_modules(bzlmod_deps) -> list[str]:
173+
"""Convert a JSON description of recursive bzlmod deps into a list of dependencies."""
174+
result = []
175+
for k, v in bzlmod_deps.items():
176+
if k == "key" and "@" in v:
177+
result.append(v.split("@")[0])
178+
elif k == "dependencies":
179+
for d in v:
180+
result.extend(bzlmod_modules(d))
181+
return result
182+
183+
145184
def main():
146185
exec_globals = {
147186
"__builtins__": None,
@@ -152,12 +191,27 @@ def main():
152191
"maybe": maybe,
153192
}
154193
exec_locals = {}
194+
try:
195+
p = subprocess.run(
196+
["bazelisk", "mod", "deps", "--output=json", "google_cloud_cpp"],
197+
capture_output=True,
198+
text=True,
199+
)
200+
modules = bzlmod_modules(json.loads(p.stdout))
201+
p = subprocess.run(
202+
["bazelisk", "mod", "show_repo"] + modules, capture_output=True, text=True
203+
)
204+
exec(compile(p.stdout, "<show repo output>", "exec"), exec_globals, exec_locals)
205+
206+
except Exception as e:
207+
sys.exit(f"{e}")
155208
for bzl in args.bzls:
156209
try:
157210
with open(bzl) as f:
158211
exec(compile(f.read(), bzl, "exec"), exec_globals, exec_locals)
159212
except Exception as e:
160213
sys.exit(f"{bzl}: {e}")
214+
161215
for key, func in exec_locals.items():
162216
# At the moment only the "gl_cpp_*0" functions load cacheable objects.
163217
if not callable(func) or not key.startswith("gl_cpp_") or not key.endswith("0"):

‎bazel/development0.bzl‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@ def gl_cpp_development0(name = None):
4545
http_archive,
4646
name = "libpfm",
4747
urls = [
48-
"https://storage.googleapis.com/cloud-cpp-community-archive/libpfm/libpfm-4.11.0.tar.gz",
4948
"https://sourceforge.net/projects/perfmon2/files/libpfm4/libpfm-4.11.0.tar.gz",
5049
],
5150
sha256 = "5da5f8872bde14b3634c9688d980f68bda28b510268723cc12973eedbab9fecc",
@@ -58,7 +57,6 @@ def gl_cpp_development0(name = None):
5857
http_archive,
5958
name = "com_google_benchmark",
6059
urls = [
61-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_google_benchmark/v1.8.4.tar.gz",
6260
"https://github.com/google/benchmark/archive/v1.8.4.tar.gz",
6361
],
6462
sha256 = "3e7059b6b11fb1bbe28e33e02519398ca94c1818874ebed18e504dc6f709be45",
@@ -71,7 +69,6 @@ def gl_cpp_development0(name = None):
7169
http_archive,
7270
name = "com_github_jbeder_yaml_cpp",
7371
urls = [
74-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_github_jbeder_yaml_cpp/0.8.0.tar.gz",
7572
"https://github.com/jbeder/yaml-cpp/archive/0.8.0.tar.gz",
7673
],
7774
sha256 = "fbe74bbdcee21d656715688706da3c8becfd946d92cd44705cc6098bb23b3a16",

‎bazel/development2.bzl‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ def gl_cpp_development2(name = None):
3434
http_archive,
3535
name = "com_github_zeux_pugixml",
3636
urls = [
37-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_github_zeux_pugixml/v1.14.tar.gz",
3837
"https://github.com/zeux/pugixml/archive/v1.14.tar.gz",
3938
],
4039
sha256 = "610f98375424b5614754a6f34a491adbddaaec074e9044577d965160ec103d2e",

‎bazel/downloader.cfg‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# Automatically provide fallbacks for Bazel downloads.
16+
17+
rewrite (github.com)/(.*) storage.googleapis.com/cloud-cpp-community-archive/$1/$2
18+
rewrite (github.com)/(.*) $1/$2

‎bazel/workspace0.bzl‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,6 @@ def gl_cpp_workspace0(name = None):
6161
http_archive,
6262
name = "platforms",
6363
urls = [
64-
"https://storage.googleapis.com/cloud-cpp-community-archive/platforms/platforms-0.0.10.tar.gz",
65-
"https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.10/platforms-0.0.10.tar.gz",
6664
"https://github.com/bazelbuild/platforms/releases/download/0.0.10/platforms-0.0.10.tar.gz",
6765
],
6866
sha256 = "218efe8ee736d26a3572663b374a253c012b716d8af0c07e842e82f238a0a7ee",
@@ -73,7 +71,6 @@ def gl_cpp_workspace0(name = None):
7371
http_archive,
7472
name = "rules_cc",
7573
urls = [
76-
"https://storage.googleapis.com/cloud-cpp-community-archive/rules_cc/rules_cc-0.0.9.tar.gz",
7774
"https://github.com/bazelbuild/rules_cc/releases/download/0.0.9/rules_cc-0.0.9.tar.gz",
7875
],
7976
sha256 = "2037875b9a4456dce4a79d112a8ae885bbc4aad968e6587dca6e64f3a0900cdf",
@@ -85,7 +82,6 @@ def gl_cpp_workspace0(name = None):
8582
http_archive,
8683
name = "build_bazel_rules_apple",
8784
urls = [
88-
"https://storage.googleapis.com/cloud-cpp-community-archive/build_bazel_rules_apple/rules_apple.3.6.0.tar.gz",
8985
"https://github.com/bazelbuild/rules_apple/releases/download/3.6.0/rules_apple.3.6.0.tar.gz",
9086
],
9187
sha256 = "d0f566ad408a6e4d179f0ac4d50a93494a70fcff8fab4c4af0a25b2c241c9b8d",
@@ -96,7 +92,6 @@ def gl_cpp_workspace0(name = None):
9692
http_archive,
9793
name = "com_google_absl",
9894
urls = [
99-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_google_absl/20240116.2.tar.gz",
10095
"https://github.com/abseil/abseil-cpp/archive/20240116.2.tar.gz",
10196
],
10297
sha256 = "733726b8c3a6d39a4120d7e45ea8b41a434cdacde401cba500f14236c49b39dc",
@@ -109,7 +104,6 @@ def gl_cpp_workspace0(name = None):
109104
http_archive,
110105
name = "com_google_googletest",
111106
urls = [
112-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_google_googletest/v1.14.0.tar.gz",
113107
"https://github.com/google/googletest/archive/v1.14.0.tar.gz",
114108
],
115109
sha256 = "8ad598c73ad796e0d8280b082cebd82a630d73e73cd3c70057938a6501bba5d7",
@@ -121,7 +115,6 @@ def gl_cpp_workspace0(name = None):
121115
http_archive,
122116
name = "com_google_googleapis",
123117
urls = [
124-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_google_googleapis/622e10a1e8b2b6908e0ac7448d347a0c1b4130de.tar.gz",
125118
"https://github.com/googleapis/googleapis/archive/622e10a1e8b2b6908e0ac7448d347a0c1b4130de.tar.gz",
126119
],
127120
sha256 = "33c62c03f9479728bdaa1a6553d8b35fa273d010706c75ea85cd8dfe1687586c",
@@ -142,7 +135,6 @@ def gl_cpp_workspace0(name = None):
142135
http_archive,
143136
name = "com_google_protobuf",
144137
urls = [
145-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_google_protobuf/v27.2.tar.gz",
146138
"https://github.com/protocolbuffers/protobuf/archive/v27.2.tar.gz",
147139
],
148140
sha256 = "e4ff2aeb767da6f4f52485c2e72468960ddfe5262483879ef6ad552e52757a77",
@@ -155,7 +147,6 @@ def gl_cpp_workspace0(name = None):
155147
http_archive,
156148
name = "boringssl",
157149
urls = [
158-
"https://storage.googleapis.com/cloud-cpp-community-archive/boringssl/82a53d8c902f940eb1310f76a0b96c40c67f632f.tar.gz",
159150
# Use https://github.com/google/boringssl instead of
160151
# https://boringssl.googlesource.com/boringssl as the
161152
# former has a (more) consistent sha256.
@@ -170,7 +161,6 @@ def gl_cpp_workspace0(name = None):
170161
http_archive,
171162
name = "com_github_grpc_grpc",
172163
urls = [
173-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_github_grpc_grpc/v1.64.2.tar.gz",
174164
"https://github.com/grpc/grpc/archive/v1.64.2.tar.gz",
175165
],
176166
sha256 = "c682fc39baefc6e804d735e6b48141157b7213602cc66dbe0bf375b904d8b5f9",
@@ -194,7 +184,6 @@ def gl_cpp_workspace0(name = None):
194184
http_archive,
195185
name = "com_github_curl_curl",
196186
urls = [
197-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_github_curl_curl/curl-7.69.1.tar.gz",
198187
"https://curl.haxx.se/download/curl-7.69.1.tar.gz",
199188
],
200189
sha256 = "01ae0c123dee45b01bbaef94c0bc00ed2aec89cb2ee0fd598e0d302a6b5e0a98",
@@ -207,7 +196,6 @@ def gl_cpp_workspace0(name = None):
207196
http_archive,
208197
name = "com_github_nlohmann_json",
209198
urls = [
210-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_github_nlohmann_json/v3.11.3.tar.gz",
211199
"https://github.com/nlohmann/json/archive/v3.11.3.tar.gz",
212200
],
213201
sha256 = "0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406",
@@ -219,7 +207,6 @@ def gl_cpp_workspace0(name = None):
219207
http_archive,
220208
name = "com_github_google_crc32c",
221209
urls = [
222-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_github_google_crc32c/1.1.2.tar.gz",
223210
"https://github.com/google/crc32c/archive/1.1.2.tar.gz",
224211
],
225212
sha256 = "ac07840513072b7fcebda6e821068aa04889018f24e10e46181068fb214d7e56",
@@ -235,7 +222,6 @@ def gl_cpp_workspace0(name = None):
235222
http_archive,
236223
name = "io_opentelemetry_cpp",
237224
urls = [
238-
"https://storage.googleapis.com/cloud-cpp-community-archive/io_opentelemetry_cpp/v1.16.0.tar.gz",
239225
"https://github.com/open-telemetry/opentelemetry-cpp/archive/v1.16.0.tar.gz",
240226
],
241227
sha256 = "2209af23f43094651ddf007d44153c23facd41d9891b9b2d8cbc2dc9bb8064dd",

‎external/googleapis/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ include(GoogleapisConfig)
2222

2323
set(GOOGLE_CLOUD_CPP_GOOGLEAPIS_URL
2424
"https://github.com/googleapis/googleapis/archive/${_GOOGLE_CLOUD_CPP_GOOGLEAPIS_COMMIT_SHA}.tar.gz"
25-
"https://storage.googleapis.com/cloud-cpp-community-archive/com_google_googleapis/${_GOOGLE_CLOUD_CPP_GOOGLEAPIS_COMMIT_SHA}.tar.gz"
25+
"https://storage.googleapis.com/cloud-cpp-community-archive/github.com/googleapis/googleapis/archive/${_GOOGLE_CLOUD_CPP_GOOGLEAPIS_COMMIT_SHA}.tar.gz"
2626
)
2727
set(GOOGLE_CLOUD_CPP_GOOGLEAPIS_URL_HASH
2828
"${_GOOGLE_CLOUD_CPP_GOOGLEAPIS_SHA256}")

0 commit comments

Comments
 (0)