Skip to content

Commit 5447190

Browse files
author
Takashi Matsuo
authored
testing: reliablly fetch gpg keys (GoogleCloudPlatform#4171)
* testing: reliablly fetch gpg keys fixes GoogleCloudPlatform#4162 * fix for trampoline_v2 2.0.0
1 parent e90996a commit 5447190

File tree

4 files changed

+84
-22
lines changed

4 files changed

+84
-22
lines changed

‎.kokoro/docker/Dockerfile‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -94,19 +94,12 @@ RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \
9494
&& rm -rf /var/lib/apt/lists/* \
9595
&& rm -f /var/cache/apt/archives/*.deb
9696

97+
COPY fetch_gpg_keys.sh /tmp
9798
# Install the desired versions of Python.
9899
RUN set -ex \
99100
&& export GNUPGHOME="$(mktemp -d)" \
100101
&& echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \
101-
&& gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
102-
# 2.7.17 (Benjamin Peterson)
103-
C01E1CAD5EA2C4F0B8E3571504C367C218ADD4FF \
104-
# 3.4.10, 3.5.9 (Larry Hastings)
105-
97FC712E4C024BBEA48A61ED3A5CA953F73C700D \
106-
# 3.6.9, 3.7.5 (Ned Deily)
107-
0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D \
108-
# 3.8.0 (Łukasz Langa)
109-
E3FF2839C048B25C084DEBE9B26995E310250568 \
102+
&& /tmp/fetch_gpg_keys.sh \
110103
&& for PYTHON_VERSION in 2.7.18 3.6.10 3.7.7 3.8.3; do \
111104
wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
112105
&& wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \

‎.kokoro/docker/fetch_gpg_keys.sh‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/bin/bash
2+
# Copyright 2020 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# A script to fetch gpg keys with retry.
17+
18+
function retry {
19+
if [[ "${#}" -le 1 ]]; then
20+
echo "Usage: ${0} retry_count commands.."
21+
exit 1
22+
fi
23+
local retries=${1}
24+
local command="${@:2}"
25+
until [[ "${retries}" -le 0 ]]; do
26+
$command && return 0
27+
if [[ $? -ne 0 ]]; then
28+
echo "command failed, retrying"
29+
((retries--))
30+
fi
31+
done
32+
return 1
33+
}
34+
35+
# 2.7.17 (Benjamin Peterson)
36+
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
37+
C01E1CAD5EA2C4F0B8E3571504C367C218ADD4FF
38+
39+
# 3.4.10, 3.5.9 (Larry Hastings)
40+
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
41+
97FC712E4C024BBEA48A61ED3A5CA953F73C700D
42+
43+
# 3.6.9, 3.7.5 (Ned Deily)
44+
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
45+
0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D
46+
47+
# 3.8.0 (Łukasz Langa)
48+
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
49+
E3FF2839C048B25C084DEBE9B26995E310250568

‎.kokoro/trampoline_v2.sh‎

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -120,15 +120,16 @@ PROGRAM_PATH="$(realpath "$0")"
120120
PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")"
121121
PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")"
122122

123-
RUNNING_IN_CI="false"
124-
TRAMPOLINE_V2="true"
123+
RUNNING_IN_CI="${RUNNING_IN_CI:-false}"
124+
TRAMPOLINE_VERSION="2.0.0"
125125

126126
# The workspace in the container, defaults to /workspace.
127127
TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}"
128128

129-
# If it's running on Kokoro, RUNNING_IN_CI will be true and
130-
# TRAMPOLINE_CI is set to 'kokoro'. Both envvars will be passing down
131-
# to the container for telling which CI system we're in.
129+
# Detect which CI systems we're in. If we're in any of the CI systems
130+
# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be
131+
# the name of the CI system. Both envvars will be passing down to the
132+
# container for telling which CI system we're in.
132133
if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then
133134
# descriptive env var for indicating it's on CI.
134135
RUNNING_IN_CI="true"
@@ -137,6 +138,9 @@ if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then
137138
log_yellow "Configuring Container Registry access"
138139
gcloud auth list
139140
gcloud auth configure-docker --quiet
141+
elif [[ "${TRAVIS:-}" == "true" ]]; then
142+
RUNNING_IN_CI="true"
143+
TRAMPOLINE_CI="travis"
140144
fi
141145

142146
# Configure the service account for pulling the docker image.
@@ -171,8 +175,8 @@ pass_down_envvars=(
171175
"RUNNING_IN_CI"
172176
# Indicates which CI system we're in.
173177
"TRAMPOLINE_CI"
174-
# Indicates we're running trampoline_v2.
175-
"TRAMPOLINE_V2"
178+
# Indicates the version of the script.
179+
"TRAMPOLINE_VERSION"
176180
# KOKORO dynamic variables.
177181
"KOKORO_BUILD_NUMBER"
178182
"KOKORO_BUILD_ID"
@@ -249,12 +253,28 @@ if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then
249253
if [[ "${TRAMPOLINE_SHOW_COMMAND:-false}" == "true" ]]; then
250254
echo "docker build" "${docker_build_flags[@]}" "${context_dir}"
251255
fi
252-
if docker build "${docker_build_flags[@]}" "${context_dir}"; then
253-
log_green "Finished building the docker image."
254-
update_cache="true"
256+
257+
# ON CI systems, we want to suppress docker build logs, only
258+
# output the logs when it fails.
259+
if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then
260+
if docker build "${docker_build_flags[@]}" "${context_dir}" \
261+
> "${tmpdir}/docker_build.log" 2>&1; then
262+
log_green "Finished building the docker image."
263+
update_cache="true"
264+
else
265+
log_red "Failed to build the Docker image, aborting."
266+
log_yellow "Dumping the build logs:"
267+
cat "${tmpdir}/docker_build.log"
268+
exit 1
269+
fi
255270
else
256-
log_red "Failed to build the Docker image. Aborting."
257-
exit 1
271+
if docker build "${docker_build_flags[@]}" "${context_dir}"; then
272+
log_green "Finished building the docker image."
273+
update_cache="true"
274+
else
275+
log_red "Failed to build the Docker image, aborting."
276+
exit 1
277+
fi
258278
fi
259279
else
260280
if [[ "${has_cache}" != "true" ]]; then

‎vision/automl/edge_container_predict/automl_vision_edge_container_predict_test.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ def edge_container_predict_server_port():
5959
['docker', 'pull', CPU_DOCKER_GCS_PATH],
6060
env={'DOCKER_API_VERSION': '1.38'})
6161

62-
if os.environ.get('TRAMPOLINE_V2') == 'true':
62+
if os.environ.get('TRAMPOLINE_VERSION'):
6363
# Use /tmp
6464
model_path = tempfile.TemporaryDirectory()
6565
else:

0 commit comments

Comments
 (0)