Skip to content

Commit 928acaa

Browse files
author
Takashi Matsuo
authored
chore: migrate docs job to Trampoline V2 (googleapis#65)
* chore: migrate docs job to Trampoline V2 This allows us to run the whole job locally. Example: ```bash $ TRAMPOLINE_IMAGE=gcr.io/tmatsuo-test/python-lib-docs \ TRAMPOLINE_DOCKERFILE=.kokoro/docker/docs/Dockerfile \ TRAMPOLINE_BUILD_FILE=.kokoro/publish-docs.sh \ STAGING_BUCKET=tmatsuo-test-bucket \ .kokoro/trampoline_v2.sh ``` You need to copy the service account with the permission on the bucket to `/dev/shm/73713_docuploader_service_account` for the above command to work. * remove unnecessary cd * only upload the image on Kokoro * add TRAMPOLINE_DOCKERFILE in rc file * change the python version for docs job * address review comments * move envvars to the kokoro config
1 parent 27bdf38 commit 928acaa

7 files changed

Lines changed: 697 additions & 18 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ pip-log.txt
4646
# Built documentation
4747
docs/_build
4848
bigquery/docs/generated
49+
docs.metadata
4950

5051
# Virtual environment
5152
env/
@@ -57,4 +58,4 @@ system_tests/local_test_setup
5758

5859
# Make sure a generated file isn't accidentally committed.
5960
pylintrc
60-
pylintrc.test
61+
pylintrc.test

‎.kokoro/docker/docs/Dockerfile‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Copyright 2020 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from ubuntu:20.04
16+
17+
ENV DEBIAN_FRONTEND noninteractive
18+
19+
# Ensure local Python is preferred over distribution Python.
20+
ENV PATH /usr/local/bin:$PATH
21+
22+
# Install dependencies.
23+
RUN apt-get update \
24+
&& apt-get install -y --no-install-recommends \
25+
apt-transport-https \
26+
build-essential \
27+
ca-certificates \
28+
curl \
29+
dirmngr \
30+
git \
31+
gpg-agent \
32+
graphviz \
33+
libbz2-dev \
34+
libdb5.3-dev \
35+
libexpat1-dev \
36+
libffi-dev \
37+
liblzma-dev \
38+
libreadline-dev \
39+
libsnappy-dev \
40+
libssl-dev \
41+
libsqlite3-dev \
42+
portaudio19-dev \
43+
redis-server \
44+
software-properties-common \
45+
ssh \
46+
sudo \
47+
tcl \
48+
tcl-dev \
49+
tk \
50+
tk-dev \
51+
uuid-dev \
52+
wget \
53+
zlib1g-dev \
54+
&& add-apt-repository universe \
55+
&& apt-get update \
56+
&& apt-get -y install jq \
57+
&& apt-get clean autoclean \
58+
&& apt-get autoremove -y \
59+
&& rm -rf /var/lib/apt/lists/* \
60+
&& rm -f /var/cache/apt/archives/*.deb
61+
62+
63+
COPY fetch_gpg_keys.sh /tmp
64+
# Install the desired versions of Python.
65+
RUN set -ex \
66+
&& export GNUPGHOME="$(mktemp -d)" \
67+
&& echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \
68+
&& /tmp/fetch_gpg_keys.sh \
69+
&& for PYTHON_VERSION in 3.7.8 3.8.5; do \
70+
wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
71+
&& wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
72+
&& gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \
73+
&& rm -r python-${PYTHON_VERSION}.tar.xz.asc \
74+
&& mkdir -p /usr/src/python-${PYTHON_VERSION} \
75+
&& tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \
76+
&& rm python-${PYTHON_VERSION}.tar.xz \
77+
&& cd /usr/src/python-${PYTHON_VERSION} \
78+
&& ./configure \
79+
--enable-shared \
80+
# This works only on Python 2.7 and throws a warning on every other
81+
# version, but seems otherwise harmless.
82+
--enable-unicode=ucs4 \
83+
--with-system-ffi \
84+
--without-ensurepip \
85+
&& make -j$(nproc) \
86+
&& make install \
87+
&& ldconfig \
88+
; done \
89+
&& rm -rf "${GNUPGHOME}" \
90+
&& rm -rf /usr/src/python* \
91+
&& rm -rf ~/.cache/
92+
93+
RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
94+
&& python3.7 /tmp/get-pip.py \
95+
&& python3.8 /tmp/get-pip.py \
96+
&& rm /tmp/get-pip.py
97+
98+
CMD ["python3.7"]

‎.kokoro/docker/docs/fetch_gpg_keys.sh‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
# Copyright 2020 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# A script to fetch gpg keys with retry.
17+
18+
function retry {
19+
if [[ "${#}" -le 1 ]]; then
20+
echo "Usage: ${0} retry_count commands.."
21+
exit 1
22+
fi
23+
local retries=${1}
24+
local command="${@:2}"
25+
until [[ "${retries}" -le 0 ]]; do
26+
$command && return 0
27+
if [[ $? -ne 0 ]]; then
28+
echo "command failed, retrying"
29+
((retries--))
30+
fi
31+
done
32+
return 1
33+
}
34+
35+
# 3.6.9, 3.7.5 (Ned Deily)
36+
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
37+
0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D
38+
39+
# 3.8.0 (Łukasz Langa)
40+
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
41+
E3FF2839C048B25C084DEBE9B26995E310250568

‎.kokoro/docs/common.cfg‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,14 @@ action {
1111
gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
1212

1313
# Use the trampoline script to run in docker.
14-
build_file: "python-texttospeech/.kokoro/trampoline.sh"
14+
build_file: "python-texttospeech/.kokoro/trampoline_v2.sh"
1515

1616
# Configure the docker image for kokoro-trampoline.
1717
env_vars: {
1818
key: "TRAMPOLINE_IMAGE"
19-
value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
19+
value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs"
2020
}
21+
2122
env_vars: {
2223
key: "TRAMPOLINE_BUILD_FILE"
2324
value: "github/python-texttospeech/.kokoro/publish-docs.sh"
@@ -28,6 +29,18 @@ env_vars: {
2829
value: "docs-staging"
2930
}
3031

32+
# It will upload the docker image after successful builds.
33+
env_vars: {
34+
key: "TRAMPOLINE_IMAGE_UPLOAD"
35+
value: "true"
36+
}
37+
38+
# It will always build the docker image.
39+
env_vars: {
40+
key: "TRAMPOLINE_DOCKERFILE"
41+
value: ".kokoro/docker/docs/Dockerfile"
42+
}
43+
3144
# Fetch the token needed for reporting release status to GitHub
3245
before_action {
3346
fetch_keystore {

‎.kokoro/publish-docs.sh‎

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,26 +18,16 @@ set -eo pipefail
1818
# Disable buffering, so that the logs stream through.
1919
export PYTHONUNBUFFERED=1
2020

21-
cd github/python-texttospeech
22-
23-
# Remove old nox
24-
python3.6 -m pip uninstall --yes --quiet nox-automation
21+
export PATH="${HOME}/.local/bin:${PATH}"
2522

2623
# Install nox
27-
python3.6 -m pip install --upgrade --quiet nox
28-
python3.6 -m nox --version
24+
python3 -m pip install --user --upgrade --quiet nox
25+
python3 -m nox --version
2926

3027
# build docs
3128
nox -s docs
3229

33-
python3 -m pip install gcp-docuploader
34-
35-
# install a json parser
36-
sudo apt-get update
37-
sudo apt-get -y install software-properties-common
38-
sudo add-apt-repository universe
39-
sudo apt-get update
40-
sudo apt-get -y install jq
30+
python3 -m pip install --user gcp-docuploader
4131

4232
# create metadata
4333
python3 -m docuploader create-metadata \
@@ -52,4 +42,4 @@ python3 -m docuploader create-metadata \
5242
cat docs.metadata
5343

5444
# upload docs
55-
python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket docs-staging
45+
python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}"

0 commit comments

Comments
 (0)