Skip to content

Commit 9b43e37

Browse files
author
Douglas Greiman
authored
Build and test python interpreters in parallel instead of serially. (GoogleCloudPlatform#186)
1 parent dd6fe8f commit 9b43e37

7 files changed

Lines changed: 80 additions & 21 deletions

File tree

‎cloudbuild.yaml‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,35 @@
11
timeout: 10800s
22
steps:
3-
- # Compile Python interpreters from source
3+
- # Compile Python interpreters from source. This step happens first, then
4+
# the next three in parallel.
45
name: gcr.io/cloud-builders/docker:latest
56
args: ['build', '--tag=${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}',
67
'--no-cache', '/workspace/python-interpreter-builder/']
7-
- # Copy interpreters back to workspace
8-
name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}
9-
args: ['cp', '/interpreters.tar.gz', '/workspace/runtime-image/interpreters.tar.gz']
8+
id: interpreter-builder
9+
- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}
10+
args: ['/scripts/build-python-3.4.sh']
11+
id: build-3.4
12+
waitFor: ['interpreter-builder']
13+
- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}
14+
args: ['/scripts/build-python-3.5.sh']
15+
id: build-3.5
16+
waitFor: ['interpreter-builder']
17+
- name: ${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}
18+
args: ['/scripts/build-python-3.6.sh']
19+
id: build-3.6
20+
waitFor: ['interpreter-builder']
1021
- # Build base runtime image
1122
name: gcr.io/cloud-builders/docker:latest
1223
args: ['build', '--tag=${_DOCKER_NAMESPACE}/python:${_TAG}',
1324
'--no-cache', '/workspace/runtime-image/']
25+
id: runtime
26+
waitFor: ['build-3.4', 'build-3.5', 'build-3.6']
1427
- # Build runtime builder image
1528
name: gcr.io/cloud-builders/docker:latest
1629
args: ['build', '--tag=${_BUILDER_DOCKER_NAMESPACE}/python/gen-dockerfile:${_TAG}',
1730
'--no-cache', '/workspace/builder/gen-dockerfile/']
31+
id: gen-dockerfile
32+
waitFor: ['runtime']
1833
images: [
1934
'${_DOCKER_NAMESPACE}/python/interpreter-builder:${_TAG}',
2035
'${_DOCKER_NAMESPACE}/python:${_TAG}',

‎cloudbuild_test.yaml‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,52 @@ steps:
55
args: [
66
'/bin/true',
77
]
8+
id: runtime
9+
810
- # Validate structure of base runtime image
911
name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1
1012
args: [
1113
'-test.v',
1214
'-image', '${_DOCKER_NAMESPACE}/python:${_TAG}',
1315
'/workspace/tests/virtualenv/virtualenv_default.yaml',
16+
]
17+
waitFor: ['runtime']
18+
- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1
19+
args: [
20+
'-test.v',
21+
'-image', '${_DOCKER_NAMESPACE}/python:${_TAG}',
1422
'/workspace/tests/virtualenv/virtualenv_python27.yaml',
23+
]
24+
waitFor: ['runtime']
25+
- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1
26+
args: [
27+
'-test.v',
28+
'-image', '${_DOCKER_NAMESPACE}/python:${_TAG}',
1529
'/workspace/tests/virtualenv/virtualenv_python34.yaml',
30+
]
31+
waitFor: ['runtime']
32+
- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1
33+
args: [
34+
'-test.v',
35+
'-image', '${_DOCKER_NAMESPACE}/python:${_TAG}',
1636
'/workspace/tests/virtualenv/virtualenv_python35.yaml',
37+
]
38+
waitFor: ['runtime']
39+
- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1
40+
args: [
41+
'-test.v',
42+
'-image', '${_DOCKER_NAMESPACE}/python:${_TAG}',
1743
'/workspace/tests/virtualenv/virtualenv_python36.yaml',
44+
]
45+
waitFor: ['runtime']
46+
- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1
47+
args: [
48+
'-test.v',
49+
'-image', '${_DOCKER_NAMESPACE}/python:${_TAG}',
1850
'/workspace/tests/no-virtualenv/no-virtualenv.yaml',
1951
]
52+
waitFor: ['runtime']
53+
2054
# Temporarily disabled because it fails on symbolic links in Ubuntu:
2155
# https://github.com/GoogleCloudPlatform/container-structure-test/issues/77
2256
#- # Check license compliance
@@ -26,35 +60,48 @@ steps:
2660
# '-image', '${_DOCKER_NAMESPACE}/python:${_TAG}',
2761
# '/workspace/tests/license-test/license-test.yaml'
2862
# ]
29-
- # Do third-party library compatibility tests
63+
# waitFor: ['runtime']
64+
65+
- # Do third-party library compatibility tests for Python 2
3066
name: gcr.io/cloud-builders/docker:latest
3167
args: [
3268
'build', '-t', 'python2-libraries-intermediate', '--build-arg',
3369
'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}',
3470
'/workspace/tests/python2-libraries'
3571
]
72+
id: python2-libraries-intermediate
73+
waitFor: ['runtime']
3674
- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1
3775
args: [
3876
'-test.v',
3977
'-image', 'python2-libraries-intermediate',
4078
'/workspace/tests/python2-libraries/python2-libraries.yaml'
4179
]
42-
- name: gcr.io/cloud-builders/docker:latest
80+
waitFor: ['python2-libraries-intermediate']
81+
82+
- # Do third-party library compatibility tests for Python 3
83+
name: gcr.io/cloud-builders/docker:latest
4384
args: [
4485
'build', '-t', 'python3-libraries-intermediate', '--build-arg',
4586
'intermediate_image=${_DOCKER_NAMESPACE}/python:${_TAG}',
4687
'/workspace/tests/python3-libraries'
4788
]
89+
id: python3-libraries-intermediate
90+
waitFor: ['runtime']
4891
- name: gcr.io/gcp-runtimes/container-structure-test:v0.2.1
4992
args: [
5093
'-test.v',
5194
'-image', 'python3-libraries-intermediate',
5295
'/workspace/tests/python3-libraries/python3-libraries.yaml'
5396
]
97+
waitFor: ['python3-libraries-intermediate']
98+
5499
- # Run other compatibility tests
55100
name: gcr.io/cloud-builders/docker:latest
56101
args: [
57102
'build', '--tag=${_DOCKER_NAMESPACE}/python/tests/eventlet:${_TAG}',
58103
'--no-cache', '/workspace/tests/eventlet/'
59104
]
105+
waitFor: ['runtime']
106+
60107
images: []

‎python-interpreter-builder/Dockerfile.in‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -46,17 +46,3 @@ ENV LANG C.UTF-8
4646
# Add build scripts
4747
ADD scripts /scripts
4848
ADD DEBIAN /DEBIAN
49-
50-
# Build the Python interpreters
51-
RUN mkdir -p /opt/packages && \
52-
echo -n "" > /opt/packages/packages.txt
53-
54-
RUN /scripts/build-python-3.4.sh
55-
56-
RUN /scripts/build-python-3.5.sh
57-
58-
RUN /scripts/build-python-3.6.sh
59-
60-
# Tar the interpreters. Tarring is needed because docker cp doesn't handle
61-
# links correctly.
62-
RUN tar czf /interpreters.tar.gz /opt/python?.?

‎python-interpreter-builder/scripts/build-python-3.4.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,3 +136,6 @@ find "$PREFIX"/lib/python3.4/test \
136136
cd /opt
137137
rm /opt/sources/Python-3.4.8.tgz
138138
rm -r /opt/sources/Python-3.4.8
139+
140+
# Archive and copy to persistent external volume
141+
tar czf /workspace/runtime-image/interpreter-3.4.tar.gz /opt/python3.4

‎python-interpreter-builder/scripts/build-python-3.5.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,3 +148,6 @@ find "$PREFIX"/lib/python3.5/test \
148148
cd /opt
149149
rm /opt/sources/Python-3.5.5.tgz
150150
rm -r /opt/sources/Python-3.5.5
151+
152+
# Archive and copy to persistent external volume
153+
tar czf /workspace/runtime-image/interpreter-3.5.tar.gz /opt/python3.5

‎python-interpreter-builder/scripts/build-python-3.6.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,3 +148,6 @@ find "$PREFIX"/lib/python3.6/test \
148148
cd /opt
149149
rm /opt/sources/Python-3.6.4.tgz
150150
rm -r /opt/sources/Python-3.6.4
151+
152+
# Archive and copy to persistent external volume
153+
tar czf /workspace/runtime-image/interpreter-3.6.tar.gz /opt/python3.6

‎runtime-image/Dockerfile.in‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@ ENV LANG C.UTF-8
1717
ENV PYTHONUNBUFFERED 1
1818

1919
# Install the Google-built interpreters
20-
ADD interpreters.tar.gz /
20+
ADD interpreter-3.4.tar.gz /
21+
ADD interpreter-3.5.tar.gz /
22+
ADD interpreter-3.6.tar.gz /
2123

2224
# Add Google-built interpreters to the path
2325
ENV PATH /opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH

0 commit comments

Comments
 (0)