Skip to content

Commit 7ba17e2

Browse files
caisqyifeif
caisq
authored and
yifeif
committed
Let build_server.sh take whl file URL as an input argument. (tensorflow#5206)
This make it possible to test OSS GRPC distributed runtime in dist_test/remote_test.sh against a release build. Usage example: 1. Build the server using a release whl file. (Obviously this means that the Linxu CPU PIP release build has to pass first.) $ export DOCKER_VERSION_TAG="0.11.0rc1" $ tensorflow/tools/dist_test/build_server.sh tensorflow/tf_grpc_test_server:${DOCKER_VERSION_TAG} http://ci.tensorflow.org/view/Release/job/release-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-${DOCKER_VERSION_TAG}-cp27-none-linux_x86_64.whl --test 2. Run remote_test.sh: $ export TF_DIST_DOCKER_NO_CACHE=1 $ export TF_DIST_SERVER_DOCKER_IMAGE="tensorflow/tf_grpc_test_server:${DOCKER_VERSION_TAG}" $ export TF_DIST_GCLOUD_PROJECT="my-project" $ export TF_DIST_GCLOUD_COMPUTE_ZONE="my-zone" $ export TF_DIST_CONTAINER_CLUSTER="my-cluster" $ export TF_DIST_GCLOUD_KEY_FILE="/path/to/my/key.json" $ tensorflow/tools/dist_test/remote_test.sh "http://ci.tensorflow.org/view/Release/job/release-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-${DOCKER_VERSION_TAG}-cp27-none-linux_x86_64.whl"
1 parent 7c6059c commit 7ba17e2

3 files changed

Lines changed: 39 additions & 14 deletions

File tree

‎tensorflow/tools/dist_test/build_server.sh‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,14 @@
1616
#
1717
# Builds the test server for distributed (GRPC) TensorFlow
1818
#
19-
# Usage: build_server.sh <docker_image_name> [--test]
19+
# Usage: build_server.sh <docker_image_name> <whl_url> [--test]
20+
#
21+
# Arguments:
22+
# docker_image_name: Name of the docker image to build.
23+
# E.g.: tensorflow/tf_grpc_test_server:0.11.0rc1
24+
#
25+
# whl_url: URL from which the TensorFlow whl file will be downloaded.
26+
# E.g.: https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl
2027
#
2128
# The optional flag --test lets the script to use the Dockerfile for the
2229
# testing GRPC server. Without the flag, the script will build the non-test
@@ -33,28 +40,44 @@ die() {
3340
}
3441

3542
# Check arguments
36-
if [[ $# != 1 ]] && [[ $# != 2 ]]; then
37-
die "Usage: $0 <docker_image_name> [--test]"
43+
if [[ $# -lt 2 ]]; then
44+
die "Usage: $0 <docker_image_name> <whl_url> [--test]"
3845
fi
3946

4047
DOCKER_IMG_NAME=$1
41-
shift
48+
WHL_URL=$2
49+
shift 2
4250

4351
# Current script directory
4452
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4553

46-
DOCKER_FILE="${DIR}/server/Dockerfile"
54+
BUILD_DIR=$(mktemp -d)
55+
echo ""
56+
echo "Using whl file URL: ${WHL_URL}"
57+
echo "Building in temporary directory: ${BUILD_DIR}"
58+
59+
cp -r ${DIR}/* "${BUILD_DIR}"/ || \
60+
die "Failed to copy files to ${BUILD_DIR}"
61+
62+
DOCKER_FILE="${BUILD_DIR}/server/Dockerfile"
4763
if [[ $1 == "--test" ]]; then
48-
DOCKER_FILE="${DIR}/server/Dockerfile.test"
64+
DOCKER_FILE="${BUILD_DIR}/server/Dockerfile.test"
4965
fi
5066
echo "Using Docker file: ${DOCKER_FILE}"
5167

68+
# Download whl file into the build context directory.
69+
wget -P "${BUILD_DIR}" ${WHL_URL} || \
70+
die "Failed to download tensorflow whl file from URL: ${WHL_URL}"
71+
5272
if [[ ! -f "${DOCKER_FILE}" ]]; then
5373
die "ERROR: Unable to find dockerfile: ${DOCKER_FILE}"
5474
fi
5575
echo "Dockerfile: ${DOCKER_FILE}"
5676

5777
# Call docker build
5878
docker build --no-cache -t "${DOCKER_IMG_NAME}" \
59-
-f "${DOCKER_FILE}" \
60-
"${DIR}"
79+
-f "${DOCKER_FILE}" "${BUILD_DIR}" || \
80+
die "Failed to build docker image: ${DOCKER_IMG_NAME}"
81+
82+
# Clean up docker build context directory.
83+
rm -rf "${BUILD_DIR}"

‎tensorflow/tools/dist_test/server/Dockerfile‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,10 @@ RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
3434
python get-pip.py && \
3535
rm get-pip.py
3636

37-
# Install TensorFlow CPU version from nightly build
38-
RUN pip --no-cache-dir install \
39-
https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl
37+
# Install TensorFlow wheel
38+
COPY tensorflow-*.whl /
39+
RUN pip install /tensorflow-*.whl && \
40+
rm -f /tensorflow-*.whl
4041

4142
# Copy files, including the GRPC server binary at
4243
# server/grpc_tensorflow_server.py

‎tensorflow/tools/dist_test/server/Dockerfile.test‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,10 @@ RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
4040
# Install python panda for the census wide&deep test
4141
RUN pip install --upgrade pandas==0.18.1
4242

43-
# Install TensorFlow CPU version.
44-
RUN pip --no-cache-dir install \
45-
https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl
43+
# Install TensorFlow wheel
44+
COPY tensorflow-*.whl /
45+
RUN pip install /tensorflow-*.whl && \
46+
rm -f /tensorflow-*.whl
4647

4748
# Copy files, including the GRPC server binary at
4849
# server/grpc_tensorflow_server.py

0 commit comments

Comments
 (0)