Skip to content

Commit e873db2

Browse files
author
Swetha Mandava
committed
moving trtis to 19.08
1 parent 4e00153 commit e873db2

5 files changed

Lines changed: 33 additions & 35 deletions

File tree

‎TensorFlow/LanguageModeling/BERT/.gitmodules‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
[submodule "tensorrt-inference-server"]
2-
url = https://github.com/NVIDIA/tensorrt-inference-server.git
32
path = tensorrt-inference-server
4-
branch = r19.06
5-
6-
3+
url = https://github.com/NVIDIA/tensorrt-inference-server.git
4+
branch = r19.08

‎TensorFlow/LanguageModeling/BERT/Dockerfile‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,15 @@ RUN git clone https://github.com/titipata/pubmed_parser
1717
RUN pip3 install /workspace/pubmed_parser
1818

1919
#Copy the perf_client over
20-
COPY --from=trt /workspace/build/perf_client /workspace/build/perf_client
21-
22-
#Copy the python wheel and install with pip
23-
COPY --from=trt /workspace/build/dist/dist/tensorrtserver*.whl /tmp/
24-
RUN pip install /tmp/tensorrtserver*.whl && rm /tmp/tensorrtserver*.whl
20+
COPY --from=trt /workspace/install/ /workspace/install/
2521

22+
#Install the python wheel with pip
23+
RUN pip install /workspace/install/python/tensorrtserver*.whl
2624

2725
WORKDIR /workspace/bert
2826
COPY . .
2927

3028
ENV PYTHONPATH /workspace/bert
3129
ENV BERT_PREP_WORKING_DIR /workspace/bert/data
30+
ENV PATH //workspace/install/bin:${PATH}
31+
ENV LD_LIBRARY_PATH /workspace/install/lib:${LD_LIBRARY_PATH}

‎TensorFlow/LanguageModeling/BERT/trtis/scripts/generate_figures.sh‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,12 @@ EXPORT_MODEL_ARGS="${precision} ${use_xla} ${seq_length} ${doc_stride} ${BERT_DI
3838
PERF_CLIENT_ARGS="1000 10 20 localhost"
3939

4040
# Start Server
41-
./trtis/scripts/launch_server.sh $precision
41+
bash trtis/scripts/launch_server.sh $precision
4242

4343
# Restart Server
4444
restart_server() {
4545
docker kill trt_server_cont
46-
./trtis/scripts/launch_server.sh $precision
46+
bash trtis/scripts/launch_server.sh $precision
4747
}
4848

4949
############## Dynamic Batching Comparison ##############
@@ -53,32 +53,32 @@ TRTIS_ENGINE_COUNT=1
5353

5454
# Dynamic batching 10 ms
5555
TRTIS_DYN_BATCHING_DELAY=10
56-
.trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
56+
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
5757
restart_server
5858
sleep 15
59-
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
59+
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
6060

6161
# Dynamic batching 5 ms
6262
TRTIS_DYN_BATCHING_DELAY=5
63-
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
63+
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
6464
restart_server
6565
sleep 15
66-
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
66+
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
6767

6868
# Dynamic batching 2 ms
6969
TRTIS_DYN_BATCHING_DELAY=2
70-
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
70+
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
7171
restart_server
7272
sleep 15
73-
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
73+
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
7474

7575

7676
# Static Batching (i.e. Dynamic batching 0 ms)
7777
TRTIS_DYN_BATCHING_DELAY=0
78-
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
78+
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
7979
restart_server
8080
sleep 15
81-
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
81+
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
8282

8383

8484
# ############## Engine Count Comparison ##############
@@ -88,24 +88,24 @@ TRTIS_DYN_BATCHING_DELAY=0
8888

8989
# Engine Count = 4
9090
TRTIS_ENGINE_COUNT=4
91-
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
91+
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
9292
restart_server
9393
sleep 15
94-
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
94+
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
9595

9696
# Engine Count = 2
9797
TRTIS_ENGINE_COUNT=2
98-
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
98+
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
9999
restart_server
100100
sleep 15
101-
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
101+
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
102102

103103
# Engine Count = 1
104104
TRTIS_ENGINE_COUNT=1
105-
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
105+
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
106106
restart_server
107107
sleep 15
108-
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
108+
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
109109

110110

111111
############## Batch Size Comparison ##############
@@ -115,32 +115,32 @@ CLIENT_BATCH_SIZE=1
115115
TRTIS_ENGINE_COUNT=1
116116
TRTIS_DYN_BATCHING_DELAY=0
117117

118-
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
118+
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
119119
restart_server
120120
sleep 15
121-
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 64 localhost
121+
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 64 localhost
122122

123123
# BATCH=2 Generate model and perf
124124
SERVER_BATCH_SIZE=2
125125
CLIENT_BATCH_SIZE=2
126-
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
126+
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
127127
restart_server
128128
sleep 15
129-
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 32 localhost
129+
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 32 localhost
130130

131131
# BATCH=4 Generate model and perf
132132
SERVER_BATCH_SIZE=4
133133
CLIENT_BATCH_SIZE=4
134-
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
134+
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
135135
restart_server
136136
sleep 15
137-
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 16 localhost
137+
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 16 localhost
138138

139139
# BATCH=8 Generate model and perf
140140
SERVER_BATCH_SIZE=8
141141
CLIENT_BATCH_SIZE=8
142-
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
142+
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
143143
restart_server
144144
sleep 15
145-
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 8 localhost
145+
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 8 localhost
146146

‎TensorFlow/LanguageModeling/BERT/trtis/scripts/launch_server.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,4 @@ nvidia-docker run -d --rm \
2121
-e NVIDIA_VISIBLE_DEVICES=$NV_VISIBLE_DEVICES \
2222
-e TF_ENABLE_AUTO_MIXED_PRECISION_GRAPH_REWRITE \
2323
-v $PWD/results/trtis_models:/models \
24-
nvcr.io/nvidia/tensorrtserver:19.06-py3 trtserver --model-store=/models --strict-model-config=false
24+
nvcr.io/nvidia/tensorrtserver:19.08-py3 trtserver --model-store=/models --strict-model-config=false

‎TensorFlow/LanguageModeling/BERT/trtis/scripts/run_perf_client.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,4 +70,4 @@ ARGS="\
7070

7171
echo "Using args: $(echo "$ARGS" | sed -e 's/ -/\n-/g')"
7272

73-
bash scripts/docker/launch.sh /workspace/build/perf_client $ARGS
73+
bash scripts/docker/launch.sh /workspace/install/bin/perf_client $ARGS

0 commit comments

Comments
 (0)