moving trtis to 19.08
This commit is contained in:
parent
4e00153ab5
commit
e873db23cd
6
TensorFlow/LanguageModeling/BERT/.gitmodules
vendored
6
TensorFlow/LanguageModeling/BERT/.gitmodules
vendored
|
@ -1,6 +1,4 @@
|
||||||
[submodule "tensorrt-inference-server"]
|
[submodule "tensorrt-inference-server"]
|
||||||
url = https://github.com/NVIDIA/tensorrt-inference-server.git
|
|
||||||
path = tensorrt-inference-server
|
path = tensorrt-inference-server
|
||||||
branch = r19.06
|
url = https://github.com/NVIDIA/tensorrt-inference-server.git
|
||||||
|
branch = r19.08
|
||||||
|
|
||||||
|
|
|
@ -17,15 +17,15 @@ RUN git clone https://github.com/titipata/pubmed_parser
|
||||||
RUN pip3 install /workspace/pubmed_parser
|
RUN pip3 install /workspace/pubmed_parser
|
||||||
|
|
||||||
#Copy the perf_client over
|
#Copy the perf_client over
|
||||||
COPY --from=trt /workspace/build/perf_client /workspace/build/perf_client
|
COPY --from=trt /workspace/install/ /workspace/install/
|
||||||
|
|
||||||
#Copy the python wheel and install with pip
|
|
||||||
COPY --from=trt /workspace/build/dist/dist/tensorrtserver*.whl /tmp/
|
|
||||||
RUN pip install /tmp/tensorrtserver*.whl && rm /tmp/tensorrtserver*.whl
|
|
||||||
|
|
||||||
|
#Install the python wheel with pip
|
||||||
|
RUN pip install /workspace/install/python/tensorrtserver*.whl
|
||||||
|
|
||||||
WORKDIR /workspace/bert
|
WORKDIR /workspace/bert
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
ENV PYTHONPATH /workspace/bert
|
ENV PYTHONPATH /workspace/bert
|
||||||
ENV BERT_PREP_WORKING_DIR /workspace/bert/data
|
ENV BERT_PREP_WORKING_DIR /workspace/bert/data
|
||||||
|
ENV PATH //workspace/install/bin:${PATH}
|
||||||
|
ENV LD_LIBRARY_PATH /workspace/install/lib:${LD_LIBRARY_PATH}
|
|
@ -38,12 +38,12 @@ EXPORT_MODEL_ARGS="${precision} ${use_xla} ${seq_length} ${doc_stride} ${BERT_DI
|
||||||
PERF_CLIENT_ARGS="1000 10 20 localhost"
|
PERF_CLIENT_ARGS="1000 10 20 localhost"
|
||||||
|
|
||||||
# Start Server
|
# Start Server
|
||||||
./trtis/scripts/launch_server.sh $precision
|
bash trtis/scripts/launch_server.sh $precision
|
||||||
|
|
||||||
# Restart Server
|
# Restart Server
|
||||||
restart_server() {
|
restart_server() {
|
||||||
docker kill trt_server_cont
|
docker kill trt_server_cont
|
||||||
./trtis/scripts/launch_server.sh $precision
|
bash trtis/scripts/launch_server.sh $precision
|
||||||
}
|
}
|
||||||
|
|
||||||
############## Dynamic Batching Comparison ##############
|
############## Dynamic Batching Comparison ##############
|
||||||
|
@ -53,32 +53,32 @@ TRTIS_ENGINE_COUNT=1
|
||||||
|
|
||||||
# Dynamic batching 10 ms
|
# Dynamic batching 10 ms
|
||||||
TRTIS_DYN_BATCHING_DELAY=10
|
TRTIS_DYN_BATCHING_DELAY=10
|
||||||
.trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
||||||
restart_server
|
restart_server
|
||||||
sleep 15
|
sleep 15
|
||||||
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
||||||
|
|
||||||
# Dynamic batching 5 ms
|
# Dynamic batching 5 ms
|
||||||
TRTIS_DYN_BATCHING_DELAY=5
|
TRTIS_DYN_BATCHING_DELAY=5
|
||||||
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
||||||
restart_server
|
restart_server
|
||||||
sleep 15
|
sleep 15
|
||||||
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
||||||
|
|
||||||
# Dynamic batching 2 ms
|
# Dynamic batching 2 ms
|
||||||
TRTIS_DYN_BATCHING_DELAY=2
|
TRTIS_DYN_BATCHING_DELAY=2
|
||||||
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
||||||
restart_server
|
restart_server
|
||||||
sleep 15
|
sleep 15
|
||||||
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
||||||
|
|
||||||
|
|
||||||
# Static Batching (i.e. Dynamic batching 0 ms)
|
# Static Batching (i.e. Dynamic batching 0 ms)
|
||||||
TRTIS_DYN_BATCHING_DELAY=0
|
TRTIS_DYN_BATCHING_DELAY=0
|
||||||
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
||||||
restart_server
|
restart_server
|
||||||
sleep 15
|
sleep 15
|
||||||
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
||||||
|
|
||||||
|
|
||||||
# ############## Engine Count Comparison ##############
|
# ############## Engine Count Comparison ##############
|
||||||
|
@ -88,24 +88,24 @@ TRTIS_DYN_BATCHING_DELAY=0
|
||||||
|
|
||||||
# Engine Count = 4
|
# Engine Count = 4
|
||||||
TRTIS_ENGINE_COUNT=4
|
TRTIS_ENGINE_COUNT=4
|
||||||
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
||||||
restart_server
|
restart_server
|
||||||
sleep 15
|
sleep 15
|
||||||
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
||||||
|
|
||||||
# Engine Count = 2
|
# Engine Count = 2
|
||||||
TRTIS_ENGINE_COUNT=2
|
TRTIS_ENGINE_COUNT=2
|
||||||
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
||||||
restart_server
|
restart_server
|
||||||
sleep 15
|
sleep 15
|
||||||
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
||||||
|
|
||||||
# Engine Count = 1
|
# Engine Count = 1
|
||||||
TRTIS_ENGINE_COUNT=1
|
TRTIS_ENGINE_COUNT=1
|
||||||
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
||||||
restart_server
|
restart_server
|
||||||
sleep 15
|
sleep 15
|
||||||
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} ${PERF_CLIENT_ARGS}
|
||||||
|
|
||||||
|
|
||||||
############## Batch Size Comparison ##############
|
############## Batch Size Comparison ##############
|
||||||
|
@ -115,32 +115,32 @@ CLIENT_BATCH_SIZE=1
|
||||||
TRTIS_ENGINE_COUNT=1
|
TRTIS_ENGINE_COUNT=1
|
||||||
TRTIS_DYN_BATCHING_DELAY=0
|
TRTIS_DYN_BATCHING_DELAY=0
|
||||||
|
|
||||||
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
||||||
restart_server
|
restart_server
|
||||||
sleep 15
|
sleep 15
|
||||||
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 64 localhost
|
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 64 localhost
|
||||||
|
|
||||||
# BATCH=2 Generate model and perf
|
# BATCH=2 Generate model and perf
|
||||||
SERVER_BATCH_SIZE=2
|
SERVER_BATCH_SIZE=2
|
||||||
CLIENT_BATCH_SIZE=2
|
CLIENT_BATCH_SIZE=2
|
||||||
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
||||||
restart_server
|
restart_server
|
||||||
sleep 15
|
sleep 15
|
||||||
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 32 localhost
|
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 32 localhost
|
||||||
|
|
||||||
# BATCH=4 Generate model and perf
|
# BATCH=4 Generate model and perf
|
||||||
SERVER_BATCH_SIZE=4
|
SERVER_BATCH_SIZE=4
|
||||||
CLIENT_BATCH_SIZE=4
|
CLIENT_BATCH_SIZE=4
|
||||||
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
||||||
restart_server
|
restart_server
|
||||||
sleep 15
|
sleep 15
|
||||||
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 16 localhost
|
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 16 localhost
|
||||||
|
|
||||||
# BATCH=8 Generate model and perf
|
# BATCH=8 Generate model and perf
|
||||||
SERVER_BATCH_SIZE=8
|
SERVER_BATCH_SIZE=8
|
||||||
CLIENT_BATCH_SIZE=8
|
CLIENT_BATCH_SIZE=8
|
||||||
./trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
bash trtis/scripts/export_model.sh ${init_checkpoint} ${SERVER_BATCH_SIZE} ${EXPORT_MODEL_ARGS} ${TRTIS_DYN_BATCHING_DELAY} ${TRTIS_ENGINE_COUNT} ${TRTIS_MODEL_OVERWRITE}
|
||||||
restart_server
|
restart_server
|
||||||
sleep 15
|
sleep 15
|
||||||
./trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 8 localhost
|
bash trtis/scripts/run_perf_client.sh ${MODEL_NAME} 1 ${precision} ${CLIENT_BATCH_SIZE} 1000 10 8 localhost
|
||||||
|
|
||||||
|
|
|
@ -21,4 +21,4 @@ nvidia-docker run -d --rm \
|
||||||
-e NVIDIA_VISIBLE_DEVICES=$NV_VISIBLE_DEVICES \
|
-e NVIDIA_VISIBLE_DEVICES=$NV_VISIBLE_DEVICES \
|
||||||
-e TF_ENABLE_AUTO_MIXED_PRECISION_GRAPH_REWRITE \
|
-e TF_ENABLE_AUTO_MIXED_PRECISION_GRAPH_REWRITE \
|
||||||
-v $PWD/results/trtis_models:/models \
|
-v $PWD/results/trtis_models:/models \
|
||||||
nvcr.io/nvidia/tensorrtserver:19.06-py3 trtserver --model-store=/models --strict-model-config=false
|
nvcr.io/nvidia/tensorrtserver:19.08-py3 trtserver --model-store=/models --strict-model-config=false
|
|
@ -70,4 +70,4 @@ ARGS="\
|
||||||
|
|
||||||
echo "Using args: $(echo "$ARGS" | sed -e 's/ -/\n-/g')"
|
echo "Using args: $(echo "$ARGS" | sed -e 's/ -/\n-/g')"
|
||||||
|
|
||||||
bash scripts/docker/launch.sh /workspace/build/perf_client $ARGS
|
bash scripts/docker/launch.sh /workspace/install/bin/perf_client $ARGS
|
||||||
|
|
Loading…
Reference in a new issue