# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved.

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

FROM nvcr.io/nvidia/tensorrtserver:19.10-py3-clientsdk AS trt

FROM continuumio/miniconda3

RUN apt-get update && apt-get install -y pbzip2 pv bzip2 cabextract mc iputils-ping wget

WORKDIR /workspace/speech_ai_demo__TTS/

# Copy the perf_client over

COPY --from=trt /workspace/install/ /workspace/install/

ENV LD_LIBRARY_PATH /workspace/install/lib:${LD_LIBRARY_PATH}

# set up env variables

ENV PATH="$PATH:/opt/conda/bin"

RUN cd /workspace/speech_ai_demo__TTS/

# jupyter lab extensions

RUN conda install -c conda-forge jupyterlab=1.0 ipywidgets=7.5 nodejs python-sounddevice librosa unidecode inflect

RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager

RUN pip install /workspace/install/python/tensorrtserver*.whl

# Copy the python wheel and install with pip

COPY --from=trt /workspace/install/python/tensorrtserver*.whl /tmp/

RUN pip install /tmp/tensorrtserver*.whl && rm /tmp/tensorrtserver*.whl

RUN cd /workspace/speech_ai_demo__TTS/

COPY ./notebooks/trtis/ .

RUN mkdir /workspace/speech_ai_demo__TTS/tacotron2/

COPY ./tacotron2/text /workspace/speech_ai_demo__TTS/tacotron2/text

RUN chmod a+x /workspace/speech_ai_demo__TTS/run_this.sh

* [Training accuracy: NVIDIA DGX-1 (8x V100 16G)](#training-accuracy-nvidia-dgx-1-8x-v100-16g)

* [Training performance: NVIDIA DGX-1 (8x V100 16G)](#training-performance-nvidia-dgx-1-8x-v100-16g)

* [Inference performance: NVIDIA V100 16G](#inference-performance-nvidia-v100-16g)

* [Inference performance: NVIDIA T4](#inference-performance-nvidia-t4)

Here, `Tacotron2_checkpoint` and `WaveGlow_checkpoint` are pre-trained

|Batch size|Input length|Precision|Avg latency (s)|Latency std (s)|Latency confidence interval 90% (s)|Latency confidence interval 95% (s)|Latency confidence interval 99% (s)|Throughput (samples/sec)|Speed-up with mixed precision|Avg mels generated (81 mels=1 sec of speech)|Avg audio length (s)|Avg RTF|

|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|

|1| 128| FP16| 1.27| 0.06| 1.34| 1.38| 1.41| 121,190| 1.37| 603| 7.00| 5.51|

|4| 128| FP16| 2.32| 0.09| 2.42| 2.45| 2.59| 277,711| 2.03| 628| 7.23| 3.12|

|1| 128| FP32| 1.70| 0.05| 1.77| 1.79| 1.84| 88,650| 1.00| 590| 6.85| 4.03|

|4| 128| FP32| 4.56| 0.12| 4.72| 4.77| 4.87| 136,518| 1.00| 608| 7.06| 1.55|

|Batch size|Input length|Precision|Avg latency (s)|Latency std (s)|Latency confidence interval 90% (s)|Latency confidence interval 95% (s)|Latency confidence interval 99% (s)|Throughput (samples/sec)|Speed-up with mixed precision|Avg mels generated (81 mels=1 sec of speech)|Avg audio length (s)|Avg RTF|

|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|

|1| 128| FP16| 3.13| 0.13| 3.28| 3.36| 3.46| 49,276| 1.26| 602| 6.99| 2.24|

|4| 128| FP16| 11.98| 0.42| 12.44| 12.70| 13.29| 53,676| 1.23| 628| 7.29| 0.61|

|1| 128| FP32| 3.88| 0.12| 4.04| 4.09| 4.19| 38,964| 1.00| 591| 6.86| 1.77|

|4| 128| FP32| 14.34| 0.42| 14.89| 15.08| 15.55| 43,489| 1.00| 609| 7.07| 0.49|

the PyTorch-19.09-py3 NGC container. Please note that to reproduce the results,

October 2019

* Tacotron 2 inference with torch.jit.script

November 2019

* Implemented training resume from checkpoint

* Added notebook for running Tacotron 2 and WaveGlow in TRTIS.

ids = torch.arange(0, max_len, device=lengths.device, dtype=lengths.dtype)

mask = torch.le(mask, 0)

import torch

import argparse

from inference import checkpoint_from_distributed, unwrap_distributed, load_and_setup_model

from dllogger.autologging import log_hardware, log_args

def parse_args(parser):

"""

parser.add_argument('--tacotron2', type=str, required=True,

help='full path to the Tacotron2 model checkpoint file')

parser.add_argument('-o', '--output', type=str, default="trtis_repo/tacotron/1/model.pt",

help='filename for the Tacotron 2 TorchScript model')

parser.add_argument('--amp-run', action='store_true',

help='inference with AMP')

return parser

def main():

parser = argparse.ArgumentParser(

description='PyTorch Tacotron 2 Inference')

parser = parse_args(parser)

args = parser.parse_args()

log_args(args)

tacotron2 = load_and_setup_model('Tacotron2', parser, args.tacotron2,

args.amp_run, rename=True)

jitted_tacotron2 = torch.jit.script(tacotron2)

torch.jit.save(jitted_tacotron2, args.output)

if __name__ == '__main__':

main()

import os

import argparse

from dllogger.autologging import log_hardware, log_args

def parse_args(parser):

"""

parser.add_argument("--trtis_model_name",

type=str,

default='tacotron2',

help="exports to appropriate directory for TRTIS")

parser.add_argument("--trtis_model_version",

type=int,

default=1,

help="exports to appropriate directory for TRTIS")

parser.add_argument("--trtis_max_batch_size",

type=int,

default=8,

help="Specifies the 'max_batch_size' in the TRTIS model config.\

See the TRTIS documentation for more info.")

parser.add_argument('--amp-run', action='store_true',

help='inference with AMP')

return parser

def main():

parser = argparse.ArgumentParser(

description='PyTorch Tacotron 2 TRTIS config exporter')

parser = parse_args(parser)

args = parser.parse_args()

log_args(args)

# prepare repository

model_folder = os.path.join('./trtis_repo', args.trtis_model_name)

version_folder = os.path.join(model_folder, str(args.trtis_model_version))

if not os.path.exists(version_folder):

os.makedirs(version_folder)

# build the config for TRTIS

config_filename = os.path.join(model_folder, "config.pbtxt")

config_template = r"""

config_values = {

"model_name": args.trtis_model_name,

"max_batch_size": args.trtis_max_batch_size,

"fp_type": "TYPE_FP16" if args.amp_run else "TYPE_FP32"

}

with open(model_folder + "/config.pbtxt", "w") as file:

final_config_str = config_template.format_map(config_values)

file.write(final_config_str)

if __name__ == '__main__':

main()