Skip to content

Commit 93fa95e

Browse files
nvpstrnvpstr
committed
Removing obsolete code (BERT)
1 parent 0e66c6d commit 93fa95e

8 files changed

Lines changed: 26 additions & 68 deletions

File tree

‎TensorFlow/LanguageModeling/BERT/optimization.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import tensorflow as tf
2323

2424

25-
def create_optimizer(loss, init_lr, num_train_steps, num_warmup_steps, use_tpu, hvd=None, use_fp16=False, fastmath=False, amp=False, amp_fastmath=False):
25+
def create_optimizer(loss, init_lr, num_train_steps, num_warmup_steps, use_tpu, hvd=None, use_fp16=False, amp=False):
2626
"""Creates an optimizer training op."""
2727
global_step = tf.train.get_or_create_global_step()
2828

@@ -75,15 +75,15 @@ def create_optimizer(loss, init_lr, num_train_steps, num_warmup_steps, use_tpu,
7575
if hvd is not None:
7676
from horovod.tensorflow.compression import Compression
7777
optimizer = hvd.DistributedOptimizer(optimizer, sparse_as_dense=True, compression=Compression.none)
78-
if use_fp16 or fastmath or amp or amp_fastmath:
78+
if use_fp16 or amp:
7979
loss_scale_manager = tf.contrib.mixed_precision.ExponentialUpdateLossScaleManager(init_loss_scale=2**32, incr_every_n_steps=1000, decr_every_n_nan_or_inf=2, decr_ratio=0.5)
8080
optimizer = tf.contrib.mixed_precision.LossScaleOptimizer(optimizer, loss_scale_manager)
8181

8282
tvars = tf.trainable_variables()
8383
grads_and_vars = optimizer.compute_gradients(loss, tvars)
8484
grads_and_vars = [(g,v) for g,v in grads_and_vars if g is not None]
8585
grads, tvars = list(zip(*grads_and_vars))
86-
all_are_finite = tf.reduce_all([tf.reduce_all(tf.is_finite(g)) for g in grads]) if use_fp16 or fastmath or amp or amp_fastmath else tf.constant(True, dtype=tf.bool)
86+
all_are_finite = tf.reduce_all([tf.reduce_all(tf.is_finite(g)) for g in grads]) if use_fp16 or amp else tf.constant(True, dtype=tf.bool)
8787

8888
# This is how the model was pre-trained.
8989
# ensure global norm is a finite number

‎TensorFlow/LanguageModeling/BERT/run_pretraining.py‎

Lines changed: 7 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -114,28 +114,24 @@
114114

115115
flags.DEFINE_bool("use_xla", False, "Whether to enable XLA JIT compilation.")
116116

117-
flags.DEFINE_bool("fastmath", False, "Whether to enable loss scaler for fasthmath ops.")
118-
119117
flags.DEFINE_bool("amp", False, "Whether to enable AMP ops.")
120118

121-
flags.DEFINE_bool("amp_fastmath", False, "Whether to enable AMP fasthmath ops.")
122-
123119
# report samples/sec, total loss and learning rate during training
124120
class _LogSessionRunHook(tf.train.SessionRunHook):
125121
def __init__(self, global_batch_size, display_every=10, hvd_rank=-1):
126122
self.global_batch_size = global_batch_size
127123
self.display_every = display_every
128124
self.hvd_rank = hvd_rank
129125
def after_create_session(self, session, coord):
130-
if FLAGS.use_fp16 or FLAGS.fastmath or FLAGS.amp or FLAGS.amp_fastmath:
126+
if FLAGS.use_fp16 or FLAGS.amp:
131127
print(' Step samples/sec MLM Loss NSP Loss Loss Learning-rate Loss-scaler')
132128
else:
133129
print(' Step samples/sec MLM Loss NSP Loss Loss Learning-rate')
134130
self.elapsed_secs = 0.
135131
self.count = 0
136132
def before_run(self, run_context):
137133
self.t0 = time.time()
138-
if FLAGS.use_fp16 or FLAGS.fastmath or FLAGS.amp or FLAGS.amp_fastmath:
134+
if FLAGS.use_fp16 or FLAGS.amp:
139135
return tf.train.SessionRunArgs(
140136
fetches=['step_update:0', 'total_loss:0',
141137
'learning_rate:0', 'nsp_loss:0',
@@ -148,7 +144,7 @@ def before_run(self, run_context):
148144
def after_run(self, run_context, run_values):
149145
self.elapsed_secs += time.time() - self.t0
150146
self.count += 1
151-
if FLAGS.use_fp16 or FLAGS.fastmath or FLAGS.amp or FLAGS.amp_fastmath:
147+
if FLAGS.use_fp16 or FLAGS.amp:
152148
global_step, total_loss, lr, nsp_loss, mlm_loss, loss_scaler = run_values.results
153149
else:
154150
global_step, total_loss, lr, nsp_loss, mlm_loss = run_values.results
@@ -157,14 +153,14 @@ def after_run(self, run_context, run_values):
157153
dt = self.elapsed_secs / self.count
158154
img_per_sec = self.global_batch_size / dt
159155
if self.hvd_rank >= 0:
160-
if FLAGS.use_fp16 or FLAGS.fastmath or FLAGS.amp or FLAGS.amp_fastmath:
156+
if FLAGS.use_fp16 or FLAGS.amp:
161157
print('%2d :: %6i %11.1f %10.4e %10.4e %6.3f %6.4e %6.4e' %
162158
(self.hvd_rank, print_step, img_per_sec, mlm_loss, nsp_loss, total_loss, lr, loss_scaler))
163159
else:
164160
print('%2d :: %6i %11.1f %10.4e %10.4e %6.3f %6.4e' %
165161
(self.hvd_rank, print_step, img_per_sec, mlm_loss, nsp_loss, total_loss, lr))
166162
else:
167-
if FLAGS.use_fp16 or FLAGS.fastmath or FLAGS.amp or FLAGS.amp_fastmath:
163+
if FLAGS.use_fp16 or FLAGS.amp:
168164
print('%6i %11.1f %10.4e %10.4e %6.3f %6.4e %6.4e' %
169165
(print_step, img_per_sec, mlm_loss, nsp_loss, total_loss, lr, loss_scaler))
170166
else:
@@ -247,7 +243,7 @@ def tpu_scaffold():
247243
if mode == tf.estimator.ModeKeys.TRAIN:
248244
train_op = optimization.create_optimizer(
249245
total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu,
250-
hvd, FLAGS.use_fp16, FLAGS.fastmath, FLAGS.amp, FLAGS.amp_fastmath)
246+
hvd, FLAGS.use_fp16, FLAGS.amp)
251247

252248
output_spec = tf.contrib.tpu.TPUEstimatorSpec(
253249
mode=mode,
@@ -483,24 +479,8 @@ def main(_):
483479
if not FLAGS.do_train and not FLAGS.do_eval:
484480
raise ValueError("At least one of `do_train` or `do_eval` must be True.")
485481

486-
if FLAGS.fastmath and FLAGS.amp:
487-
raise ValueError("Only one of fastmath or amp must be True.")
488-
489-
if FLAGS.fastmath and FLAGS.amp_fastmath:
490-
raise ValueError("Only one of fastmath or amp_fastmath must be True.")
491-
492-
if FLAGS.amp and FLAGS.amp_fastmath:
493-
raise ValueError("Only one of amp or amp_fastmath must be True.")
494-
495-
if FLAGS.fastmath:
496-
os.environ["TF_ENABLE_CUBLAS_TENSOR_OP_MATH_FP32"] = "1"
497-
os.environ["TF_ENABLE_CUDNN_TENSOR_OP_MATH_FP32"] = "1"
498-
os.environ["TF_ENABLE_CUDNN_RNN_TENSOR_OP_MATH_FP32"] = "1"
499-
elif FLAGS.amp:
500-
os.environ["TF_ENABLE_AUTO_MIXED_PRECISION_GRAPH_REWRITE"] = "1"
501-
elif FLAGS.amp_fastmath:
482+
if FLAGS.amp:
502483
os.environ["TF_ENABLE_AUTO_MIXED_PRECISION_GRAPH_REWRITE"] = "1"
503-
os.environ["TF_AUTO_MIXED_PRECISION_GRAPH_REWRITE_LEVEL"] = "TENSOR_CORES_ONLY"
504484

505485
if FLAGS.horovod:
506486
import horovod.tensorflow as hvd

‎TensorFlow/LanguageModeling/BERT/run_squad.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import six
3030
import tensorflow as tf
3131
import horovod.tensorflow as hvd
32+
import time
3233
flags = tf.flags
3334

3435
FLAGS = flags.FLAGS

‎TensorFlow/LanguageModeling/BERT/scripts/finetune_inference_benchmark.sh‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,7 @@ if [ "$task" = "squad" ] ; then
1818
if [ "$precision" = "fp16" ] ; then
1919
echo "fp16 activated!"
2020
export TF_ENABLE_AUTO_MIXED_PRECISION_GRAPH_REWRITE=1
21-
use_fp16="--fast_math"
22-
elif [ "$precision" = "fast_math" ] ; then
23-
echo "fastmath activated!"
24-
export TF_ENABLE_CUBLAS_TENSOR_OP_MATH_FP32=1
25-
export TF_ENABLE_CUDNN_TENSOR_OP_MATH_FP32=1
26-
export TF_ENABLE_CUDNN_RNN_TENSOR_OP_MATH_FP32=1
27-
export TF_USE_DEFAULT_LOSS_SCALING=1
21+
use_fp16="--use_fp16"
2822
fi
2923

3024
if [ "$use_xla" = "true" ] ; then

‎TensorFlow/LanguageModeling/BERT/scripts/finetune_train_benchmark.sh‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,7 @@ if [ "$task" = "squad" ] ; then
1717
LOGFILE="/results/${task}_training_benchmark.log"
1818
if [ "$precision" = "fp16" ] ; then
1919
export TF_ENABLE_AUTO_MIXED_PRECISION_GRAPH_REWRITE=1
20-
use_fp16="--fast_math"
21-
elif [ "$precision" = "fast_math" ] ; then
22-
export TF_ENABLE_CUBLAS_TENSOR_OP_MATH_FP32=1
23-
export TF_ENABLE_CUDNN_TENSOR_OP_MATH_FP32=1
24-
export TF_ENABLE_CUDNN_RNN_TENSOR_OP_MATH_FP32=1
25-
export TF_USE_DEFAULT_LOSS_SCALING=1
20+
use_fp16="--use_fp16"
2621
fi
2722

2823

‎TensorFlow/LanguageModeling/BERT/scripts/run_pretraining.sh‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,6 @@ elif [ "$precision" = "fp16_xla" ] ; then
5050
PREC="--use_fp16 --use_xla"
5151
elif [ "$precision" = "fp32" ] ; then
5252
PREC=""
53-
elif [ "$precision" = "fastmath" ] ; then
54-
PREC="--fast_math"
55-
elif [ "$precision" = "amp_fm" ] ; then
56-
PREC="--amp_fastmath"
57-
elif [ "$precision" = "amp_fm_xla" ] ; then
58-
PREC="--amp_fastmath --use_xla"
5953
elif [ "$precision" = "amp" ] ; then
6054
PREC="--amp"
6155
elif [ "$precision" = "amp_xla" ] ; then

‎TensorFlow/LanguageModeling/BERT/scripts/run_squad_inference.sh‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,7 @@ use_fp16=""
1414
if [ "$precision" = "fp16" ] ; then
1515
echo "fp16 activated!"
1616
export TF_ENABLE_AUTO_MIXED_PRECISION_GRAPH_REWRITE=1
17-
use_fp16="--fast_math"
18-
elif [ "$precision" = "fast_math" ] ; then
19-
echo "fastmath activated!"
20-
export TF_ENABLE_CUBLAS_TENSOR_OP_MATH_FP32=1
21-
export TF_ENABLE_CUDNN_TENSOR_OP_MATH_FP32=1
22-
export TF_ENABLE_CUDNN_RNN_TENSOR_OP_MATH_FP32=1
23-
export TF_USE_DEFAULT_LOSS_SCALING=1
17+
use_fp16="--use_fp16"
2418
fi
2519

2620
if [ "$use_xla" = "true" ] ; then
@@ -30,7 +24,6 @@ else
3024
use_xla_tag=""
3125
fi
3226

33-
3427
python run_squad.py \
3528
--vocab_file=$BERT_DIR/vocab.txt \
3629
--bert_config_file=$BERT_DIR/bert_config.json \

‎TensorFlow/LanguageModeling/BERT/scripts/start_pretraining.sh‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ export EXTRA_PARAMS
6363

6464
set -x
6565
cd $CODEDIR
66-
pwd
66+
pwd
6767

6868
PART=""
6969
if [ "$partition" != "default" ] ; then
@@ -75,15 +75,16 @@ printf -v TAG "%s_%dn_%s_gbs%d" "$job_name" $num_nodes "$precision" $GBS
7575
export DATESTAMP=`date +'%y%m%d%H%M%S'`
7676

7777
sbatch $PART \
78-
-N $num_nodes \
79-
-t $wall_time \
80-
-J $job_name \
81-
--exclusive \
78+
-N $num_nodes \
79+
-t $wall_time \
80+
-J $job_name \
81+
--exclusive \
8282
--mem=0 \
83-
--mail-type=FAIL \
84-
--ntasks-per-node=$DGXNGPU \
85-
--threads-per-core=$DGXHT \
86-
--cores-per-socket=$DGXSOCKETCORES \
87-
--output=$LOGDIR/$TAG.$DATESTAMP.log \
88-
$CODEDIR/scripts/run.sub
83+
--mail-type=FAIL \
84+
--ntasks-per-node=$DGXNGPU \
85+
--threads-per-core=$DGXHT \
86+
--cores-per-socket=$DGXSOCKETCORES \
87+
--output=$LOGDIR/$TAG.$DATESTAMP.log \
88+
$CODEDIR/scripts/run.sub
8989
set +x
90+

0 commit comments

Comments
 (0)