[RN50/MX] Skip first iterations for better metrics calculation

hXl3s · nv-kkudrynski · commit 88eb3cff2f03 · 2021-09-08T08:09:23.000-07:00
diff --git a/MxNet/Classification/RN50v1.5/fit.py b/MxNet/Classification/RN50v1.5/fit.py
@@ -234,6 +234,7 @@ def model_score(args, net, val_data, metric, kvstore):
         tic = time.time()
 
     metric = reduce_metrics(args, metric.get_global(), kvstore)
+    durations = durations[min(len(durations) // 10, 100):]
     duration_stats = {
         'ips': total_batch_size / np.mean(durations),
         'latency_avg': np.mean(durations),
@@ -365,9 +366,10 @@ def transform_data(images, labels):
             durations.append(time.time() - tic)
             tic = time.time()
 
+        durations = durations[min(len(durations) // 10, 100):]
         dllogger_epoch_data = {
             'train.loss': loss_metric.get_global()[1],
-            'train.ips': (i + 1) * total_batch_size / (time.time() - etic)
+            'train.ips': total_batch_size / np.mean(durations)
         }
         if args.mode == 'train_val':
             logging.info('Validating epoch {}'.format(epoch))
