apply dynamic loss scaling (deepspeedai#161)

conglongli · web-flow · commit 068e6561188e · 2022-02-09T13:55:29.000-08:00
diff --git a/Megatron-LM-v1.1.5-ZeRO3/examples/ds_zero_stage_3_config_release.json b/Megatron-LM-v1.1.5-ZeRO3/examples/ds_zero_stage_3_config_release.json
@@ -19,7 +19,7 @@
   "gradient_clipping": 1.0,
   "fp16": {
     "enabled": true,
-    "loss_scale": 1024,
+    "loss_scale": 0,
     "loss_scale_window": 1000,
     "hysteresis": 2,
     "min_loss_scale": 1
diff --git a/Megatron-LM-v1.1.5-ZeRO3/examples/ds_zero_stage_infinity_config.json b/Megatron-LM-v1.1.5-ZeRO3/examples/ds_zero_stage_infinity_config.json
@@ -30,7 +30,7 @@
   "gradient_clipping": 1.0,
   "fp16": {
     "enabled": true,
-    "loss_scale": 1024,
+    "loss_scale": 0,
     "loss_scale_window": 1000,
     "hysteresis": 2,
     "min_loss_scale": 1
diff --git a/Megatron-LM/scripts/ds_zero-offload_10B_config.json b/Megatron-LM/scripts/ds_zero-offload_10B_config.json
@@ -18,7 +18,7 @@
   "gradient_clipping": 1.0,
   "fp16": {
     "enabled": true,
-    "loss_scale": 4096,
+    "loss_scale": 0,
     "loss_scale_window": 1000,
     "hysteresis": 2,
     "min_loss_scale": 1
