Add default value for tokenizer path (deepspeedai#699)

xu-song · awan-10 · mrwyattii · web-flow · commit 0d11c63672e8 · 2023-10-03T13:32:39.000-07:00
Co-authored-by: Ammar Ahmad Awan &lt;ammar.awan@microsoft.com&gt;
Co-authored-by: Michael Wyatt &lt;mrwyattii@gmail.com&gt;
diff --git a/applications/DeepSpeed-Chat/training/utils/utils.py b/applications/DeepSpeed-Chat/training/utils/utils.py
@@ -69,7 +69,8 @@ def load_hf_tokenizer(model_name_or_path, fast_tokenizer=True):
         model_json = os.path.join(model_name_or_path, "config.json")
         if os.path.exists(model_json):
             model_json_file = json.load(open(model_json))
-            model_name = model_json_file["_name_or_path"]
+            model_name = model_json_file.get("_name_or_path",
+                                             model_name_or_path)
             tokenizer = get_tokenizer(model_name,
                                       fast_tokenizer=fast_tokenizer)
     else:
