Skip to content

Commit cc12f3e

Browse files
sayakpaulsayakpaul
authored
[Examples] Update with HFApi (huggingface#5393)
* update training examples to use HFAPI. * update training example. * reflect the changes in the korean version too. * Empty-Commit
1 parent 0ea78f9 commit cc12f3e

6 files changed

Lines changed: 79 additions & 142 deletions

File tree

‎docs/source/en/tutorials/basic_training.md‎

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -284,22 +284,11 @@ Now you can wrap all these components together in a training loop with 🤗 Acce
284284

285285
```py
286286
>>> from accelerate import Accelerator
287-
>>> from huggingface_hub import HfFolder, Repository, whoami
287+
>>> from huggingface_hub import create_repo, upload_folder
288288
>>> from tqdm.auto import tqdm
289289
>>> from pathlib import Path
290290
>>> import os
291291

292-
293-
>>> def get_full_repo_name(model_id: str, organization: str = None, token: str = None):
294-
... if token is None:
295-
... token = HfFolder.get_token()
296-
... if organization is None:
297-
... username = whoami(token)["name"]
298-
... return f"{username}/{model_id}"
299-
... else:
300-
... return f"{organization}/{model_id}"
301-
302-
303292
>>> def train_loop(config, model, noise_scheduler, optimizer, train_dataloader, lr_scheduler):
304293
... # Initialize accelerator and tensorboard logging
305294
... accelerator = Accelerator(
@@ -309,11 +298,12 @@ Now you can wrap all these components together in a training loop with 🤗 Acce
309298
... project_dir=os.path.join(config.output_dir, "logs"),
310299
... )
311300
... if accelerator.is_main_process:
312-
... if config.push_to_hub:
313-
... repo_name = get_full_repo_name(Path(config.output_dir).name)
314-
... repo = Repository(config.output_dir, clone_from=repo_name)
315-
... elif config.output_dir is not None:
301+
... if config.output_dir is not None:
316302
... os.makedirs(config.output_dir, exist_ok=True)
303+
... if config.push_to_hub:
304+
... repo_id = create_repo(
305+
... repo_id=config.hub_model_id or Path(config.output_dir).name, exist_ok=True
306+
... )
317307
... accelerator.init_trackers("train_example")
318308

319309
... # Prepare everything
@@ -371,7 +361,12 @@ Now you can wrap all these components together in a training loop with 🤗 Acce
371361

372362
... if (epoch + 1) % config.save_model_epochs == 0 or epoch == config.num_epochs - 1:
373363
... if config.push_to_hub:
374-
... repo.push_to_hub(commit_message=f"Epoch {epoch}", blocking=True)
364+
... upload_folder(
365+
... repo_id=repo_id,
366+
... folder_path=config.output_dir,
367+
... commit_message=f"Epoch {epoch}",
368+
... ignore_patterns=["step_*", "epoch_*"],
369+
... )
375370
... else:
376371
... pipeline.save_pretrained(config.output_dir)
377372
```

‎docs/source/ko/tutorials/basic_training.md‎

Lines changed: 14 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -283,36 +283,27 @@ TensorBoard에 로깅, 그래디언트 누적 및 혼합 정밀도 학습을 쉽
283283

284284
```py
285285
>>> from accelerate import Accelerator
286-
>>> from huggingface_hub import HfFolder, Repository, whoami
286+
>>> from huggingface_hub import create_repo, upload_folder
287287
>>> from tqdm.auto import tqdm
288288
>>> from pathlib import Path
289289
>>> import os
290290

291291

292-
>>> def get_full_repo_name(model_id: str, organization: str = None, token: str = None):
293-
... if token is None:
294-
... token = HfFolder.get_token()
295-
... if organization is None:
296-
... username = whoami(token)["name"]
297-
... return f"{username}/{model_id}"
298-
... else:
299-
... return f"{organization}/{model_id}"
300-
301-
302292
>>> def train_loop(config, model, noise_scheduler, optimizer, train_dataloader, lr_scheduler):
303-
... # accelerator와 tensorboard 로깅 초기화
293+
... # Initialize accelerator and tensorboard logging
304294
... accelerator = Accelerator(
305295
... mixed_precision=config.mixed_precision,
306296
... gradient_accumulation_steps=config.gradient_accumulation_steps,
307297
... log_with="tensorboard",
308-
... logging_dir=os.path.join(config.output_dir, "logs"),
298+
... project_dir=os.path.join(config.output_dir, "logs"),
309299
... )
310300
... if accelerator.is_main_process:
311-
... if config.push_to_hub:
312-
... repo_name = get_full_repo_name(Path(config.output_dir).name)
313-
... repo = Repository(config.output_dir, clone_from=repo_name)
314-
... elif config.output_dir is not None:
301+
... if config.output_dir is not None:
315302
... os.makedirs(config.output_dir, exist_ok=True)
303+
... if config.push_to_hub:
304+
... repo_id = create_repo(
305+
... repo_id=config.hub_model_id or Path(config.output_dir).name, exist_ok=True
306+
... )
316307
... accelerator.init_trackers("train_example")
317308

318309
... # 모든 것이 준비되었습니다.
@@ -369,7 +360,12 @@ TensorBoard에 로깅, 그래디언트 누적 및 혼합 정밀도 학습을 쉽
369360

370361
... if (epoch + 1) % config.save_model_epochs == 0 or epoch == config.num_epochs - 1:
371362
... if config.push_to_hub:
372-
... repo.push_to_hub(commit_message=f"Epoch {epoch}", blocking=True)
363+
... upload_folder(
364+
... repo_id=repo_id,
365+
... folder_path=config.output_dir,
366+
... commit_message=f"Epoch {epoch}",
367+
... ignore_patterns=["step_*", "epoch_*"],
368+
... )
373369
... else:
374370
... pipeline.save_pretrained(config.output_dir)
375371
```

‎examples/dreambooth/train_dreambooth_flax.py‎

Lines changed: 13 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import math
55
import os
66
from pathlib import Path
7-
from typing import Optional
87

98
import jax
109
import jax.numpy as jnp
@@ -16,7 +15,7 @@
1615
from flax import jax_utils
1716
from flax.training import train_state
1817
from flax.training.common_utils import shard
19-
from huggingface_hub import HfFolder, Repository, create_repo, whoami
18+
from huggingface_hub import create_repo, upload_folder
2019
from jax.experimental.compilation_cache import compilation_cache as cc
2120
from PIL import Image
2221
from torch.utils.data import Dataset
@@ -318,16 +317,6 @@ def __getitem__(self, index):
318317
return example
319318

320319

321-
def get_full_repo_name(model_id: str, organization: Optional[str] = None, token: Optional[str] = None):
322-
if token is None:
323-
token = HfFolder.get_token()
324-
if organization is None:
325-
username = whoami(token)["name"]
326-
return f"{username}/{model_id}"
327-
else:
328-
return f"{organization}/{model_id}"
329-
330-
331320
def get_params_to_save(params):
332321
return jax.device_get(jax.tree_util.tree_map(lambda x: x[0], params))
333322

@@ -392,22 +381,14 @@ def main():
392381

393382
# Handle the repository creation
394383
if jax.process_index() == 0:
395-
if args.push_to_hub:
396-
if args.hub_model_id is None:
397-
repo_name = get_full_repo_name(Path(args.output_dir).name, token=args.hub_token)
398-
else:
399-
repo_name = args.hub_model_id
400-
create_repo(repo_name, exist_ok=True, token=args.hub_token)
401-
repo = Repository(args.output_dir, clone_from=repo_name, token=args.hub_token)
402-
403-
with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
404-
if "step_*" not in gitignore:
405-
gitignore.write("step_*\n")
406-
if "epoch_*" not in gitignore:
407-
gitignore.write("epoch_*\n")
408-
elif args.output_dir is not None:
384+
if args.output_dir is not None:
409385
os.makedirs(args.output_dir, exist_ok=True)
410386

387+
if args.push_to_hub:
388+
repo_id = create_repo(
389+
repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token
390+
).repo_id
391+
411392
# Load the tokenizer and add the placeholder token as a additional special token
412393
if args.tokenizer_name:
413394
tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_name)
@@ -668,7 +649,12 @@ def checkpoint(step=None):
668649

669650
if args.push_to_hub:
670651
message = f"checkpoint-{step}" if step is not None else "End of training"
671-
repo.push_to_hub(commit_message=message, blocking=False, auto_lfs_prune=True)
652+
upload_folder(
653+
repo_id=repo_id,
654+
folder_path=args.output_dir,
655+
commit_message=message,
656+
ignore_patterns=["step_*", "epoch_*"],
657+
)
672658

673659
global_step = 0
674660

‎examples/research_projects/intel_opts/textual_inversion_dfq/textual_inversion.py‎

Lines changed: 14 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
import os
55
import random
66
from pathlib import Path
7-
from typing import Iterable, Optional
7+
from typing import Iterable
88

99
import numpy as np
1010
import PIL
@@ -13,7 +13,7 @@
1313
import torch.utils.checkpoint
1414
from accelerate import Accelerator
1515
from accelerate.utils import ProjectConfiguration, set_seed
16-
from huggingface_hub import HfFolder, Repository, whoami
16+
from huggingface_hub import create_repo, upload_folder
1717
from neural_compressor.utils import logger
1818
from packaging import version
1919
from PIL import Image
@@ -413,16 +413,6 @@ def __getitem__(self, i):
413413
return example
414414

415415

416-
def get_full_repo_name(model_id: str, organization: Optional[str] = None, token: Optional[str] = None):
417-
if token is None:
418-
token = HfFolder.get_token()
419-
if organization is None:
420-
username = whoami(token)["name"]
421-
return f"{username}/{model_id}"
422-
else:
423-
return f"{organization}/{model_id}"
424-
425-
426416
def freeze_params(params):
427417
for param in params:
428418
param.requires_grad = False
@@ -461,21 +451,14 @@ def main():
461451

462452
# Handle the repository creation
463453
if accelerator.is_main_process:
464-
if args.push_to_hub:
465-
if args.hub_model_id is None:
466-
repo_name = get_full_repo_name(Path(args.output_dir).name, token=args.hub_token)
467-
else:
468-
repo_name = args.hub_model_id
469-
repo = Repository(args.output_dir, clone_from=repo_name)
470-
471-
with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
472-
if "step_*" not in gitignore:
473-
gitignore.write("step_*\n")
474-
if "epoch_*" not in gitignore:
475-
gitignore.write("epoch_*\n")
476-
elif args.output_dir is not None:
454+
if args.output_dir is not None:
477455
os.makedirs(args.output_dir, exist_ok=True)
478456

457+
if args.push_to_hub:
458+
repo_id = create_repo(
459+
repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token
460+
).repo_id
461+
479462
# Load the tokenizer and add the placeholder token as a additional special token
480463
if args.tokenizer_name:
481464
tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_name)
@@ -982,7 +965,12 @@ def attention_fetcher(x):
982965
)
983966

984967
if args.push_to_hub:
985-
repo.push_to_hub(commit_message="End of training", blocking=False, auto_lfs_prune=True)
968+
upload_folder(
969+
repo_id=repo_id,
970+
folder_path=args.output_dir,
971+
commit_message="End of training",
972+
ignore_patterns=["step_*", "epoch_*"],
973+
)
986974

987975
accelerator.end_training()
988976

‎examples/research_projects/onnxruntime/unconditional_image_generation/train_unconditional.py‎

Lines changed: 13 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import math
55
import os
66
from pathlib import Path
7-
from typing import Optional
87

98
import accelerate
109
import datasets
@@ -14,7 +13,7 @@
1413
from accelerate.logging import get_logger
1514
from accelerate.utils import ProjectConfiguration
1615
from datasets import load_dataset
17-
from huggingface_hub import HfFolder, Repository, create_repo, whoami
16+
from huggingface_hub import create_repo, upload_folder
1817
from onnxruntime.training.optim.fp16_optimizer import FP16_Optimizer as ORT_FP16_Optimizer
1918
from onnxruntime.training.ortmodule import ORTModule
2019
from packaging import version
@@ -277,16 +276,6 @@ def parse_args():
277276
return args
278277

279278

280-
def get_full_repo_name(model_id: str, organization: Optional[str] = None, token: Optional[str] = None):
281-
if token is None:
282-
token = HfFolder.get_token()
283-
if organization is None:
284-
username = whoami(token)["name"]
285-
return f"{username}/{model_id}"
286-
else:
287-
return f"{organization}/{model_id}"
288-
289-
290279
def main(args):
291280
logging_dir = os.path.join(args.output_dir, args.logging_dir)
292281
accelerator_project_config = ProjectConfiguration(
@@ -360,22 +349,14 @@ def load_model_hook(models, input_dir):
360349

361350
# Handle the repository creation
362351
if accelerator.is_main_process:
363-
if args.push_to_hub:
364-
if args.hub_model_id is None:
365-
repo_name = get_full_repo_name(Path(args.output_dir).name, token=args.hub_token)
366-
else:
367-
repo_name = args.hub_model_id
368-
create_repo(repo_name, exist_ok=True, token=args.hub_token)
369-
repo = Repository(args.output_dir, clone_from=repo_name, token=args.hub_token)
370-
371-
with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
372-
if "step_*" not in gitignore:
373-
gitignore.write("step_*\n")
374-
if "epoch_*" not in gitignore:
375-
gitignore.write("epoch_*\n")
376-
elif args.output_dir is not None:
352+
if args.output_dir is not None:
377353
os.makedirs(args.output_dir, exist_ok=True)
378354

355+
if args.push_to_hub:
356+
repo_id = create_repo(
357+
repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token
358+
).repo_id
359+
379360
# Initialize the model
380361
if args.model_config_name_or_path is None:
381362
model = UNet2DModel(
@@ -691,7 +672,12 @@ def transform_images(examples):
691672
ema_model.restore(unet.parameters())
692673

693674
if args.push_to_hub:
694-
repo.push_to_hub(commit_message=f"Epoch {epoch}", blocking=False)
675+
upload_folder(
676+
repo_id=repo_id,
677+
folder_path=args.output_dir,
678+
commit_message=f"Epoch {epoch}",
679+
ignore_patterns=["step_*", "epoch_*"],
680+
)
695681

696682
accelerator.end_training()
697683

0 commit comments

Comments
 (0)