Skip to content

Commit 53c8147

Browse files
williambermanwilliamberman
authored
unCLIP image variation (huggingface#1781)
* unCLIP image variation * remove prior comment re: @pcuenca * stable diffusion -> unCLIP re: @pcuenca * add copy froms re: @patil-suraj
1 parent cf5265a commit 53c8147

10 files changed

Lines changed: 962 additions & 2 deletions

File tree

‎docs/source/api/pipelines/unclip.mdx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,6 @@ The unCLIP model in diffusers comes from kakaobrain's karlo and the original cod
2828

2929
## UnCLIPPipeline
3030
[[autodoc]] pipelines.unclip.pipeline_unclip.UnCLIPPipeline
31+
- __call__
32+
[[autodoc]] pipelines.unclip.pipeline_unclip_image_variation.UnCLIPImageVariationPipeline
3133
- __call__

‎scripts/convert_unclip_txt2img_to_image_variation.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import argparse
2+
3+
from diffusers import UnCLIPImageVariationPipeline, UnCLIPPipeline
4+
from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection
5+
6+
7+
if __name__ == "__main__":
8+
parser = argparse.ArgumentParser()
9+
10+
parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.")
11+
12+
parser.add_argument(
13+
"--txt2img_unclip",
14+
default="kakaobrain/karlo-v1-alpha",
15+
type=str,
16+
required=False,
17+
help="The pretrained txt2img unclip.",
18+
)
19+
20+
args = parser.parse_args()
21+
22+
txt2img = UnCLIPPipeline.from_pretrained(args.txt2img_unclip)
23+
24+
feature_extractor = CLIPImageProcessor()
25+
image_encoder = CLIPVisionModelWithProjection.from_pretrained("openai/clip-vit-large-patch14")
26+
27+
img2img = UnCLIPImageVariationPipeline(
28+
decoder=txt2img.decoder,
29+
text_encoder=txt2img.text_encoder,
30+
tokenizer=txt2img.tokenizer,
31+
text_proj=txt2img.text_proj,
32+
feature_extractor=feature_extractor,
33+
image_encoder=image_encoder,
34+
super_res_first=txt2img.super_res_first,
35+
super_res_last=txt2img.super_res_last,
36+
decoder_scheduler=txt2img.decoder_scheduler,
37+
super_res_scheduler=txt2img.super_res_scheduler,
38+
)
39+
40+
img2img.save_pretrained(args.dump_path)

‎src/diffusers/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@
105105
StableDiffusionPipeline,
106106
StableDiffusionPipelineSafe,
107107
StableDiffusionUpscalePipeline,
108+
UnCLIPImageVariationPipeline,
108109
UnCLIPPipeline,
109110
VersatileDiffusionDualGuidedPipeline,
110111
VersatileDiffusionImageVariationPipeline,

‎src/diffusers/pipelines/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
StableDiffusionUpscalePipeline,
5454
)
5555
from .stable_diffusion_safe import StableDiffusionPipelineSafe
56-
from .unclip import UnCLIPPipeline
56+
from .unclip import UnCLIPImageVariationPipeline, UnCLIPPipeline
5757
from .versatile_diffusion import (
5858
VersatileDiffusionDualGuidedPipeline,
5959
VersatileDiffusionImageVariationPipeline,

‎src/diffusers/pipelines/unclip/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,5 @@
1313
from ...utils.dummy_torch_and_transformers_objects import UnCLIPPipeline
1414
else:
1515
from .pipeline_unclip import UnCLIPPipeline
16+
from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline
1617
from .text_proj import UnCLIPTextProjModel

‎src/diffusers/pipelines/unclip/pipeline_unclip.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ class UnCLIPPipeline(DiffusionPipeline):
4545
[CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
4646
prior ([`PriorTransformer`]):
4747
The canonincal unCLIP prior to approximate the image embedding from the text embedding.
48+
text_proj ([`UnCLIPTextProjModel`]):
49+
Utility class to prepare and combine the embeddings before they are passed to the decoder.
4850
decoder ([`UNet2DConditionModel`]):
4951
The decoder to invert the image embedding into an image.
5052
super_res_first ([`UNet2DModel`]):

0 commit comments

Comments
 (0)