Skip to content

Commit 47bf8e5

Browse files
williambermanwilliamberman
authored
can call encode_prompt with out setting a text encoder instance variable (huggingface#4396)
* can call encode_prompt with out setting a text encoder instance variable * fix
1 parent 18fc40c commit 47bf8e5

26 files changed

Lines changed: 234 additions & 52 deletions

‎src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,14 @@ def _encode_prompt(
334334
)
335335
prompt_embeds = prompt_embeds[0]
336336

337-
prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
337+
if self.text_encoder is not None:
338+
prompt_embeds_dtype = self.text_encoder.dtype
339+
elif self.unet is not None:
340+
prompt_embeds_dtype = self.unet.dtype
341+
else:
342+
prompt_embeds_dtype = prompt_embeds.dtype
343+
344+
prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
338345

339346
bs_embed, seq_len, _ = prompt_embeds.shape
340347
# duplicate text embeddings for each generation per prompt, using mps friendly method
@@ -390,7 +397,7 @@ def _encode_prompt(
390397
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
391398
seq_len = negative_prompt_embeds.shape[1]
392399

393-
negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
400+
negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
394401

395402
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
396403
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)

‎src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -335,7 +335,14 @@ def _encode_prompt(
335335
)
336336
prompt_embeds = prompt_embeds[0]
337337

338-
prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
338+
if self.text_encoder is not None:
339+
prompt_embeds_dtype = self.text_encoder.dtype
340+
elif self.unet is not None:
341+
prompt_embeds_dtype = self.unet.dtype
342+
else:
343+
prompt_embeds_dtype = prompt_embeds.dtype
344+
345+
prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
339346

340347
bs_embed, seq_len, _ = prompt_embeds.shape
341348
# duplicate text embeddings for each generation per prompt, using mps friendly method
@@ -391,7 +398,7 @@ def _encode_prompt(
391398
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
392399
seq_len = negative_prompt_embeds.shape[1]
393400

394-
negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
401+
negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
395402

396403
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
397404
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)

‎src/diffusers/pipelines/controlnet/pipeline_controlnet.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,14 @@ def _encode_prompt(
326326
)
327327
prompt_embeds = prompt_embeds[0]
328328

329-
prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
329+
if self.text_encoder is not None:
330+
prompt_embeds_dtype = self.text_encoder.dtype
331+
elif self.unet is not None:
332+
prompt_embeds_dtype = self.unet.dtype
333+
else:
334+
prompt_embeds_dtype = prompt_embeds.dtype
335+
336+
prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
330337

331338
bs_embed, seq_len, _ = prompt_embeds.shape
332339
# duplicate text embeddings for each generation per prompt, using mps friendly method
@@ -382,7 +389,7 @@ def _encode_prompt(
382389
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
383390
seq_len = negative_prompt_embeds.shape[1]
384391

385-
negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
392+
negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
386393

387394
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
388395
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)

‎src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,14 @@ def _encode_prompt(
352352
)
353353
prompt_embeds = prompt_embeds[0]
354354

355-
prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
355+
if self.text_encoder is not None:
356+
prompt_embeds_dtype = self.text_encoder.dtype
357+
elif self.unet is not None:
358+
prompt_embeds_dtype = self.unet.dtype
359+
else:
360+
prompt_embeds_dtype = prompt_embeds.dtype
361+
362+
prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
356363

357364
bs_embed, seq_len, _ = prompt_embeds.shape
358365
# duplicate text embeddings for each generation per prompt, using mps friendly method
@@ -408,7 +415,7 @@ def _encode_prompt(
408415
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
409416
seq_len = negative_prompt_embeds.shape[1]
410417

411-
negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
418+
negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
412419

413420
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
414421
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)

‎src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -469,7 +469,14 @@ def _encode_prompt(
469469
)
470470
prompt_embeds = prompt_embeds[0]
471471

472-
prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
472+
if self.text_encoder is not None:
473+
prompt_embeds_dtype = self.text_encoder.dtype
474+
elif self.unet is not None:
475+
prompt_embeds_dtype = self.unet.dtype
476+
else:
477+
prompt_embeds_dtype = prompt_embeds.dtype
478+
479+
prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
473480

474481
bs_embed, seq_len, _ = prompt_embeds.shape
475482
# duplicate text embeddings for each generation per prompt, using mps friendly method
@@ -525,7 +532,7 @@ def _encode_prompt(
525532
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
526533
seq_len = negative_prompt_embeds.shape[1]
527534

528-
negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
535+
negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
529536

530537
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
531538
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)

‎src/diffusers/pipelines/stable_diffusion/pipeline_cycle_diffusion.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,14 @@ def _encode_prompt(
346346
)
347347
prompt_embeds = prompt_embeds[0]
348348

349-
prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
349+
if self.text_encoder is not None:
350+
prompt_embeds_dtype = self.text_encoder.dtype
351+
elif self.unet is not None:
352+
prompt_embeds_dtype = self.unet.dtype
353+
else:
354+
prompt_embeds_dtype = prompt_embeds.dtype
355+
356+
prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
350357

351358
bs_embed, seq_len, _ = prompt_embeds.shape
352359
# duplicate text embeddings for each generation per prompt, using mps friendly method
@@ -402,7 +409,7 @@ def _encode_prompt(
402409
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
403410
seq_len = negative_prompt_embeds.shape[1]
404411

405-
negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
412+
negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
406413

407414
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
408415
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)

‎src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -336,7 +336,14 @@ def _encode_prompt(
336336
)
337337
prompt_embeds = prompt_embeds[0]
338338

339-
prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
339+
if self.text_encoder is not None:
340+
prompt_embeds_dtype = self.text_encoder.dtype
341+
elif self.unet is not None:
342+
prompt_embeds_dtype = self.unet.dtype
343+
else:
344+
prompt_embeds_dtype = prompt_embeds.dtype
345+
346+
prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
340347

341348
bs_embed, seq_len, _ = prompt_embeds.shape
342349
# duplicate text embeddings for each generation per prompt, using mps friendly method
@@ -392,7 +399,7 @@ def _encode_prompt(
392399
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
393400
seq_len = negative_prompt_embeds.shape[1]
394401

395-
negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
402+
negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
396403

397404
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
398405
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)

‎src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_attend_and_excite.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -335,7 +335,14 @@ def _encode_prompt(
335335
)
336336
prompt_embeds = prompt_embeds[0]
337337

338-
prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
338+
if self.text_encoder is not None:
339+
prompt_embeds_dtype = self.text_encoder.dtype
340+
elif self.unet is not None:
341+
prompt_embeds_dtype = self.unet.dtype
342+
else:
343+
prompt_embeds_dtype = prompt_embeds.dtype
344+
345+
prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
339346

340347
bs_embed, seq_len, _ = prompt_embeds.shape
341348
# duplicate text embeddings for each generation per prompt, using mps friendly method
@@ -391,7 +398,7 @@ def _encode_prompt(
391398
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
392399
seq_len = negative_prompt_embeds.shape[1]
393400

394-
negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
401+
negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
395402

396403
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
397404
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)

‎src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,14 @@ def _encode_prompt(
220220
)
221221
prompt_embeds = prompt_embeds[0]
222222

223-
prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
223+
if self.text_encoder is not None:
224+
prompt_embeds_dtype = self.text_encoder.dtype
225+
elif self.unet is not None:
226+
prompt_embeds_dtype = self.unet.dtype
227+
else:
228+
prompt_embeds_dtype = prompt_embeds.dtype
229+
230+
prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
224231

225232
bs_embed, seq_len, _ = prompt_embeds.shape
226233
# duplicate text embeddings for each generation per prompt, using mps friendly method
@@ -276,7 +283,7 @@ def _encode_prompt(
276283
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
277284
seq_len = negative_prompt_embeds.shape[1]
278285

279-
negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
286+
negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
280287

281288
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
282289
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)

‎src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_diffedit.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -521,7 +521,14 @@ def _encode_prompt(
521521
)
522522
prompt_embeds = prompt_embeds[0]
523523

524-
prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
524+
if self.text_encoder is not None:
525+
prompt_embeds_dtype = self.text_encoder.dtype
526+
elif self.unet is not None:
527+
prompt_embeds_dtype = self.unet.dtype
528+
else:
529+
prompt_embeds_dtype = prompt_embeds.dtype
530+
531+
prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
525532

526533
bs_embed, seq_len, _ = prompt_embeds.shape
527534
# duplicate text embeddings for each generation per prompt, using mps friendly method
@@ -577,7 +584,7 @@ def _encode_prompt(
577584
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
578585
seq_len = negative_prompt_embeds.shape[1]
579586

580-
negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
587+
negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
581588

582589
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
583590
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)

0 commit comments

Comments
 (0)