Skip to content

[pull] master from Comfy-Org:master #980

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pull merged 3 commits into from
Jul 21, 2026
Merged

[pull] master from Comfy-Org:master #980

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
593786e
FreSca: 5D+ (ex. Anima) fix, model-agnostic iteration (#15007)
TheToxin-git Jul 21, 2026
ac3a7a6
Add native Uni3C Controlnet support for Wan models (CORE-365) (#14946)
drozbay Jul 21, 2026
78b43d2
[Partner Nodes] fix(Gemini-Omni): pass videos as inline data (#15014)
bigcat88 Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions comfy/ldm/wan/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,7 @@ def forward_orig(
List of denoised video tensors with original input shapes [C_out, F, H / 8, W / 8]
"""
# embeddings
x_input = x
x = self.patch_embedding(x.float()).to(x.dtype)
grid_sizes = x.shape[2:]
transformer_options["grid_sizes"] = grid_sizes
Expand All @@ -564,11 +565,13 @@ def forward_orig(
e0 = self.time_projection(e).unflatten(2, (6, self.dim))

full_ref = None
img_offset = 0
if self.ref_conv is not None:
full_ref = kwargs.get("reference_latent", None)
if full_ref is not None:
full_ref = self.ref_conv(full_ref).flatten(2).transpose(1, 2)
x = torch.concat((full_ref, x), dim=1)
img_offset = full_ref.shape[1]

# In-context reference (Bernini)
context_latents = kwargs.get("context_latents", None)
Expand All @@ -589,6 +592,7 @@ def forward_orig(
context_img_len = clip_fea.shape[-2]

patches_replace = transformer_options.get("patches_replace", {})
patches = transformer_options.get("patches", {})
blocks_replace = patches_replace.get("dit", {})
transformer_options["total_blocks"] = len(self.blocks)
transformer_options["block_type"] = "double"
Expand All @@ -604,6 +608,11 @@ def block_wrap(args):
else:
x = block(x, e=e0, freqs=freqs, context=context, context_img_len=context_img_len, transformer_options=transformer_options)

if "double_block" in patches:
for p in patches["double_block"]:
out = p({"img": x, "x": x_input, "vec": e, "block_index": i, "img_offset": img_offset, "transformer_options": transformer_options})
x = out["img"]

# head
x = self.head(x, e)

Expand Down Expand Up @@ -777,6 +786,7 @@ def forward_orig(
**kwargs,
):
# embeddings
x_input = x
x = self.patch_embedding(x.float()).to(x.dtype)
grid_sizes = x.shape[2:]
transformer_options["grid_sizes"] = grid_sizes
Expand Down Expand Up @@ -807,6 +817,7 @@ def forward_orig(
x_orig = x

patches_replace = transformer_options.get("patches_replace", {})
patches = transformer_options.get("patches", {})
blocks_replace = patches_replace.get("dit", {})
transformer_options["total_blocks"] = len(self.blocks)
transformer_options["block_type"] = "double"
Expand All @@ -822,6 +833,11 @@ def block_wrap(args):
else:
x = block(x, e=e0, freqs=freqs, context=context, context_img_len=context_img_len, transformer_options=transformer_options)

if "double_block" in patches:
for p in patches["double_block"]:
out = p({"img": x, "x": x_input, "vec": e, "block_index": i, "img_offset": 0, "transformer_options": transformer_options})
x = out["img"]

ii = self.vace_layers_mapping.get(i, None)
if ii is not None:
for iii in range(len(c)):
Expand Down Expand Up @@ -887,6 +903,7 @@ def forward_orig(
**kwargs,
):
# embeddings
x_input = x
x = self.patch_embedding(x.float()).to(x.dtype)
if self.control_adapter is not None and camera_conditions is not None:
x = x + self.control_adapter(camera_conditions).to(x.dtype)
Expand All @@ -909,6 +926,7 @@ def forward_orig(
context_img_len = clip_fea.shape[-2]

patches_replace = transformer_options.get("patches_replace", {})
patches = transformer_options.get("patches", {})
blocks_replace = patches_replace.get("dit", {})
transformer_options["total_blocks"] = len(self.blocks)
transformer_options["block_type"] = "double"
Expand All @@ -924,6 +942,11 @@ def block_wrap(args):
else:
x = block(x, e=e0, freqs=freqs, context=context, context_img_len=context_img_len, transformer_options=transformer_options)

if "double_block" in patches:
for p in patches["double_block"]:
out = p({"img": x, "x": x_input, "vec": e, "block_index": i, "img_offset": 0, "transformer_options": transformer_options})
x = out["img"]

# head
x = self.head(x, e)

Expand Down Expand Up @@ -1335,6 +1358,7 @@ def forward_orig(

# embeddings
bs, _, time, height, width = x.shape
x_input = x
x = self.patch_embedding(x.float()).to(x.dtype)
if control_video is not None:
x = x + self.cond_encoder(control_video)
Expand Down Expand Up @@ -1379,6 +1403,7 @@ def forward_orig(
context = self.text_embedding(context)

patches_replace = transformer_options.get("patches_replace", {})
patches = transformer_options.get("patches", {})
blocks_replace = patches_replace.get("dit", {})
transformer_options["total_blocks"] = len(self.blocks)
transformer_options["block_type"] = "double"
Expand All @@ -1393,6 +1418,12 @@ def block_wrap(args):
x = out["img"]
else:
x = block(x, e=e0, freqs=freqs, context=context, transformer_options=transformer_options)

if "double_block" in patches:
for p in patches["double_block"]:
out = p({"img": x, "x": x_input, "vec": e, "block_index": i, "img_offset": 0, "transformer_options": transformer_options})
x = out["img"]

if audio_emb is not None:
x = self.audio_injector(x, i, audio_emb, audio_emb_global, seq_len)
# head
Expand Down Expand Up @@ -1599,6 +1630,7 @@ def forward_orig(
bs, _, time, height, width = x.shape

# embeddings
x_input = x
x = self.patch_embedding(x.float()).to(x.dtype)
grid_sizes = x.shape[2:]
x = x.flatten(2).transpose(1, 2)
Expand Down Expand Up @@ -1630,6 +1662,7 @@ def forward_orig(
audio = None

patches_replace = transformer_options.get("patches_replace", {})
patches = transformer_options.get("patches", {})
blocks_replace = patches_replace.get("dit", {})
transformer_options["total_blocks"] = len(self.blocks)
transformer_options["block_type"] = "double"
Expand All @@ -1645,6 +1678,11 @@ def block_wrap(args):
else:
x = block(x, e=e0, freqs=freqs, context=context, context_img_len=context_img_len, audio=audio, transformer_options=transformer_options)

if "double_block" in patches:
for p in patches["double_block"]:
out = p({"img": x, "x": x_input, "vec": e, "block_index": i, "img_offset": 0, "transformer_options": transformer_options})
x = out["img"]

# head
x = self.head(x, e)

Expand All @@ -1660,8 +1698,14 @@ def __init__(self, model_type="scail", patch_size=(1, 2, 2), in_dim=20, dim=5120

def forward_orig(self, x, t, context, clip_fea=None, freqs=None, transformer_options={}, pose_latents=None, reference_latent=None, ref_mask_latents=None, sam_latents=None, **kwargs):

x_input = x

img_offset = 0
if reference_latent is not None:
x = torch.cat((reference_latent, x), dim=2)
img_offset = (reference_latent.shape[2] // self.patch_size[0]) * \
(reference_latent.shape[3] // self.patch_size[1]) * \
(reference_latent.shape[4] // self.patch_size[2])

# embeddings
x = self.patch_embedding(x.float()).to(x.dtype)
Expand Down Expand Up @@ -1697,6 +1741,7 @@ def forward_orig(self, x, t, context, clip_fea=None, freqs=None, transformer_opt
context_img_len = clip_fea.shape[-2]

patches_replace = transformer_options.get("patches_replace", {})
patches = transformer_options.get("patches", {})
blocks_replace = patches_replace.get("dit", {})
transformer_options["total_blocks"] = len(self.blocks)
transformer_options["block_type"] = "double"
Expand All @@ -1712,6 +1757,11 @@ def block_wrap(args):
else:
x = block(x, e=e0, freqs=freqs, context=context, context_img_len=context_img_len, transformer_options=transformer_options)

if "double_block" in patches:
for p in patches["double_block"]:
out = p({"img": x, "x": x_input, "vec": e, "block_index": i, "img_offset": img_offset, "transformer_options": transformer_options})
x = out["img"]

# head
x = self.head(x, e)

Expand Down
9 changes: 9 additions & 0 deletions comfy/ldm/wan/model_animate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -493,6 +493,7 @@ def forward_orig(
**kwargs,
):
# embeddings
x_input = x
x = self.patch_embedding(x.float()).to(x.dtype)
x, motion_vec = self.after_patch_embedding(x, pose_latents, face_pixel_values)
grid_sizes = x.shape[2:]
Expand All @@ -505,11 +506,13 @@ def forward_orig(
e0 = self.time_projection(e).unflatten(2, (6, self.dim))

full_ref = None
img_offset = 0
if self.ref_conv is not None:
full_ref = kwargs.get("reference_latent", None)
if full_ref is not None:
full_ref = self.ref_conv(full_ref).flatten(2).transpose(1, 2)
x = torch.concat((full_ref, x), dim=1)
img_offset = full_ref.shape[1]

# context
context = self.text_embedding(context)
Expand All @@ -522,6 +525,7 @@ def forward_orig(
context_img_len = clip_fea.shape[-2]

patches_replace = transformer_options.get("patches_replace", {})
patches = transformer_options.get("patches", {})
blocks_replace = patches_replace.get("dit", {})
transformer_options["total_blocks"] = len(self.blocks)
transformer_options["block_type"] = "double"
Expand All @@ -537,6 +541,11 @@ def block_wrap(args):
else:
x = block(x, e=e0, freqs=freqs, context=context, context_img_len=context_img_len, transformer_options=transformer_options)

if "double_block" in patches:
for p in patches["double_block"]:
out = p({"img": x, "x": x_input, "vec": e, "block_index": i, "img_offset": img_offset, "transformer_options": transformer_options})
x = out["img"]

if i % 5 == 0 and motion_vec is not None:
x = x + self.face_adapter.fuser_blocks[i // 5](x, motion_vec)

Expand Down
10 changes: 10 additions & 0 deletions comfy/ldm/wan/model_wandancer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ def __init__(self,

def forward_orig(self, x, t, context, clip_fea=None, clip_fea_ref=None, freqs=None, audio_embed=None, fps=30, audio_inject_scale=1.0, transformer_options={}, **kwargs):
# embeddings
x_input = x
if int(fps + 0.5) != 30:
x = self.patch_embedding_global(x.float()).to(x.dtype)
else:
Expand All @@ -128,11 +129,13 @@ def forward_orig(self, x, t, context, clip_fea=None, clip_fea_ref=None, freqs=No
e0 = self.time_projection(e).unflatten(2, (6, self.dim))

full_ref = None
img_offset = 0
if self.ref_conv is not None: # model has the weight, but this wasn't used in the original pipeline
full_ref = kwargs.get("reference_latent", None)
if full_ref is not None:
full_ref = self.ref_conv(full_ref).flatten(2).transpose(1, 2)
x = torch.concat((full_ref, x), dim=1)
img_offset = full_ref.shape[1]

# context
context = self.text_embedding(context)
Expand Down Expand Up @@ -163,6 +166,7 @@ def forward_orig(self, x, t, context, clip_fea=None, clip_fea_ref=None, freqs=No
context_img_len += clip_fea_ref.shape[-2]

patches_replace = transformer_options.get("patches_replace", {})
patches = transformer_options.get("patches", {})
blocks_replace = patches_replace.get("dit", {})
transformer_options["total_blocks"] = len(self.blocks)
transformer_options["block_type"] = "double"
Expand All @@ -177,6 +181,12 @@ def block_wrap(args):
x = out["img"]
else:
x = block(x, e=e0, freqs=freqs, context=context, context_img_len=context_img_len, transformer_options=transformer_options)

if "double_block" in patches:
for p in patches["double_block"]:
out = p({"img": x, "x": x_input, "vec": e, "block_index": i, "img_offset": img_offset, "transformer_options": transformer_options})
x = out["img"]

if audio_emb is not None:
x = self.music_injector(x, i, audio_emb, audio_emb_global=None, seq_len=seq_len, scale=audio_inject_scale)

Expand Down
Loading
Loading