The GLIGEN model was created by researchers and engineers from [University of Wisconsin-Madison, Columbia University, and Microsoft](https://github.com/gligen/GLIGEN). The [`StableDiffusionGLIGENPipeline`] can generate photorealistic images conditioned on grounding inputs. Along with text and bounding boxes, if input images are given, this pipeline can insert objects described by text at the region defined by bounding boxes. Otherwise, it'll generate an image described by the caption/prompt and insert objects described by text at the region defined by bounding boxes. It's trained on COCO2014D and COCO2014CD datasets, and the model uses a frozen CLIP ViT-L/14 text encoder to condition itself on grounding inputs.

The abstract from the [paper](https://huggingface.co/papers/2301.07093) is:

*Large-scale text-to-image diffusion models have made amazing advances. However, the status quo is to use text input alone, which can impede controllability. In this work, we propose GLIGEN, Grounded-Language-to-Image Generation, a novel approach that builds upon and extends the functionality of existing pre-trained text-to-image diffusion models by enabling them to also be conditioned on grounding inputs. To preserve the vast concept knowledge of the pre-trained model, we freeze all of its weights and inject the grounding information into new trainable layers via a gated mechanism. Our model achieves open-world grounded text2img generation with caption and bounding box condition inputs, and the grounding ability generalizes well to novel spatial configurations and concepts. GLIGEN’s zeroshot performance on COCO and LVIS outperforms existing supervised layout-to-image baselines by a large margin.*

<Tip>

Make sure to check out the Stable Diffusion [Tips](https://huggingface.co/docs/diffusers/en/api/pipelines/stable_diffusion/overview#tips) section to learn how to explore the tradeoff between scheduler speed and quality and how to reuse pipeline components efficiently!

If you want to use one of the official checkpoints for a task, explore the [gligen](https://huggingface.co/gligen) Hub organizations!

</Tip>

This pipeline was contributed by [Nikhil Gajendrakumar](https://github.com/nikhil-masterful).

[[autodoc]] StableDiffusionGLIGENPipeline

- all

- __call__

- enable_vae_slicing

- disable_vae_slicing

- enable_vae_tiling

- disable_vae_tiling

- enable_model_cpu_offload

- prepare_latents

- enable_fuser

[[autodoc]] pipelines.stable_diffusion.StableDiffusionPipelineOutput

StableDiffusionGLIGENPipeline,

class GatedSelfAttentionDense(nn.Module):

def __init__(self, query_dim, context_dim, n_heads, d_head):

super().__init__()

# we need a linear projection since we need cat visual feature and obj feature

self.linear = nn.Linear(context_dim, query_dim)

self.attn = Attention(query_dim=query_dim, heads=n_heads, dim_head=d_head)

self.ff = FeedForward(query_dim, activation_fn="geglu")

self.norm1 = nn.LayerNorm(query_dim)

self.norm2 = nn.LayerNorm(query_dim)

self.register_parameter("alpha_attn", nn.Parameter(torch.tensor(0.0)))

self.register_parameter("alpha_dense", nn.Parameter(torch.tensor(0.0)))

self.enabled = True

def forward(self, x, objs):

if not self.enabled:

return x

n_visual = x.shape[1]

objs = self.linear(objs)

x = x + self.alpha_attn.tanh() * self.attn(self.norm1(torch.cat([x, objs], dim=1)))[:, :n_visual, :]

x = x + self.alpha_dense.tanh() * self.ff(self.norm2(x))

return x

attention_type: str = "default",

# 4. Fuser

if attention_type == "gated":

self.fuser = GatedSelfAttentionDense(dim, cross_attention_dim, num_attention_heads, attention_head_dim)

# 0. Prepare GLIGEN inputs

cross_attention_kwargs = cross_attention_kwargs.copy() if cross_attention_kwargs is not None else {}

gligen_kwargs = cross_attention_kwargs.pop("gligen", None)

# 1.5 GLIGEN Control

if gligen_kwargs is not None:

hidden_states = self.fuser(hidden_states, gligen_kwargs["objs"])

# 1.5 ends

class FourierEmbedder(nn.Module):

def __init__(self, num_freqs=64, temperature=100):

super().__init__()

self.num_freqs = num_freqs

self.temperature = temperature

freq_bands = temperature ** (torch.arange(num_freqs) / num_freqs)

freq_bands = freq_bands[None, None, None]

self.register_buffer("freq_bands", freq_bands, persistent=False)

def __call__(self, x):

x = self.freq_bands * x.unsqueeze(-1)

return torch.stack((x.sin(), x.cos()), dim=-1).permute(0, 1, 3, 4, 2).reshape(*x.shape[:2], -1)

class PositionNet(nn.Module):

def __init__(self, positive_len, out_dim, fourier_freqs=8):

super().__init__()

self.positive_len = positive_len

self.out_dim = out_dim

self.fourier_embedder = FourierEmbedder(num_freqs=fourier_freqs)

self.position_dim = fourier_freqs * 2 * 4 # 2: sin/cos, 4: xyxy

if isinstance(out_dim, tuple):

out_dim = out_dim[0]

self.linears = nn.Sequential(

nn.Linear(self.positive_len + self.position_dim, 512),

nn.SiLU(),

nn.Linear(512, 512),

nn.SiLU(),

nn.Linear(512, out_dim),

)

self.null_positive_feature = torch.nn.Parameter(torch.zeros([self.positive_len]))

self.null_position_feature = torch.nn.Parameter(torch.zeros([self.position_dim]))

def forward(self, boxes, masks, positive_embeddings):

masks = masks.unsqueeze(-1)

# embedding position (it may includes padding as placeholder)

xyxy_embedding = self.fourier_embedder(boxes) # B*N*4 -> B*N*C

# learnable null embedding

positive_null = self.null_positive_feature.view(1, 1, -1)

xyxy_null = self.null_position_feature.view(1, 1, -1)

# replace padding with learnable null embedding

positive_embeddings = positive_embeddings * masks + (1 - masks) * positive_null

xyxy_embedding = xyxy_embedding * masks + (1 - masks) * xyxy_null

objs = self.linears(torch.cat([positive_embeddings, xyxy_embedding], dim=-1))

return objs

attention_type: str = "default",

attention_type=attention_type,

attention_type="default",

attention_type=attention_type,

attention_type="default",

attention_type=attention_type,

attention_type="default",

attention_type=attention_type,

attention_type="default",

attention_type=attention_type,

attention_type="default",

attention_type=attention_type,

PositionNet,

attention_type: str = "default",

attention_type=attention_type,

attention_type=attention_type,

attention_type=attention_type,

if attention_type == "gated":

positive_len = 768

if isinstance(cross_attention_dim, int):

positive_len = cross_attention_dim

elif isinstance(cross_attention_dim, tuple) or isinstance(cross_attention_dim, list):

positive_len = cross_attention_dim[0]

self.position_net = PositionNet(positive_len=positive_len, out_dim=cross_attention_dim)

# 2.5 GLIGEN position net

if cross_attention_kwargs is not None and cross_attention_kwargs.get("gligen", None) is not None:

cross_attention_kwargs = cross_attention_kwargs.copy()

gligen_args = cross_attention_kwargs.pop("gligen")

cross_attention_kwargs["gligen"] = {"objs": self.position_net(**gligen_args)}

StableDiffusionGLIGENPipeline,

from .pipeline_stable_diffusion_gligen import StableDiffusionGLIGENPipeline