| 197 | class MemoryEfficientCrossAttention(nn.Module): |
| 198 | # https://github.com/MatthieuTPHR/diffusers/blob/d80b531ff8060ec1ea982b65a1b8df70f73aa67c/src/diffusers/models/attention.py#L223 |
| 199 | def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.0): |
| 200 | super().__init__() |
| 201 | print(f"Setting up {self.__class__.__name__}. Query dim is {query_dim}, context_dim is {context_dim} and using " |
| 202 | f"{heads} heads.") |
| 203 | inner_dim = dim_head * heads |
| 204 | context_dim = default(context_dim, query_dim) |
| 205 | |
| 206 | self.heads = heads |
| 207 | self.dim_head = dim_head |
| 208 | |
| 209 | self.to_q = nn.Linear(query_dim, inner_dim, bias=False) |
| 210 | self.to_k = nn.Linear(context_dim, inner_dim, bias=False) |
| 211 | self.to_v = nn.Linear(context_dim, inner_dim, bias=False) |
| 212 | |
| 213 | self.to_out = nn.Sequential(nn.Linear(inner_dim, query_dim), nn.Dropout(dropout)) |
| 214 | self.attention_op: Optional[Any] = None |
| 215 | |
| 216 | def forward(self, x, context=None, mask=None): |
| 217 | q = self.to_q(x) |