| 25 | return x |
| 26 | |
| 27 | class IpAdapterModule(torch.nn.Module): |
| 28 | def __init__(self, num_attention_heads, attention_head_dim, input_dim): |
| 29 | super().__init__() |
| 30 | self.num_heads = num_attention_heads |
| 31 | self.head_dim = attention_head_dim |
| 32 | output_dim = num_attention_heads * attention_head_dim |
| 33 | self.to_k_ip = torch.nn.Linear(input_dim, output_dim, bias=False) |
| 34 | self.to_v_ip = torch.nn.Linear(input_dim, output_dim, bias=False) |
| 35 | self.norm_added_k = RMSNorm(attention_head_dim, eps=1e-5, elementwise_affine=False) |
| 36 | |
| 37 | |
| 38 | def forward(self, hidden_states): |
| 39 | batch_size = hidden_states.shape[0] |
| 40 | # ip_k |
| 41 | ip_k = self.to_k_ip(hidden_states) |
| 42 | ip_k = ip_k.view(batch_size, -1, self.num_heads, self.head_dim).transpose(1, 2) |
| 43 | ip_k = self.norm_added_k(ip_k) |
| 44 | # ip_v |
| 45 | ip_v = self.to_v_ip(hidden_states) |
| 46 | ip_v = ip_v.view(batch_size, -1, self.num_heads, self.head_dim).transpose(1, 2) |
| 47 | return ip_k, ip_v |
| 48 | |
| 49 | |
| 50 | class FluxIpAdapter(torch.nn.Module): |