| 219 | |
| 220 | |
| 221 | def apply_rope(self, xq, xk, freqs_cis): |
| 222 | xq_ = xq.float().reshape(*xq.shape[:-1], -1, 1, 2) |
| 223 | xk_ = xk.float().reshape(*xk.shape[:-1], -1, 1, 2) |
| 224 | xq_out = freqs_cis[..., 0] * xq_[..., 0] + freqs_cis[..., 1] * xq_[..., 1] |
| 225 | xk_out = freqs_cis[..., 0] * xk_[..., 0] + freqs_cis[..., 1] * xk_[..., 1] |
| 226 | return xq_out.reshape(*xq.shape).type_as(xq), xk_out.reshape(*xk.shape).type_as(xk) |
| 227 | |
| 228 | |
| 229 | def process_attention(self, hidden_states, image_rotary_emb, attn_mask=None, ipadapter_kwargs_list=None): |