x: [B, L, N, C]. grid_sizes: [B, 3]. freqs: [M, C // 2].
(x, grid_sizes, freqs)
| 32 | |
| 33 | @amp.autocast(enabled=False) |
| 34 | def rope_apply(x, grid_sizes, freqs): |
| 35 | """ |
| 36 | x: [B, L, N, C]. |
| 37 | grid_sizes: [B, 3]. |
| 38 | freqs: [M, C // 2]. |
| 39 | """ |
| 40 | s, n, c = x.size(1), x.size(2), x.size(3) // 2 |
| 41 | # split freqs |
| 42 | freqs = freqs.split([c - 2 * (c // 3), c // 3, c // 3], dim=1) # [[N, head_dim/2], [N, head_dim/2], [N, head_dim/2]] # T H W 极坐标 |
| 43 | |
| 44 | # loop over samples |
| 45 | output = [] |
| 46 | for i, (f, h, w) in enumerate(grid_sizes.tolist()): |
| 47 | seq_len = f * h * w |
| 48 | |
| 49 | # precompute multipliers |
| 50 | x_i = torch.view_as_complex(x[i, :s].to(torch.float64).reshape( |
| 51 | s, n, -1, 2)) # [L, N, C/2] # 极坐标 |
| 52 | freqs_i = torch.cat([ |
| 53 | freqs[0][:f].view(f, 1, 1, -1).expand(f, h, w, -1), |
| 54 | freqs[1][:h].view(1, h, 1, -1).expand(f, h, w, -1), |
| 55 | freqs[2][:w].view(1, 1, w, -1).expand(f, h, w, -1) |
| 56 | ], |
| 57 | dim=-1).reshape(seq_len, 1, -1) # seq_lens, 1, 3 * dim / 2 (T H W) |
| 58 | |
| 59 | # apply rotary embedding |
| 60 | sp_size = get_sequence_parallel_world_size() |
| 61 | sp_rank = get_sequence_parallel_rank() |
| 62 | freqs_i = pad_freqs(freqs_i, s * sp_size) |
| 63 | s_per_rank = s |
| 64 | freqs_i_rank = freqs_i[(sp_rank * s_per_rank):((sp_rank + 1) * |
| 65 | s_per_rank), :, :] |
| 66 | x_i = torch.view_as_real(x_i * freqs_i_rank).flatten(2) |
| 67 | x_i = torch.cat([x_i, x[i, s:]]) |
| 68 | |
| 69 | # append to collection |
| 70 | output.append(x_i) |
| 71 | return torch.stack(output).float() |
| 72 | |
| 73 | |
| 74 | def usp_dit_forward_vace(self, x, vace_context, seq_len, kwargs): |
no test coverage detected