| 201 | |
| 202 | |
| 203 | def pos_interpolate(pos, seq_len): |
| 204 | if pos.size(1) == seq_len: |
| 205 | return pos |
| 206 | else: |
| 207 | src_grid = int(math.sqrt(pos.size(1))) |
| 208 | tar_grid = int(math.sqrt(seq_len)) |
| 209 | n = pos.size(1) - src_grid * src_grid |
| 210 | return torch.cat([ |
| 211 | pos[:, :n], |
| 212 | F.interpolate( |
| 213 | pos[:, n:].float().reshape(1, src_grid, src_grid, -1).permute( |
| 214 | 0, 3, 1, 2), |
| 215 | size=(tar_grid, tar_grid), |
| 216 | mode='bicubic', |
| 217 | align_corners=False).flatten(2).transpose(1, 2) |
| 218 | ], |
| 219 | dim=1) |
| 220 | |
| 221 | |
| 222 | class QuickGELU(nn.Module): |