| 57 | return tensor[:, :, :oh - 1, :ow - 1] |
| 58 | |
| 59 | class SinusoidalPositionEmbeddings(nn.Module): |
| 60 | def __init__(self, dim): |
| 61 | super().__init__() |
| 62 | self.dim = dim |
| 63 | |
| 64 | def forward(self, time): |
| 65 | device = time.device |
| 66 | half_dim = self.dim // 2 |
| 67 | embeddings = math.log(10000) / (half_dim - 1) |
| 68 | embeddings = torch.exp(torch.arange(half_dim, device=device) * -embeddings) |
| 69 | embeddings = time[:, None] * embeddings[None, :] |
| 70 | embeddings = torch.cat((embeddings.sin(), embeddings.cos()), dim=-1) |
| 71 | return embeddings |
| 72 | |
| 73 | |
| 74 | class GaussianFourierProjection(nn.Module): |