| 38 | |
| 39 | |
| 40 | class TimeEmbedding(nn.Module): |
| 41 | def __init__(self, dim): |
| 42 | super().__init__() |
| 43 | self.dim = dim |
| 44 | inv_freq = torch.exp( |
| 45 | torch.arange(0, dim, 2, dtype=torch.float32) * (-math.log(10000) / dim) |
| 46 | ) |
| 47 | self.register_buffer("inv_freq", inv_freq) |
| 48 | |
| 49 | def forward(self, input): |
| 50 | shape = input.shape |
| 51 | sinusoid_in = torch.ger(input.view(-1).float(), self.inv_freq) |
| 52 | pos_emb = torch.cat([sinusoid_in.sin(), sinusoid_in.cos()], dim=-1) |
| 53 | pos_emb = pos_emb.view(*shape, self.dim) |
| 54 | return pos_emb |
| 55 | |
| 56 | |
| 57 | class Swish(nn.Module): |