### Embeddings for $t$
| 17 | |
| 18 | |
| 19 | class TimeEmbedding(nn.Module): |
| 20 | """ |
| 21 | ### Embeddings for $t$ |
| 22 | """ |
| 23 | |
| 24 | def __init__(self, n_channels: int): |
| 25 | """ |
| 26 | * `n_channels` is the number of dimensions in the embedding |
| 27 | """ |
| 28 | super().__init__() |
| 29 | self.n_channels = n_channels |
| 30 | # First linear layer |
| 31 | self.lin1 = nn.Linear(self.n_channels // 4, self.n_channels) |
| 32 | # Activation |
| 33 | self.act = Swish() |
| 34 | # Second linear layer |
| 35 | self.lin2 = nn.Linear(self.n_channels, self.n_channels) |
| 36 | |
| 37 | def forward(self, t: torch.Tensor): |
| 38 | # Create sinusoidal position embeddings |
| 39 | # [same as those from the transformer](../../transformers/positional_encoding.html) |
| 40 | # |
| 41 | # \begin{align} |
| 42 | # PE^{(1)}_{t,i} &= sin\Bigg(\frac{t}{10000^{\frac{i}{d - 1}}}\Bigg) \\ |
| 43 | # PE^{(2)}_{t,i} &= cos\Bigg(\frac{t}{10000^{\frac{i}{d - 1}}}\Bigg) |
| 44 | # \end{align} |
| 45 | # |
| 46 | # where $d$ is `half_dim` |
| 47 | half_dim = self.n_channels // 8 |
| 48 | emb = math.log(10_000) / (half_dim - 1) |
| 49 | emb = torch.exp(torch.arange(half_dim, device=t.device) * -emb) |
| 50 | emb = t[:, None] * emb[None, :] |
| 51 | emb = torch.cat((emb.sin(), emb.cos()), dim=1) |
| 52 | |
| 53 | # Transform with the MLP |
| 54 | emb = self.act(self.lin1(emb)) |
| 55 | emb = self.lin2(emb) |
| 56 | |
| 57 | # |
| 58 | return emb |
| 59 | |
| 60 | |
| 61 | class ResidualBlock(nn.Module): |