Embeds scalar timesteps into vector representations.
| 87 | return x |
| 88 | |
| 89 | class TimestepEmbedder(nn.Module): |
| 90 | """ |
| 91 | Embeds scalar timesteps into vector representations. |
| 92 | """ |
| 93 | def __init__(self, hidden_size, frequency_embedding_size = 256): |
| 94 | super().__init__() |
| 95 | self.mlp = nn.Sequential( |
| 96 | nn.Linear(frequency_embedding_size, hidden_size, bias=True), |
| 97 | nn.SiLU(), |
| 98 | nn.Linear(hidden_size, hidden_size, bias=True), |
| 99 | ) |
| 100 | self.frequency_embedding_size = frequency_embedding_size |
| 101 | |
| 102 | @staticmethod |
| 103 | def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor: |
| 104 | """ |
| 105 | Create sinusoidal timestep embeddings. |
| 106 | :param t: a 1-D Tensor of N indices, one per batch element. |
| 107 | These may be fractional. |
| 108 | :param dim: the dimension of the output. |
| 109 | :param max_period: controls the minimum frequency of the embeddings. |
| 110 | :return: an (N, D) Tensor of positional embeddings. |
| 111 | """ |
| 112 | # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py |
| 113 | half = dim // 2 |
| 114 | freqs = torch.exp( |
| 115 | -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half |
| 116 | ).to(device=t.device) |
| 117 | args = t[:, None].float() * freqs[None] |
| 118 | embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) |
| 119 | if dim % 2: |
| 120 | embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) |
| 121 | return embedding |
| 122 | |
| 123 | def forward(self, t: torch.Tensor) -> torch.Tensor: |
| 124 | t_freq = self.timestep_embedding(t, self.frequency_embedding_size) |
| 125 | t_emb = self.mlp(t_freq) |
| 126 | return t_emb |
| 127 | |
| 128 | class SequenceEmbed(nn.Module): |
| 129 | def __init__( |