Embeds scalar timesteps into vector representations.
| 34 | ############################################################################# |
| 35 | |
| 36 | class ParallelTimestepEmbedder(nn.Module): |
| 37 | """ |
| 38 | Embeds scalar timesteps into vector representations. |
| 39 | """ |
| 40 | def __init__(self, hidden_size, frequency_embedding_size=256): |
| 41 | super().__init__() |
| 42 | self.mlp = nn.Sequential( |
| 43 | ColumnParallelLinear( |
| 44 | frequency_embedding_size, hidden_size, bias=True, |
| 45 | gather_output=False, |
| 46 | init_method=functools.partial(nn.init.normal_, std=0.02), |
| 47 | ), |
| 48 | nn.SiLU(), |
| 49 | RowParallelLinear( |
| 50 | hidden_size, hidden_size, bias=True, input_is_parallel=True, |
| 51 | init_method=functools.partial(nn.init.normal_, std=0.02), |
| 52 | ), |
| 53 | ) |
| 54 | self.frequency_embedding_size = frequency_embedding_size |
| 55 | |
| 56 | @staticmethod |
| 57 | def timestep_embedding(t, dim, max_period=10000): |
| 58 | """ |
| 59 | Create sinusoidal timestep embeddings. |
| 60 | :param t: a 1-D Tensor of N indices, one per batch element. |
| 61 | These may be fractional. |
| 62 | :param dim: the dimension of the output. |
| 63 | :param max_period: controls the minimum frequency of the embeddings. |
| 64 | :return: an (N, D) Tensor of positional embeddings. |
| 65 | """ |
| 66 | # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py |
| 67 | half = dim // 2 |
| 68 | freqs = torch.exp( |
| 69 | -math.log(max_period) * torch.arange( |
| 70 | start=0, end=half, dtype=torch.float32 |
| 71 | ) / half |
| 72 | ).to(device=t.device) |
| 73 | args = t[:, None].float() * freqs[None] |
| 74 | embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) |
| 75 | if dim % 2: |
| 76 | embedding = torch.cat([ |
| 77 | embedding, torch.zeros_like(embedding[:, :1]) |
| 78 | ], dim=-1) |
| 79 | return embedding |
| 80 | |
| 81 | def forward(self, t): |
| 82 | t_freq = self.timestep_embedding(t, self.frequency_embedding_size) |
| 83 | t_emb = self.mlp(t_freq.to(self.mlp[0].weight.dtype)) |
| 84 | return t_emb |
| 85 | |
| 86 | |
| 87 | class ParallelLabelEmbedder(nn.Module): |