| 38 | Embeds scalar timesteps into vector representations. |
| 39 | """ |
| 40 | def __init__(self, hidden_size, frequency_embedding_size=256): |
| 41 | super().__init__() |
| 42 | self.mlp = nn.Sequential( |
| 43 | ColumnParallelLinear( |
| 44 | frequency_embedding_size, hidden_size, bias=True, |
| 45 | gather_output=False, |
| 46 | init_method=functools.partial(nn.init.normal_, std=0.02), |
| 47 | ), |
| 48 | nn.SiLU(), |
| 49 | RowParallelLinear( |
| 50 | hidden_size, hidden_size, bias=True, input_is_parallel=True, |
| 51 | init_method=functools.partial(nn.init.normal_, std=0.02), |
| 52 | ), |
| 53 | ) |
| 54 | self.frequency_embedding_size = frequency_embedding_size |
| 55 | |
| 56 | @staticmethod |
| 57 | def timestep_embedding(t, dim, max_period=10000): |