(self,
cfg,
)
| 67 | """ |
| 68 | |
| 69 | def __init__(self, |
| 70 | cfg, |
| 71 | ): |
| 72 | super().__init__() |
| 73 | self.hidden_size = cfg.hidden_size |
| 74 | self.params_dtype = cfg.params_dtype |
| 75 | self.fp32_residual_connection = cfg.fp32_residual_connection |
| 76 | self.embedding_weights_in_fp32 = cfg.embedding_weights_in_fp32 |
| 77 | self.word_embeddings = torch.nn.Embedding( |
| 78 | cfg.padded_vocab_size, self.hidden_size, |
| 79 | ) |
| 80 | self.embedding_dropout = torch.nn.Dropout(cfg.hidden_dropout) |
| 81 | |
| 82 | def forward(self, input_ids): |
| 83 | # Embeddings. |