| 780 | """ |
| 781 | |
| 782 | def __init__( |
| 783 | self, |
| 784 | hidden_size, |
| 785 | vocab_size, |
| 786 | max_sequence_length, |
| 787 | ): |
| 788 | super(Embedding, self).__init__() |
| 789 | self.hidden_size = hidden_size |
| 790 | self.vocab_size = vocab_size |
| 791 | self.max_sequence_length = max_sequence_length |
| 792 | |
| 793 | # Word embeddings. |
| 794 | self.word_embeddings = torch.nn.Embedding(self.vocab_size, self.hidden_size) |
| 795 | self._word_embeddings_key = 'word_embeddings' |
| 796 | |
| 797 | # Position embedding. |
| 798 | self.position_embeddings = torch.nn.Embedding(self.max_sequence_length, self.hidden_size) |
| 799 | self.position_embeddings = self.position_embeddings.half() |
| 800 | self._position_embeddings_key = 'position_embeddings' |
| 801 | |
| 802 | def forward(self, input_ids, position_ids): |
| 803 | # Embeddings. |