(self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None)
| 161 | self.dropout = nn.Dropout(config.hidden_dropout_prob) |
| 162 | |
| 163 | def forward(self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None): |
| 164 | if input_ids is not None: |
| 165 | input_shape = input_ids.size() |
| 166 | else: |
| 167 | input_shape = inputs_embeds.size()[:-1] |
| 168 | |
| 169 | seq_length = input_shape[1] |
| 170 | device = input_ids.device if input_ids is not None else inputs_embeds.device |
| 171 | if position_ids is None: |
| 172 | position_ids = torch.arange(seq_length, dtype=torch.long, device=device) |
| 173 | position_ids = position_ids.unsqueeze(0).expand(input_shape) |
| 174 | if token_type_ids is None: |
| 175 | token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) |
| 176 | |
| 177 | if inputs_embeds is None: |
| 178 | inputs_embeds = self.word_embeddings(input_ids) |
| 179 | position_embeddings = self.position_embeddings(position_ids) |
| 180 | token_type_embeddings = self.token_type_embeddings(token_type_ids) |
| 181 | |
| 182 | embeddings = inputs_embeds + position_embeddings + token_type_embeddings |
| 183 | embeddings = self.LayerNorm(embeddings) |
| 184 | embeddings = self.dropout(embeddings) |
| 185 | return embeddings |
| 186 | |
| 187 | |
| 188 | class BertSelfAttention(nn.Module): |
nothing calls this directly
no outgoing calls
no test coverage detected