(
self,
input_ids: Optional[torch.Tensor] = None,
attention_mask: Optional[torch.Tensor] = None,
token_type_ids: Optional[torch.Tensor] = None,
position_ids: Optional[torch.Tensor] = None,
head_mask: Optional[torch.Tensor] = None,
inputs_embeds: Optional[torch.Tensor] = None,
encoder_hidden_states: Optional[torch.Tensor] = None,
encoder_attention_mask: Optional[torch.Tensor] = None,
labels: Optional[torch.Tensor] = None,
output_attentions: Optional[bool] = None,
output_hidden_states: Optional[bool] = None,
return_dict: Optional[bool] = None,
)
| 348 | self.cls.predictions.decoder = new_embeddings |
| 349 | |
| 350 | def forward( |
| 351 | self, |
| 352 | input_ids: Optional[torch.Tensor] = None, |
| 353 | attention_mask: Optional[torch.Tensor] = None, |
| 354 | token_type_ids: Optional[torch.Tensor] = None, |
| 355 | position_ids: Optional[torch.Tensor] = None, |
| 356 | head_mask: Optional[torch.Tensor] = None, |
| 357 | inputs_embeds: Optional[torch.Tensor] = None, |
| 358 | encoder_hidden_states: Optional[torch.Tensor] = None, |
| 359 | encoder_attention_mask: Optional[torch.Tensor] = None, |
| 360 | labels: Optional[torch.Tensor] = None, |
| 361 | output_attentions: Optional[bool] = None, |
| 362 | output_hidden_states: Optional[bool] = None, |
| 363 | return_dict: Optional[bool] = None, |
| 364 | ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: |
| 365 | # labels should be a `torch.LongTensor` of shape |
| 366 | # `(batch_size, sequence_length)`. These are used for computing the |
| 367 | # masked language modeling loss. |
| 368 | # |
| 369 | # Indices should be in `[-100, 0, ..., config.vocab_size]` (see |
| 370 | # `input_ids` docstring) Tokens with indices set to `-100` are ignored |
| 371 | # (masked), the loss is only computed for the tokens with labels in `[0, |
| 372 | # ..., config.vocab_size]` |
| 373 | # |
| 374 | # Prediction scores are only computed for masked tokens and the (bs, |
| 375 | # seqlen) dimensions are flattened |
| 376 | if (input_ids is not None) == (inputs_embeds is not None): |
| 377 | raise ValueError("Must specify either input_ids or input_embeds!") |
| 378 | |
| 379 | if labels is None: |
| 380 | masked_tokens_mask = None |
| 381 | else: |
| 382 | masked_tokens_mask = labels > 0 |
| 383 | |
| 384 | return_dict = return_dict if return_dict is not None else self.config.use_return_dict |
| 385 | |
| 386 | outputs = self.bert( |
| 387 | input_ids, |
| 388 | attention_mask=attention_mask, |
| 389 | token_type_ids=token_type_ids, |
| 390 | position_ids=position_ids, |
| 391 | head_mask=head_mask, |
| 392 | inputs_embeds=inputs_embeds, |
| 393 | encoder_hidden_states=encoder_hidden_states, |
| 394 | encoder_attention_mask=encoder_attention_mask, |
| 395 | output_attentions=output_attentions, |
| 396 | output_hidden_states=output_hidden_states, |
| 397 | return_dict=return_dict, |
| 398 | masked_tokens_mask=masked_tokens_mask, |
| 399 | ) |
| 400 | |
| 401 | sequence_output = outputs[0] |
| 402 | prediction_scores = self.cls(sequence_output) |
| 403 | |
| 404 | loss = None |
| 405 | if labels is not None: |
| 406 | # Compute loss |
| 407 | loss_fct = nn.CrossEntropyLoss() |
nothing calls this directly
no test coverage detected