r""" labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): Labels for computing the masked language modeling loss. Indices should be in ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring)
(
self,
input_ids=None,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
inputs_embeds=None,
labels=None,
encoder_hidden_states=None,
encoder_attention_mask=None,
output_attentions=None,
output_hidden_states=None,
**kwargs
)
| 1020 | @add_start_docstrings_to_callable(BERT_INPUTS_DOCSTRING.format("(batch_size, sequence_length)")) |
| 1021 | @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="bert-base-uncased") |
| 1022 | def forward( |
| 1023 | self, |
| 1024 | input_ids=None, |
| 1025 | attention_mask=None, |
| 1026 | token_type_ids=None, |
| 1027 | position_ids=None, |
| 1028 | head_mask=None, |
| 1029 | inputs_embeds=None, |
| 1030 | labels=None, |
| 1031 | encoder_hidden_states=None, |
| 1032 | encoder_attention_mask=None, |
| 1033 | output_attentions=None, |
| 1034 | output_hidden_states=None, |
| 1035 | **kwargs |
| 1036 | ): |
| 1037 | r""" |
| 1038 | labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): |
| 1039 | Labels for computing the masked language modeling loss. |
| 1040 | Indices should be in ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) |
| 1041 | Tokens with indices set to ``-100`` are ignored (masked), the loss is only computed for the tokens with labels |
| 1042 | in ``[0, ..., config.vocab_size]`` |
| 1043 | kwargs (:obj:`Dict[str, any]`, optional, defaults to `{}`): |
| 1044 | Used to hide legacy arguments that have been deprecated. |
| 1045 | |
| 1046 | Returns: |
| 1047 | :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs: |
| 1048 | masked_lm_loss (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: |
| 1049 | Masked language modeling loss. |
| 1050 | prediction_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, config.vocab_size)`) |
| 1051 | Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). |
| 1052 | hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``): |
| 1053 | Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) |
| 1054 | of shape :obj:`(batch_size, sequence_length, hidden_size)`. |
| 1055 | |
| 1056 | Hidden-states of the model at the output of each layer plus the initial embedding outputs. |
| 1057 | attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``): |
| 1058 | Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape |
| 1059 | :obj:`(batch_size, num_heads, sequence_length, sequence_length)`. |
| 1060 | |
| 1061 | Attentions weights after the attention softmax, used to compute the weighted average in the self-attention |
| 1062 | heads. |
| 1063 | """ |
| 1064 | if "masked_lm_labels" in kwargs: |
| 1065 | warnings.warn( |
| 1066 | "The `masked_lm_labels` argument is deprecated and will be removed in a future version, use `labels` instead.", |
| 1067 | DeprecationWarning, |
| 1068 | ) |
| 1069 | labels = kwargs.pop("masked_lm_labels") |
| 1070 | assert "lm_labels" not in kwargs, "Use `BertWithLMHead` for autoregressive language modeling task." |
| 1071 | assert kwargs == {}, f"Unexpected keyword arguments: {list(kwargs.keys())}." |
| 1072 | |
| 1073 | outputs = self.bert( |
| 1074 | input_ids, |
| 1075 | attention_mask=attention_mask, |
| 1076 | token_type_ids=token_type_ids, |
| 1077 | position_ids=position_ids, |
| 1078 | head_mask=head_mask, |
| 1079 | inputs_embeds=inputs_embeds, |