| 883 | |
| 884 | |
| 885 | class BertLMHeadModel(BertPreTrainedModel): |
| 886 | |
| 887 | _keys_to_ignore_on_load_unexpected = [r"pooler"] |
| 888 | _keys_to_ignore_on_load_missing = [r"position_ids", r"predictions.decoder.bias"] |
| 889 | |
| 890 | def __init__(self, config): |
| 891 | super().__init__(config) |
| 892 | |
| 893 | self.bert = BertModel(config, add_pooling_layer=False) |
| 894 | self.cls = BertOnlyMLMHead(config) |
| 895 | |
| 896 | self.init_weights() |
| 897 | |
| 898 | def get_output_embeddings(self): |
| 899 | return self.cls.predictions.decoder |
| 900 | |
| 901 | def set_output_embeddings(self, new_embeddings): |
| 902 | self.cls.predictions.decoder = new_embeddings |
| 903 | |
| 904 | def forward( |
| 905 | self, |
| 906 | input_ids=None, |
| 907 | attention_mask=None, |
| 908 | position_ids=None, |
| 909 | head_mask=None, |
| 910 | inputs_embeds=None, |
| 911 | encoder_hidden_states=None, |
| 912 | encoder_attention_mask=None, |
| 913 | labels=None, |
| 914 | past_key_values=None, |
| 915 | use_cache=None, |
| 916 | output_attentions=None, |
| 917 | output_hidden_states=None, |
| 918 | return_dict=None, |
| 919 | return_logits=False, |
| 920 | is_decoder=True, |
| 921 | reduction='mean', |
| 922 | mode='multimodal', |
| 923 | ): |
| 924 | r""" |
| 925 | encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): |
| 926 | Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if |
| 927 | the model is configured as a decoder. |
| 928 | encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): |
| 929 | Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in |
| 930 | the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: |
| 931 | - 1 for tokens that are **not masked**, |
| 932 | - 0 for tokens that are **masked**. |
| 933 | labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): |
| 934 | Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in |
| 935 | ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are |
| 936 | ignored (masked), the loss is only computed for the tokens with labels n ``[0, ..., config.vocab_size]`` |
| 937 | past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): |
| 938 | Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. |
| 939 | If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` |
| 940 | (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` |
| 941 | instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. |
| 942 | use_cache (:obj:`bool`, `optional`): |