Returns a dictionary containing the encoded sequence or sequence pair and additional information: the mask for sequence classification and the overflowing elements if a ``max_length`` is specified. Args: text (:obj:`str`, :obj:`List[str]` or :obj:`List[int]` (th
(
self,
text: Union[TextInput, PreTokenizedInput, EncodedInput],
text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None,
add_special_tokens: bool = True,
padding: Union[bool, str] = False,
truncation: Union[bool, str] = False,
max_length: Optional[int] = None,
stride: int = 0,
is_pretokenized: bool = False,
pad_to_multiple_of: Optional[int] = None,
return_tensors: Optional[Union[str, TensorType]] = None,
return_token_type_ids: Optional[bool] = None,
return_attention_mask: Optional[bool] = None,
return_overflowing_tokens: bool = False,
return_special_tokens_mask: bool = False,
return_offsets_mapping: bool = False,
return_length: bool = False,
verbose: bool = True,
**kwargs
)
| 1671 | |
| 1672 | @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING) |
| 1673 | def encode_plus( |
| 1674 | self, |
| 1675 | text: Union[TextInput, PreTokenizedInput, EncodedInput], |
| 1676 | text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None, |
| 1677 | add_special_tokens: bool = True, |
| 1678 | padding: Union[bool, str] = False, |
| 1679 | truncation: Union[bool, str] = False, |
| 1680 | max_length: Optional[int] = None, |
| 1681 | stride: int = 0, |
| 1682 | is_pretokenized: bool = False, |
| 1683 | pad_to_multiple_of: Optional[int] = None, |
| 1684 | return_tensors: Optional[Union[str, TensorType]] = None, |
| 1685 | return_token_type_ids: Optional[bool] = None, |
| 1686 | return_attention_mask: Optional[bool] = None, |
| 1687 | return_overflowing_tokens: bool = False, |
| 1688 | return_special_tokens_mask: bool = False, |
| 1689 | return_offsets_mapping: bool = False, |
| 1690 | return_length: bool = False, |
| 1691 | verbose: bool = True, |
| 1692 | **kwargs |
| 1693 | ) -> BatchEncoding: |
| 1694 | """ |
| 1695 | Returns a dictionary containing the encoded sequence or sequence pair and additional information: |
| 1696 | the mask for sequence classification and the overflowing elements if a ``max_length`` is specified. |
| 1697 | |
| 1698 | Args: |
| 1699 | text (:obj:`str`, :obj:`List[str]` or :obj:`List[int]` (the later only for not-fast tokenizers)): |
| 1700 | The first sequence to be encoded. This can be a string, a list of strings (tokenized string using |
| 1701 | the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids` |
| 1702 | method) |
| 1703 | text_pair (:obj:`str`, :obj:`List[str]` or :obj:`List[int]`, `optional`, defaults to :obj:`None`): |
| 1704 | Optional second sequence to be encoded. This can be a string, a list of strings (tokenized |
| 1705 | string using the `tokenize` method) or a list of integers (tokenized string ids using the |
| 1706 | `convert_tokens_to_ids` method) |
| 1707 | """ |
| 1708 | |
| 1709 | # Backward compatibility for 'truncation_strategy', 'pad_to_max_length' |
| 1710 | padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies( |
| 1711 | padding=padding, |
| 1712 | truncation=truncation, |
| 1713 | max_length=max_length, |
| 1714 | pad_to_multiple_of=pad_to_multiple_of, |
| 1715 | verbose=verbose, |
| 1716 | **kwargs, |
| 1717 | ) |
| 1718 | |
| 1719 | return self._encode_plus( |
| 1720 | text=text, |
| 1721 | text_pair=text_pair, |
| 1722 | add_special_tokens=add_special_tokens, |
| 1723 | padding_strategy=padding_strategy, |
| 1724 | truncation_strategy=truncation_strategy, |
| 1725 | max_length=max_length, |
| 1726 | stride=stride, |
| 1727 | is_pretokenized=is_pretokenized, |
| 1728 | pad_to_multiple_of=pad_to_multiple_of, |
| 1729 | return_tensors=return_tensors, |
| 1730 | return_token_type_ids=return_token_type_ids, |
no test coverage detected