Method
_encode_plus
(
self,
text: Union[TextInput, PreTokenizedInput, EncodedInput],
text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None,
add_special_tokens: bool = True,
padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
max_length: Optional[int] = None,
stride: int = 0,
is_pretokenized: bool = False,
pad_to_multiple_of: Optional[int] = None,
return_tensors: Optional[Union[str, TensorType]] = None,
return_token_type_ids: Optional[bool] = None,
return_attention_mask: Optional[bool] = None,
return_overflowing_tokens: bool = False,
return_special_tokens_mask: bool = False,
return_offsets_mapping: bool = False,
return_length: bool = False,
verbose: bool = True,
**kwargs
)
Source from the content-addressed store, hash-verified
| 1738 | ) |
| 1739 | |
| 1740 | def _encode_plus( |
| 1741 | self, |
| 1742 | text: Union[TextInput, PreTokenizedInput, EncodedInput], |
| 1743 | text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None, |
| 1744 | add_special_tokens: bool = True, |
| 1745 | padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, |
| 1746 | truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE, |
| 1747 | max_length: Optional[int] = None, |
| 1748 | stride: int = 0, |
| 1749 | is_pretokenized: bool = False, |
| 1750 | pad_to_multiple_of: Optional[int] = None, |
| 1751 | return_tensors: Optional[Union[str, TensorType]] = None, |
| 1752 | return_token_type_ids: Optional[bool] = None, |
| 1753 | return_attention_mask: Optional[bool] = None, |
| 1754 | return_overflowing_tokens: bool = False, |
| 1755 | return_special_tokens_mask: bool = False, |
| 1756 | return_offsets_mapping: bool = False, |
| 1757 | return_length: bool = False, |
| 1758 | verbose: bool = True, |
| 1759 | **kwargs |
| 1760 | ) -> BatchEncoding: |
| 1761 | raise NotImplementedError |
| 1762 | |
| 1763 | @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING) |
| 1764 | def batch_encode_plus( |
Tested by
no test coverage detected