Method
_batch_encode_plus
(
self,
batch_text_or_text_pairs: Union[
List[TextInput],
List[TextInputPair],
List[PreTokenizedInput],
List[PreTokenizedInputPair],
List[EncodedInput],
List[EncodedInputPair],
],
add_special_tokens: bool = True,
padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
max_length: Optional[int] = None,
stride: int = 0,
is_pretokenized: bool = False,
pad_to_multiple_of: Optional[int] = None,
return_tensors: Optional[Union[str, TensorType]] = None,
return_token_type_ids: Optional[bool] = None,
return_attention_mask: Optional[bool] = None,
return_overflowing_tokens: bool = False,
return_special_tokens_mask: bool = False,
return_offsets_mapping: bool = False,
return_length: bool = False,
verbose: bool = True,
**kwargs
)
Source from the content-addressed store, hash-verified
| 1833 | ) |
| 1834 | |
| 1835 | def _batch_encode_plus( |
| 1836 | self, |
| 1837 | batch_text_or_text_pairs: Union[ |
| 1838 | List[TextInput], |
| 1839 | List[TextInputPair], |
| 1840 | List[PreTokenizedInput], |
| 1841 | List[PreTokenizedInputPair], |
| 1842 | List[EncodedInput], |
| 1843 | List[EncodedInputPair], |
| 1844 | ], |
| 1845 | add_special_tokens: bool = True, |
| 1846 | padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, |
| 1847 | truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE, |
| 1848 | max_length: Optional[int] = None, |
| 1849 | stride: int = 0, |
| 1850 | is_pretokenized: bool = False, |
| 1851 | pad_to_multiple_of: Optional[int] = None, |
| 1852 | return_tensors: Optional[Union[str, TensorType]] = None, |
| 1853 | return_token_type_ids: Optional[bool] = None, |
| 1854 | return_attention_mask: Optional[bool] = None, |
| 1855 | return_overflowing_tokens: bool = False, |
| 1856 | return_special_tokens_mask: bool = False, |
| 1857 | return_offsets_mapping: bool = False, |
| 1858 | return_length: bool = False, |
| 1859 | verbose: bool = True, |
| 1860 | **kwargs |
| 1861 | ) -> BatchEncoding: |
| 1862 | raise NotImplementedError |
| 1863 | |
| 1864 | def pad( |
| 1865 | self, |
Tested by
no test coverage detected