Converts a string in a sequence of ids (integer), using the tokenizer and vocabulary. Same as doing ``self.convert_tokens_to_ids(self.tokenize(text))``. Args: text (:obj:`str`, :obj:`List[str]` or :obj:`List[int]`): The first sequence to be enco
(
self,
text: Union[TextInput, PreTokenizedInput, EncodedInput],
text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None,
add_special_tokens: bool = True,
padding: Union[bool, str] = False,
truncation: Union[bool, str] = False,
max_length: Optional[int] = None,
stride: int = 0,
return_tensors: Optional[Union[str, TensorType]] = None,
**kwargs
)
| 1387 | """, |
| 1388 | ) |
| 1389 | def encode( |
| 1390 | self, |
| 1391 | text: Union[TextInput, PreTokenizedInput, EncodedInput], |
| 1392 | text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None, |
| 1393 | add_special_tokens: bool = True, |
| 1394 | padding: Union[bool, str] = False, |
| 1395 | truncation: Union[bool, str] = False, |
| 1396 | max_length: Optional[int] = None, |
| 1397 | stride: int = 0, |
| 1398 | return_tensors: Optional[Union[str, TensorType]] = None, |
| 1399 | **kwargs |
| 1400 | ): |
| 1401 | """ |
| 1402 | Converts a string in a sequence of ids (integer), using the tokenizer and vocabulary. |
| 1403 | |
| 1404 | Same as doing ``self.convert_tokens_to_ids(self.tokenize(text))``. |
| 1405 | |
| 1406 | Args: |
| 1407 | text (:obj:`str`, :obj:`List[str]` or :obj:`List[int]`): |
| 1408 | The first sequence to be encoded. This can be a string, a list of strings (tokenized string using |
| 1409 | the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids` |
| 1410 | method) |
| 1411 | text_pair (:obj:`str`, :obj:`List[str]` or :obj:`List[int]`, `optional`, defaults to :obj:`None`): |
| 1412 | Optional second sequence to be encoded. This can be a string, a list of strings (tokenized |
| 1413 | string using the `tokenize` method) or a list of integers (tokenized string ids using the |
| 1414 | `convert_tokens_to_ids` method) |
| 1415 | """ |
| 1416 | encoded_inputs = self.encode_plus( |
| 1417 | text, |
| 1418 | text_pair=text_pair, |
| 1419 | add_special_tokens=add_special_tokens, |
| 1420 | padding=padding, |
| 1421 | truncation=truncation, |
| 1422 | max_length=max_length, |
| 1423 | stride=stride, |
| 1424 | return_tensors=return_tensors, |
| 1425 | **kwargs, |
| 1426 | ) |
| 1427 | |
| 1428 | return encoded_inputs["input_ids"] |
| 1429 | |
| 1430 | def num_special_tokens_to_add(self, pair: bool = False) -> int: |
| 1431 | raise NotImplementedError |
no test coverage detected