Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It adds special tokens, truncates sequences if overflowing while taking into account the special tokens and manages a moving window (with user defined stride) for overflowing t
(
self,
ids: List[int],
pair_ids: Optional[List[int]] = None,
add_special_tokens: bool = True,
padding: Union[bool, str] = False,
truncation: Union[bool, str] = False,
max_length: Optional[int] = None,
stride: int = 0,
pad_to_multiple_of: Optional[int] = None,
return_tensors: Optional[Union[str, TensorType]] = None,
return_token_type_ids: Optional[bool] = None,
return_attention_mask: Optional[bool] = None,
return_overflowing_tokens: bool = False,
return_special_tokens_mask: bool = False,
return_offsets_mapping: bool = False,
return_length: bool = False,
verbose: bool = True,
prepend_batch_axis: bool = False,
**kwargs
)
| 1977 | |
| 1978 | @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING) |
| 1979 | def prepare_for_model( |
| 1980 | self, |
| 1981 | ids: List[int], |
| 1982 | pair_ids: Optional[List[int]] = None, |
| 1983 | add_special_tokens: bool = True, |
| 1984 | padding: Union[bool, str] = False, |
| 1985 | truncation: Union[bool, str] = False, |
| 1986 | max_length: Optional[int] = None, |
| 1987 | stride: int = 0, |
| 1988 | pad_to_multiple_of: Optional[int] = None, |
| 1989 | return_tensors: Optional[Union[str, TensorType]] = None, |
| 1990 | return_token_type_ids: Optional[bool] = None, |
| 1991 | return_attention_mask: Optional[bool] = None, |
| 1992 | return_overflowing_tokens: bool = False, |
| 1993 | return_special_tokens_mask: bool = False, |
| 1994 | return_offsets_mapping: bool = False, |
| 1995 | return_length: bool = False, |
| 1996 | verbose: bool = True, |
| 1997 | prepend_batch_axis: bool = False, |
| 1998 | **kwargs |
| 1999 | ) -> BatchEncoding: |
| 2000 | """ Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. |
| 2001 | It adds special tokens, truncates sequences if overflowing while taking into account the special tokens and |
| 2002 | manages a moving window (with user defined stride) for overflowing tokens |
| 2003 | |
| 2004 | Args: |
| 2005 | ids: list of tokenized input ids. Can be obtained from a string by chaining the |
| 2006 | `tokenize` and `convert_tokens_to_ids` methods. |
| 2007 | pair_ids: Optional second list of input ids. Can be obtained from a string by chaining the |
| 2008 | `tokenize` and `convert_tokens_to_ids` methods. |
| 2009 | """ |
| 2010 | |
| 2011 | if "return_lengths" in kwargs: |
| 2012 | if verbose: |
| 2013 | warnings.warn( |
| 2014 | "The PreTrainedTokenizerBase.prepare_for_model `return_lengths` parameter is deprecated. " |
| 2015 | "Please use `return_length` instead.", |
| 2016 | FutureWarning, |
| 2017 | ) |
| 2018 | return_length = kwargs["return_lengths"] |
| 2019 | |
| 2020 | # Backward compatibility for 'truncation_strategy', 'pad_to_max_length' |
| 2021 | padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies( |
| 2022 | padding=padding, |
| 2023 | truncation=truncation, |
| 2024 | max_length=max_length, |
| 2025 | pad_to_multiple_of=pad_to_multiple_of, |
| 2026 | verbose=verbose, |
| 2027 | **kwargs, |
| 2028 | ) |
| 2029 | |
| 2030 | pair = bool(pair_ids is not None) |
| 2031 | len_ids = len(ids) |
| 2032 | len_pair_ids = len(pair_ids) if pair else 0 |
| 2033 | |
| 2034 | # Load from model defaults |
| 2035 | if return_token_type_ids is None: |
| 2036 | return_token_type_ids = "token_type_ids" in self.model_input_names |
no test coverage detected