Pad encoded inputs (on left/right and up to predefined legnth or max length in the batch) Args: encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). max_length: maximum length of the returned list and optiona
(
self,
encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
max_length: Optional[int] = None,
padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
pad_to_multiple_of: Optional[int] = None,
return_attention_mask: Optional[bool] = None,
)
| 2181 | return (ids, pair_ids, overflowing_tokens) |
| 2182 | |
| 2183 | def _pad( |
| 2184 | self, |
| 2185 | encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], |
| 2186 | max_length: Optional[int] = None, |
| 2187 | padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, |
| 2188 | pad_to_multiple_of: Optional[int] = None, |
| 2189 | return_attention_mask: Optional[bool] = None, |
| 2190 | ) -> dict: |
| 2191 | """ Pad encoded inputs (on left/right and up to predefined legnth or max length in the batch) |
| 2192 | |
| 2193 | Args: |
| 2194 | encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). |
| 2195 | max_length: maximum length of the returned list and optionally padding length (see below). |
| 2196 | Will truncate by taking into account the special tokens. |
| 2197 | padding_strategy: PaddingStrategy to use for padding. |
| 2198 | - PaddingStrategy.LONGEST Pad to the longest sequence in the batch |
| 2199 | - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) |
| 2200 | - PaddingStrategy.DO_NOT_PAD: Do not pad |
| 2201 | The tokenizer padding sides are defined in self.padding_side: |
| 2202 | - 'left': pads on the left of the sequences |
| 2203 | - 'right': pads on the right of the sequences |
| 2204 | pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. |
| 2205 | This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability |
| 2206 | >= 7.5 (Volta). |
| 2207 | return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics) |
| 2208 | """ |
| 2209 | # Load from model defaults |
| 2210 | if return_attention_mask is None: |
| 2211 | return_attention_mask = "attention_mask" in self.model_input_names |
| 2212 | |
| 2213 | if padding_strategy == PaddingStrategy.LONGEST: |
| 2214 | max_length = len(encoded_inputs["input_ids"]) |
| 2215 | |
| 2216 | if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0): |
| 2217 | max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of |
| 2218 | |
| 2219 | needs_to_be_padded = ( |
| 2220 | padding_strategy != PaddingStrategy.DO_NOT_PAD and len(encoded_inputs["input_ids"]) != max_length |
| 2221 | ) |
| 2222 | |
| 2223 | if needs_to_be_padded: |
| 2224 | difference = max_length - len(encoded_inputs["input_ids"]) |
| 2225 | if self.padding_side == "right": |
| 2226 | if return_attention_mask: |
| 2227 | encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) + [0] * difference |
| 2228 | if "token_type_ids" in encoded_inputs: |
| 2229 | encoded_inputs["token_type_ids"] = ( |
| 2230 | encoded_inputs["token_type_ids"] + [self.pad_token_type_id] * difference |
| 2231 | ) |
| 2232 | if "special_tokens_mask" in encoded_inputs: |
| 2233 | encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference |
| 2234 | encoded_inputs["input_ids"] = encoded_inputs["input_ids"] + [self.pad_token_id] * difference |
| 2235 | elif self.padding_side == "left": |
| 2236 | if return_attention_mask: |
| 2237 | encoded_inputs["attention_mask"] = [0] * difference + [1] * len(encoded_inputs["input_ids"]) |
| 2238 | if "token_type_ids" in encoded_inputs: |
| 2239 | encoded_inputs["token_type_ids"] = [self.pad_token_type_id] * difference + encoded_inputs[ |
| 2240 | "token_type_ids" |