Method pad

bert/tokenization_utils_base.py:1864–1962

Pad a single encoded input or a batch of encoded inputs up to predefined length or to the max sequence length in the batch. Padding side (left/right) padding token ids are defined at the tokenizer level (with ``self.padding_side``, ``self.pad_token_id`` and ``self.pad_token

(
        self,
        encoded_inputs: Union[
            BatchEncoding,
            List[BatchEncoding],
            Dict[str, EncodedInput],
            Dict[str, List[EncodedInput]],
            List[Dict[str, EncodedInput]],
        ],
        padding: Union[bool, str] = True,
        max_length: Optional[int] = None,
        pad_to_multiple_of: Optional[int] = None,
        return_attention_mask: Optional[bool] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        verbose: bool = True,
    )

        raise NotImplementedError

    def pad(
        self,
        encoded_inputs: Union[
            BatchEncoding,
            List[BatchEncoding],
            Dict[str, EncodedInput],
            Dict[str, List[EncodedInput]],
            List[Dict[str, EncodedInput]],
        ],
        padding: Union[bool, str] = True,
        max_length: Optional[int] = None,
        pad_to_multiple_of: Optional[int] = None,
        return_attention_mask: Optional[bool] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        verbose: bool = True,
    ) -> BatchEncoding:
        """ Pad a single encoded input or a batch of encoded inputs up to predefined length or to the max sequence length in the batch.

            Padding side (left/right) padding token ids are defined at the tokenizer level
            (with ``self.padding_side``, ``self.pad_token_id`` and ``self.pad_token_type_id``)

        Args:
            encoded_inputs: Dictionary of tokenized inputs (`Dict[str, List[int]]`) or batch of tokenized inputs.
                Batch of tokenized inputs can be given as dicts of lists or lists of dicts, both work so you can
                use ``tokenizer.pad()`` during pre-processing as well as in a PyTorch Dataloader collate function.
                (`Dict[str, List[List[int]]]` or `List[Dict[str, List[int]]]`).
            padding: Boolean or specific strategy to use for padding.
                Select a strategy to pad the returned sequences (according to the model's padding side and padding index) among:
                - 'longest' (or `True`) Pad to the longest sequence in the batch
                - 'max_length': Pad to the max length (default)
                - 'do_not_pad' (or `False`): Do not pad
            max_length: maximum length of the returned list and optionally padding length (see below).
                Will truncate by taking into account the special tokens.
            pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
                This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
                >= 7.5 (Volta).
            return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics)
            return_tensors (:obj:`str`, `optional`, defaults to :obj:`None`):
                Can be set to 'tf', 'pt' or 'np' to return respectively TensorFlow :obj:`tf.constant`,
                PyTorch :obj:`torch.Tensor` or Numpy :oj: `np.ndarray` instead of a list of python integers.
            verbose (:obj:`bool`, `optional`, defaults to :obj:`True`):
                Set to ``False`` to avoid printing infos and warnings.
        """
        # If we have a list of dicts, let's convert it in a dict of lists
        if isinstance(encoded_inputs, (list, tuple)) and isinstance(encoded_inputs[0], (dict, BatchEncoding)):
            encoded_inputs = {key: [example[key] for example in encoded_inputs] for key in encoded_inputs[0].keys()}

        assert "input_ids" in encoded_inputs, (
            "You should supply an encoding or a list of encodings to this method. "
            "An encoding is the output of one the encoding methods of the tokenizer, i.e. "
            "__call__/encode_plus/batch_encode_plus. "
        )

        if not encoded_inputs["input_ids"]:
            if return_attention_mask:
                encoded_inputs["attention_mask"] = []
            return encoded_inputs