Method convert_to_tensors

bert/tokenization_utils_base.py:476–519

(self, tensor_type: Union[None, str, TensorType], prepend_batch_axis: bool = False)

        return self._encodings[batch_index].char_to_word(char_index)

    def convert_to_tensors(self, tensor_type: Union[None, str, TensorType], prepend_batch_axis: bool = False):
        if tensor_type is None:
            return self

        # Convert to TensorType
        if not isinstance(tensor_type, TensorType):
            tensor_type = TensorType(tensor_type)

        # Get a function reference for the correct framework
        if tensor_type == TensorType.TENSORFLOW and is_tf_available():
            as_tensor = tf.constant
        elif tensor_type == TensorType.PYTORCH and is_torch_available():
            as_tensor = torch.tensor
        elif tensor_type == TensorType.NUMPY:
            as_tensor = np.asarray
        else:
            raise ImportError(
                "Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format(
                    tensor_type
                )
            )

        # Do the tensor conversion in batch
        for key, value in self.items():
            try:
                if prepend_batch_axis:
                    value = [value]

                tensor = as_tensor(value)

                # at-least2d
                if tensor.ndim > 2:
                    tensor = tensor.squeeze(0)
                elif tensor.ndim < 2:
                    tensor = tensor[None, :]

                self[key] = tensor
            except:  # noqa E722
                raise ValueError(
                    "Unable to create tensor, you should probably activate truncation and/or padding "
                    "with 'padding=True' 'truncation=True' to have batched tensors with the same length."
                )

        return self

    @torch_required
    def to(self, device: str):