BatchEncoding hold the output of the encode and batch_encode methods (tokens, attention_masks, etc). This class is derived from a python Dictionary and can be used as a dictionnary. In addition, this class expose utility methods to map from word/char space to token space. A
| 127 | |
| 128 | |
| 129 | class BatchEncoding(UserDict): |
| 130 | """ BatchEncoding hold the output of the encode and batch_encode methods (tokens, attention_masks, etc). |
| 131 | This class is derived from a python Dictionary and can be used as a dictionnary. |
| 132 | In addition, this class expose utility methods to map from word/char space to token space. |
| 133 | |
| 134 | Args: |
| 135 | data (:obj:`dict`): Dictionary of lists/arrays returned by the encode/batch_encode methods ('input_ids', 'attention_mask'...) |
| 136 | encoding (:obj:`EncodingFast`, :obj:`list(EncodingFast)`, `optional`, defaults to :obj:`None`): |
| 137 | If the tokenizer is a fast tokenizer which outputs additional informations like mapping from word/char space to token space |
| 138 | the `EncodingFast` instance or list of instance (for batches) hold these informations. |
| 139 | tensor_type (:obj:`Union[None, str, TensorType]`, `optional`, defaults to :obj:`None`): |
| 140 | You can give a tensor_type here to convert the lists of integers in PyTorch/TF/Numpy Tensors at initialization |
| 141 | prepend_batch_axis (:obj:`bool`, `optional`, defaults to :obj:`False`): |
| 142 | Set to True to add a batch axis when converting in Tensors (see :obj:`tensor_type` above) |
| 143 | """ |
| 144 | |
| 145 | def __init__( |
| 146 | self, |
| 147 | data: Optional[Dict[str, Any]] = None, |
| 148 | encoding: Optional[Union[EncodingFast, Sequence[EncodingFast]]] = None, |
| 149 | tensor_type: Union[None, str, TensorType] = None, |
| 150 | prepend_batch_axis: bool = False, |
| 151 | ): |
| 152 | super().__init__(data) |
| 153 | |
| 154 | if isinstance(encoding, EncodingFast): |
| 155 | encoding = [encoding] |
| 156 | |
| 157 | self._encodings = encoding |
| 158 | |
| 159 | self.convert_to_tensors(tensor_type=tensor_type, prepend_batch_axis=prepend_batch_axis) |
| 160 | |
| 161 | @property |
| 162 | def is_fast(self): |
| 163 | """ |
| 164 | Indicate if this BatchEncoding was generated from the result of a PreTrainedTokenizerFast |
| 165 | Returns: True if generated from subclasses of PreTrainedTokenizerFast, else otherwise |
| 166 | """ |
| 167 | return self._encodings is not None |
| 168 | |
| 169 | def __getitem__(self, item: Union[int, str]) -> EncodingFast: |
| 170 | """ If the key is a string, get the value of the dict associated to `key` ('input_ids', 'attention_mask'...) |
| 171 | If the key is an integer, get the EncodingFast for batch item with index `key` |
| 172 | """ |
| 173 | if isinstance(item, str): |
| 174 | return self.data[item] |
| 175 | elif self._encodings is not None: |
| 176 | return self._encodings[item] |
| 177 | else: |
| 178 | raise KeyError( |
| 179 | "Indexing with integers (to access backend Encoding for a given batch index) " |
| 180 | "is not available when using Python based tokenizers" |
| 181 | ) |
| 182 | |
| 183 | def __getattr__(self, item: str): |
| 184 | try: |
| 185 | return self.data[item] |
| 186 | except KeyError: |
no outgoing calls
no test coverage detected