(self, inputs)
| 205 | return example |
| 206 | |
| 207 | def _tokenize(self, inputs) -> Dict[str, tf.Tensor]: |
| 208 | tokenized_inputs = {} |
| 209 | for k, v in inputs.items(): |
| 210 | if k == self._params.src_lang: |
| 211 | tokenized_inputs['inputs'] = self._tokenizer.tokenize(v) |
| 212 | elif k == self._params.tgt_lang: |
| 213 | tokenized_inputs['targets'] = self._tokenizer.tokenize(v) |
| 214 | else: |
| 215 | tokenized_inputs[k] = v |
| 216 | print(tokenized_inputs) |
| 217 | return tokenized_inputs |
| 218 | |
| 219 | def _filter_max_length(self, inputs): |
| 220 | # return tf.constant(True) |