(
self, data: Dict[str, Union[str, np.ndarray]]
)
| 732 | ) |
| 733 | |
| 734 | def _text_process( |
| 735 | self, data: Dict[str, Union[str, np.ndarray]] |
| 736 | ) -> Dict[str, np.ndarray]: |
| 737 | for text_n in self.text_name: |
| 738 | if text_n in data and self.tokenizer is not None: |
| 739 | text = data[text_n] |
| 740 | text = self.text_cleaner(text) |
| 741 | tokens = self.tokenizer.text2tokens(text) |
| 742 | text_ints = self.token_id_converter.tokens2ids(tokens) |
| 743 | data[text_n] = np.array(text_ints, dtype=np.int64) |
| 744 | if self.aux_task_names is not None and self.tokenizer is not None: |
| 745 | for name in self.aux_task_names: |
| 746 | if name in data: |
| 747 | text = data[name] |
| 748 | text = self.text_cleaner(text) |
| 749 | tokens = self.tokenizer.text2tokens(text) |
| 750 | text_ints = self.token_id_converter.tokens2ids(tokens) |
| 751 | data[name] = np.array(text_ints, dtype=np.int64) |
| 752 | return data |
| 753 | |
| 754 | @typechecked |
| 755 | def __call__( |
no test coverage detected