| 621 | self.transcript_token_id_converter = None |
| 622 | |
| 623 | def _text_process( |
| 624 | self, data: Dict[str, Union[str, np.ndarray]] |
| 625 | ) -> Dict[str, np.ndarray]: |
| 626 | if self.text_name in data and self.tokenizer is not None: |
| 627 | text = data[self.text_name] |
| 628 | text = self.text_cleaner(text) |
| 629 | tokens = self.tokenizer.text2tokens(text) |
| 630 | text_ints = self.token_id_converter.tokens2ids(tokens) |
| 631 | data[self.text_name] = np.array(text_ints, dtype=np.int64) |
| 632 | if "transcript" in data and self.tokenizer is not None: |
| 633 | text = data["transcript"] |
| 634 | text = self.text_cleaner(text) |
| 635 | tokens = self.transcript_tokenizer.text2tokens(text) |
| 636 | text_ints = self.transcript_token_id_converter.tokens2ids(tokens) |
| 637 | data["transcript"] = np.array(text_ints, dtype=np.int64) |
| 638 | return data |
| 639 | |
| 640 | |
| 641 | class CommonPreprocessor_multi(CommonPreprocessor): |