(
self, data: Dict[str, Union[str, np.ndarray]]
)
| 2769 | |
| 2770 | @typechecked |
| 2771 | def _text_process( |
| 2772 | self, data: Dict[str, Union[str, np.ndarray]] |
| 2773 | ) -> Dict[str, np.ndarray]: |
| 2774 | |
| 2775 | # NOTE: the order is important |
| 2776 | text_names = [self.text_name, self.text_prev_name, self.text_ctc_name] |
| 2777 | if self.tokenizer is not None: |
| 2778 | for name in text_names: |
| 2779 | if name in data: |
| 2780 | text = data[name] |
| 2781 | |
| 2782 | # Remove prev text by setting it to <na> |
| 2783 | if ( |
| 2784 | self.train |
| 2785 | and name == self.text_prev_name |
| 2786 | and np.random.uniform() > self.text_prev_apply_prob |
| 2787 | ): |
| 2788 | text = self.na_symbol |
| 2789 | |
| 2790 | text = self.text_cleaner(text) |
| 2791 | tokens = self.tokenizer.text2tokens(text) |
| 2792 | text_ints = self.token_id_converter.tokens2ids(tokens) |
| 2793 | text_ints = np.array(text_ints, dtype=np.int64) |
| 2794 | |
| 2795 | # Augment text |
| 2796 | if name == self.text_name: |
| 2797 | # NOTE(yifan): The first token is always space |
| 2798 | # which should be removed. |
| 2799 | # No space is allowed between special tokens. |
| 2800 | # This works for bpe, but maybe not for the other types. |
| 2801 | text_ints = text_ints[1:] |
| 2802 | |
| 2803 | # First two tokens are <lang> and <task> |
| 2804 | # NOTE: must copy the array |
| 2805 | data["prefix"] = copy.deepcopy(text_ints[:2]) |
| 2806 | if self.train and np.random.uniform() > self.lang_apply_prob: |
| 2807 | data["prefix"][0] = self.nolang |
| 2808 | |
| 2809 | elif name == self.text_ctc_name: |
| 2810 | # Add <lang> and <task> to ASR Text as well |
| 2811 | text_ints = np.concatenate( |
| 2812 | [data[self.text_name][:2], text_ints] |
| 2813 | ) |
| 2814 | |
| 2815 | elif name == self.text_prev_name: |
| 2816 | # Remove space before <na> |
| 2817 | if text == self.na_symbol: |
| 2818 | assert len(text_ints) == 2, text_ints |
| 2819 | text_ints = text_ints[1:] |
| 2820 | |
| 2821 | data[name] = text_ints |
| 2822 | |
| 2823 | return data |
| 2824 | |
| 2825 | @typechecked |
| 2826 | def __call__( |
no test coverage detected