| 136 | |
| 137 | |
| 138 | class CommonPreprocessor(AbsPreprocessor): |
| 139 | def __init__( |
| 140 | self, |
| 141 | train: bool, |
| 142 | use_lang_prompt: bool = False, |
| 143 | use_nlp_prompt: bool = False, |
| 144 | token_type: Optional[str] = None, |
| 145 | token_list: Union[Path, str, Iterable[str]] = None, |
| 146 | bpemodel: Union[Path, str, Iterable[str]] = None, |
| 147 | text_cleaner: Collection[str] = None, |
| 148 | g2p_type: Optional[str] = None, |
| 149 | unk_symbol: str = "<unk>", |
| 150 | space_symbol: str = "<space>", |
| 151 | non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, |
| 152 | delimiter: Optional[str] = None, |
| 153 | force_single_channel: bool = False, |
| 154 | rir_scp: Optional[str] = None, |
| 155 | rir_apply_prob: float = 1.0, |
| 156 | noise_scp: Optional[str] = None, |
| 157 | noise_apply_prob: float = 1.0, |
| 158 | noise_db_range: str = "3_10", |
| 159 | short_noise_thres: float = 0.5, |
| 160 | aux_task_names: Collection[str] = None, |
| 161 | speech_volume_normalize: float = None, |
| 162 | speech_name: str = "speech", |
| 163 | text_name: str = "text", |
| 164 | fs: int = 0, |
| 165 | nonsplit_symbol: Iterable[str] = None, |
| 166 | data_aug_effects: List = None, |
| 167 | data_aug_num: List[int] = [1, 1], |
| 168 | data_aug_prob: float = 0.0, |
| 169 | # for padding of chunk iterator, working when > 0 |
| 170 | min_sample_size: int = -1, |
| 171 | audio_pad_value: Union[float, int] = 0.0, |
| 172 | # only use for whisper |
| 173 | whisper_language: Optional[str] = None, |
| 174 | whisper_task: Optional[str] = None, |
| 175 | ): |
| 176 | super().__init__(train) |
| 177 | self.train = train |
| 178 | self.speech_name = speech_name |
| 179 | self.text_name = text_name |
| 180 | self.speech_volume_normalize = speech_volume_normalize |
| 181 | self.force_single_channel = force_single_channel |
| 182 | self.rir_apply_prob = rir_apply_prob |
| 183 | self.noise_apply_prob = noise_apply_prob |
| 184 | self.short_noise_thres = short_noise_thres |
| 185 | self.aux_task_names = aux_task_names |
| 186 | self.use_lang_prompt = use_lang_prompt |
| 187 | self.use_nlp_prompt = use_nlp_prompt |
| 188 | |
| 189 | if token_type is not None: |
| 190 | if token_list is None: |
| 191 | raise ValueError("token_list is required if token_type is not None") |
| 192 | self.text_cleaner = TextCleaner(text_cleaner) |
| 193 | |
| 194 | self.tokenizer = build_tokenizer( |
| 195 | token_type=token_type, |
no outgoing calls
no test coverage detected
searching dependent graphs…