(
self,
train: bool,
use_lang_prompt: bool = False,
use_nlp_prompt: bool = False,
token_type: Optional[str] = None,
token_list: Union[Path, str, Iterable[str]] = None,
bpemodel: Union[Path, str, Iterable[str]] = None,
text_cleaner: Collection[str] = None,
g2p_type: Optional[str] = None,
unk_symbol: str = "<unk>",
space_symbol: str = "<space>",
non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,
delimiter: Optional[str] = None,
force_single_channel: bool = False,
rir_scp: Optional[str] = None,
rir_apply_prob: float = 1.0,
noise_scp: Optional[str] = None,
noise_apply_prob: float = 1.0,
noise_db_range: str = "3_10",
short_noise_thres: float = 0.5,
aux_task_names: Collection[str] = None,
speech_volume_normalize: float = None,
speech_name: str = "speech",
text_name: str = "text",
fs: int = 0,
nonsplit_symbol: Iterable[str] = None,
data_aug_effects: List = None,
data_aug_num: List[int] = [1, 1],
data_aug_prob: float = 0.0,
# for padding of chunk iterator, working when > 0
min_sample_size: int = -1,
audio_pad_value: Union[float, int] = 0.0,
# only use for whisper
whisper_language: Optional[str] = None,
whisper_task: Optional[str] = None,
)
| 137 | |
| 138 | class CommonPreprocessor(AbsPreprocessor): |
| 139 | def __init__( |
| 140 | self, |
| 141 | train: bool, |
| 142 | use_lang_prompt: bool = False, |
| 143 | use_nlp_prompt: bool = False, |
| 144 | token_type: Optional[str] = None, |
| 145 | token_list: Union[Path, str, Iterable[str]] = None, |
| 146 | bpemodel: Union[Path, str, Iterable[str]] = None, |
| 147 | text_cleaner: Collection[str] = None, |
| 148 | g2p_type: Optional[str] = None, |
| 149 | unk_symbol: str = "<unk>", |
| 150 | space_symbol: str = "<space>", |
| 151 | non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, |
| 152 | delimiter: Optional[str] = None, |
| 153 | force_single_channel: bool = False, |
| 154 | rir_scp: Optional[str] = None, |
| 155 | rir_apply_prob: float = 1.0, |
| 156 | noise_scp: Optional[str] = None, |
| 157 | noise_apply_prob: float = 1.0, |
| 158 | noise_db_range: str = "3_10", |
| 159 | short_noise_thres: float = 0.5, |
| 160 | aux_task_names: Collection[str] = None, |
| 161 | speech_volume_normalize: float = None, |
| 162 | speech_name: str = "speech", |
| 163 | text_name: str = "text", |
| 164 | fs: int = 0, |
| 165 | nonsplit_symbol: Iterable[str] = None, |
| 166 | data_aug_effects: List = None, |
| 167 | data_aug_num: List[int] = [1, 1], |
| 168 | data_aug_prob: float = 0.0, |
| 169 | # for padding of chunk iterator, working when > 0 |
| 170 | min_sample_size: int = -1, |
| 171 | audio_pad_value: Union[float, int] = 0.0, |
| 172 | # only use for whisper |
| 173 | whisper_language: Optional[str] = None, |
| 174 | whisper_task: Optional[str] = None, |
| 175 | ): |
| 176 | super().__init__(train) |
| 177 | self.train = train |
| 178 | self.speech_name = speech_name |
| 179 | self.text_name = text_name |
| 180 | self.speech_volume_normalize = speech_volume_normalize |
| 181 | self.force_single_channel = force_single_channel |
| 182 | self.rir_apply_prob = rir_apply_prob |
| 183 | self.noise_apply_prob = noise_apply_prob |
| 184 | self.short_noise_thres = short_noise_thres |
| 185 | self.aux_task_names = aux_task_names |
| 186 | self.use_lang_prompt = use_lang_prompt |
| 187 | self.use_nlp_prompt = use_nlp_prompt |
| 188 | |
| 189 | if token_type is not None: |
| 190 | if token_list is None: |
| 191 | raise ValueError("token_list is required if token_type is not None") |
| 192 | self.text_cleaner = TextCleaner(text_cleaner) |
| 193 | |
| 194 | self.tokenizer = build_tokenizer( |
| 195 | token_type=token_type, |
| 196 | bpemodel=bpemodel, |
nothing calls this directly
no test coverage detected