(
self,
train: bool,
token_type: List[str] = [None],
token_list: List[Union[Path, str, Iterable[str]]] = [None],
bpemodel: List[Union[Path, str, Iterable[str]]] = [None],
text_cleaner: Collection[str] = None,
g2p_type: Union[List[str], str] = None,
unk_symbol: str = "<unk>",
space_symbol: str = "<space>",
non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,
delimiter: Optional[str] = None,
rir_scp: Optional[str] = None,
rir_apply_prob: float = 1.0,
noise_scp: Optional[str] = None,
noise_apply_prob: float = 1.0,
noise_db_range: str = "3_10",
short_noise_thres: float = 0.5,
speech_volume_normalize: float = None,
speech_name: str = "speech",
text_name: List[str] = ["text"],
tokenizer_encode_conf: List[Dict] = [dict(), dict()],
fs: int = 0,
data_aug_effects: List = None,
data_aug_num: List[int] = [1, 1],
data_aug_prob: float = 0.0,
# only use for whisper
whisper_language: List[str] = None,
whisper_task: Optional[str] = None,
)
| 763 | |
| 764 | class MutliTokenizerCommonPreprocessor(CommonPreprocessor): |
| 765 | def __init__( |
| 766 | self, |
| 767 | train: bool, |
| 768 | token_type: List[str] = [None], |
| 769 | token_list: List[Union[Path, str, Iterable[str]]] = [None], |
| 770 | bpemodel: List[Union[Path, str, Iterable[str]]] = [None], |
| 771 | text_cleaner: Collection[str] = None, |
| 772 | g2p_type: Union[List[str], str] = None, |
| 773 | unk_symbol: str = "<unk>", |
| 774 | space_symbol: str = "<space>", |
| 775 | non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, |
| 776 | delimiter: Optional[str] = None, |
| 777 | rir_scp: Optional[str] = None, |
| 778 | rir_apply_prob: float = 1.0, |
| 779 | noise_scp: Optional[str] = None, |
| 780 | noise_apply_prob: float = 1.0, |
| 781 | noise_db_range: str = "3_10", |
| 782 | short_noise_thres: float = 0.5, |
| 783 | speech_volume_normalize: float = None, |
| 784 | speech_name: str = "speech", |
| 785 | text_name: List[str] = ["text"], |
| 786 | tokenizer_encode_conf: List[Dict] = [dict(), dict()], |
| 787 | fs: int = 0, |
| 788 | data_aug_effects: List = None, |
| 789 | data_aug_num: List[int] = [1, 1], |
| 790 | data_aug_prob: float = 0.0, |
| 791 | # only use for whisper |
| 792 | whisper_language: List[str] = None, |
| 793 | whisper_task: Optional[str] = None, |
| 794 | ): |
| 795 | # TODO(jiatong): sync with Kamo and Jing on interface for preprocessor |
| 796 | super().__init__( |
| 797 | train=train, |
| 798 | token_type=token_type[0], |
| 799 | token_list=token_list[0], |
| 800 | bpemodel=bpemodel[0], |
| 801 | text_cleaner=text_cleaner, |
| 802 | g2p_type=( |
| 803 | g2p_type[0] |
| 804 | if type(g2p_type) is not str and g2p_type is not None |
| 805 | else g2p_type |
| 806 | ), |
| 807 | unk_symbol=unk_symbol, |
| 808 | space_symbol=space_symbol, |
| 809 | non_linguistic_symbols=non_linguistic_symbols, |
| 810 | delimiter=delimiter, |
| 811 | speech_name=speech_name, |
| 812 | text_name=text_name[0], |
| 813 | rir_scp=rir_scp, |
| 814 | rir_apply_prob=rir_apply_prob, |
| 815 | noise_scp=noise_scp, |
| 816 | noise_apply_prob=noise_apply_prob, |
| 817 | noise_db_range=noise_db_range, |
| 818 | short_noise_thres=short_noise_thres, |
| 819 | speech_volume_normalize=speech_volume_normalize, |
| 820 | fs=fs, |
| 821 | data_aug_effects=data_aug_effects, |
| 822 | data_aug_num=data_aug_num, |
nothing calls this directly
no test coverage detected