| 2669 | """Preprocessor for OWSM-CTC.""" |
| 2670 | |
| 2671 | def __init__( |
| 2672 | self, |
| 2673 | train: bool, |
| 2674 | token_type: str = None, |
| 2675 | token_list: Union[Path, str, Iterable[str]] = None, |
| 2676 | bpemodel: Union[Path, str, Iterable[str]] = None, |
| 2677 | text_cleaner: Collection[str] = None, |
| 2678 | g2p_type: str = None, |
| 2679 | unk_symbol: str = "<unk>", |
| 2680 | space_symbol: str = "<space>", |
| 2681 | non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, |
| 2682 | delimiter: str = None, |
| 2683 | rir_scp: str = None, |
| 2684 | rir_apply_prob: float = 1.0, |
| 2685 | noise_scp: str = None, |
| 2686 | noise_apply_prob: float = 1.0, |
| 2687 | noise_db_range: str = "3_10", |
| 2688 | short_noise_thres: float = 0.5, |
| 2689 | speech_volume_normalize: float = None, |
| 2690 | speech_name: str = "speech", |
| 2691 | text_name: str = "text", |
| 2692 | text_prev_name: str = "text_prev", |
| 2693 | text_ctc_name: str = "text_ctc", |
| 2694 | fs: int = 16000, |
| 2695 | na_symbol: str = "<na>", # text is not available e.g. for prev or ctc |
| 2696 | speech_length: float = 30, # pad or trim speech to this value in seconds |
| 2697 | speech_init_silence: float = 1.0, # max silence before speech for data aug |
| 2698 | text_prev_apply_prob: float = 0.5, # whether to condition on text_prev |
| 2699 | lang_apply_prob: float = 0.5, # whether to use groundtruth language or unknown |
| 2700 | nolang_symbol: str = "<nolang>", |
| 2701 | ): |
| 2702 | super().__init__( |
| 2703 | train=train, |
| 2704 | token_type=token_type, |
| 2705 | token_list=token_list, |
| 2706 | bpemodel=bpemodel, |
| 2707 | text_cleaner=text_cleaner, |
| 2708 | g2p_type=g2p_type, |
| 2709 | unk_symbol=unk_symbol, |
| 2710 | space_symbol=space_symbol, |
| 2711 | non_linguistic_symbols=non_linguistic_symbols, |
| 2712 | delimiter=delimiter, |
| 2713 | rir_scp=rir_scp, |
| 2714 | rir_apply_prob=rir_apply_prob, |
| 2715 | noise_scp=noise_scp, |
| 2716 | noise_apply_prob=noise_apply_prob, |
| 2717 | noise_db_range=noise_db_range, |
| 2718 | short_noise_thres=short_noise_thres, |
| 2719 | speech_volume_normalize=speech_volume_normalize, |
| 2720 | speech_name=speech_name, |
| 2721 | text_name=text_name, |
| 2722 | fs=fs, |
| 2723 | ) |
| 2724 | self.text_prev_name = text_prev_name |
| 2725 | self.text_ctc_name = text_ctc_name |
| 2726 | self.speech_length = int(speech_length * fs) |
| 2727 | self.speech_init_silence = int(speech_init_silence * fs) |
| 2728 | self.text_prev_apply_prob = text_prev_apply_prob |