(
self,
train: bool,
token_type: Optional[str] = None,
token_list: Union[Path, str, Iterable[str]] = None,
transcript_token_list: Union[Path, str, Iterable[str]] = None,
bpemodel: Union[Path, str, Iterable[str]] = None,
text_cleaner: Collection[str] = None,
g2p_type: Optional[str] = None,
unk_symbol: str = "<unk>",
space_symbol: str = "<space>",
non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,
delimiter: Optional[str] = None,
rir_scp: Optional[str] = None,
rir_apply_prob: float = 1.0,
noise_scp: Optional[str] = None,
noise_apply_prob: float = 1.0,
noise_db_range: str = "3_10",
short_noise_thres: float = 0.5,
speech_volume_normalize: float = None,
speech_name: str = "speech",
text_name: str = "text",
fs: int = 0,
data_aug_effects: List = None,
data_aug_num: List[int] = [1, 1],
data_aug_prob: float = 0.0,
)
| 551 | |
| 552 | class SLUPreprocessor(CommonPreprocessor): |
| 553 | def __init__( |
| 554 | self, |
| 555 | train: bool, |
| 556 | token_type: Optional[str] = None, |
| 557 | token_list: Union[Path, str, Iterable[str]] = None, |
| 558 | transcript_token_list: Union[Path, str, Iterable[str]] = None, |
| 559 | bpemodel: Union[Path, str, Iterable[str]] = None, |
| 560 | text_cleaner: Collection[str] = None, |
| 561 | g2p_type: Optional[str] = None, |
| 562 | unk_symbol: str = "<unk>", |
| 563 | space_symbol: str = "<space>", |
| 564 | non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, |
| 565 | delimiter: Optional[str] = None, |
| 566 | rir_scp: Optional[str] = None, |
| 567 | rir_apply_prob: float = 1.0, |
| 568 | noise_scp: Optional[str] = None, |
| 569 | noise_apply_prob: float = 1.0, |
| 570 | noise_db_range: str = "3_10", |
| 571 | short_noise_thres: float = 0.5, |
| 572 | speech_volume_normalize: float = None, |
| 573 | speech_name: str = "speech", |
| 574 | text_name: str = "text", |
| 575 | fs: int = 0, |
| 576 | data_aug_effects: List = None, |
| 577 | data_aug_num: List[int] = [1, 1], |
| 578 | data_aug_prob: float = 0.0, |
| 579 | ): |
| 580 | super().__init__( |
| 581 | train=train, |
| 582 | token_type=token_type, |
| 583 | token_list=token_list, |
| 584 | bpemodel=bpemodel, |
| 585 | text_cleaner=text_cleaner, |
| 586 | g2p_type=g2p_type, |
| 587 | unk_symbol=unk_symbol, |
| 588 | space_symbol=space_symbol, |
| 589 | non_linguistic_symbols=non_linguistic_symbols, |
| 590 | delimiter=delimiter, |
| 591 | rir_scp=rir_scp, |
| 592 | rir_apply_prob=rir_apply_prob, |
| 593 | noise_scp=noise_scp, |
| 594 | noise_apply_prob=noise_apply_prob, |
| 595 | noise_db_range=noise_db_range, |
| 596 | short_noise_thres=short_noise_thres, |
| 597 | speech_volume_normalize=speech_volume_normalize, |
| 598 | speech_name=speech_name, |
| 599 | text_name=text_name, |
| 600 | fs=fs, |
| 601 | data_aug_effects=data_aug_effects, |
| 602 | data_aug_num=data_aug_num, |
| 603 | data_aug_prob=data_aug_prob, |
| 604 | ) |
| 605 | if transcript_token_list is not None: |
| 606 | print("using transcript") |
| 607 | self.transcript_tokenizer = build_tokenizer( |
| 608 | token_type="word", |
| 609 | bpemodel=bpemodel, |
| 610 | delimiter=delimiter, |
nothing calls this directly
no test coverage detected