(configs)
| 24 | |
| 25 | |
| 26 | def init_tokenizer(configs) -> BaseTokenizer: |
| 27 | # TODO(xcsong): Forcefully read the 'tokenizer' attribute. |
| 28 | tokenizer_type = configs.get("tokenizer", "char") |
| 29 | if tokenizer_type == "whisper": |
| 30 | tokenizer = WhisperTokenizer( |
| 31 | multilingual=configs['tokenizer_conf']['is_multilingual'], |
| 32 | num_languages=configs['tokenizer_conf']['num_languages']) |
| 33 | elif tokenizer_type == "char": |
| 34 | tokenizer = CharTokenizer( |
| 35 | configs['tokenizer_conf']['symbol_table_path'], |
| 36 | configs['tokenizer_conf']['non_lang_syms_path'], |
| 37 | split_with_space=configs['tokenizer_conf'].get( |
| 38 | 'split_with_space', False), |
| 39 | connect_symbol=configs['tokenizer_conf'].get('connect_symbol', '')) |
| 40 | elif tokenizer_type == "bpe": |
| 41 | tokenizer = BpeTokenizer( |
| 42 | configs['tokenizer_conf']['bpe_path'], |
| 43 | configs['tokenizer_conf']['symbol_table_path'], |
| 44 | configs['tokenizer_conf']['non_lang_syms_path'], |
| 45 | split_with_space=configs['tokenizer_conf'].get( |
| 46 | 'split_with_space', False)) |
| 47 | elif tokenizer_type == 'paraformer': |
| 48 | tokenizer = ParaformerTokenizer( |
| 49 | symbol_table=configs['tokenizer_conf']['symbol_table_path'], |
| 50 | seg_dict=configs['tokenizer_conf']['seg_dict_path']) |
| 51 | elif tokenizer_type == 'huggingface': |
| 52 | tokenizer = HuggingFaceTokenizer( |
| 53 | model=configs['tokenizer_conf']['llm_path']) |
| 54 | else: |
| 55 | raise NotImplementedError |
| 56 | logging.info("use {} tokenizer".format(configs["tokenizer"])) |
| 57 | |
| 58 | return tokenizer |
no test coverage detected