Helper function to instantiate a tokenizer given common combinations of options.
(tokenizer_type, corpus, model_path=None, vocab_size=None, model_type=None, pad_token=0,
character_coverage=1.0, command_tokens=None, type_tokens=None, fix_command_token=False, **kwargs)
| 33 | |
| 34 | |
| 35 | def make_tokenizer(tokenizer_type, corpus, model_path=None, vocab_size=None, model_type=None, pad_token=0, |
| 36 | character_coverage=1.0, command_tokens=None, type_tokens=None, fix_command_token=False, **kwargs): |
| 37 | """ |
| 38 | Helper function to instantiate a tokenizer given common combinations of options. |
| 39 | """ |
| 40 | tokenizer_class = tokenizer_type |
| 41 | if isinstance(tokenizer_class, str): |
| 42 | tokenizer_class = eval(tokenizer_class) |
| 43 | if tokenizer_class is BertWordPieceTokenizer: |
| 44 | return BertWordPieceTokenizer(model_type, **kwargs) |
| 45 | elif tokenizer_class is GPT2BPETokenizer: |
| 46 | if model_type is None: |
| 47 | model_type = 'gpt2' |
| 48 | return GPT2BPETokenizer(model_type, **kwargs) |
| 49 | elif tokenizer_class is ChineseSPTokenizer: |
| 50 | return ChineseSPTokenizer(fix_command_token=fix_command_token, **kwargs) |
| 51 | text_tokenizer = tokenizer_class(corpus=corpus, vocab_size=vocab_size, model_path=model_path, model_type=model_type, |
| 52 | pad_token=pad_token, character_coverage=character_coverage) |
| 53 | return Tokenizer(text_tokenizer, command_tokens, type_tokens) |
| 54 | |
| 55 | |
| 56 | class Tokenization(object): |
no test coverage detected