(self, vocab_file, do_lower_case=True)
| 111 | """Runs end-to-end tokenziation.""" |
| 112 | |
| 113 | def __init__(self, vocab_file, do_lower_case=True): |
| 114 | self.vocab = load_vocab(vocab_file) |
| 115 | self.inv_vocab = {v: k for k, v in self.vocab.items()} |
| 116 | self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) |
| 117 | self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) |
| 118 | |
| 119 | def tokenize(self, text): |
| 120 | split_tokens = [] |
nothing calls this directly
no test coverage detected