Runs end-to-end tokenziation.
| 161 | |
| 162 | |
| 163 | class FullTokenizer(object): |
| 164 | """Runs end-to-end tokenziation.""" |
| 165 | |
| 166 | def __init__(self, vocab_file, do_lower_case=True): |
| 167 | self.vocab = load_vocab(vocab_file) |
| 168 | self.inv_vocab = {v: k for k, v in self.vocab.items()} |
| 169 | self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) |
| 170 | self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) |
| 171 | |
| 172 | def tokenize(self, text): |
| 173 | split_tokens = [] |
| 174 | for token in self.basic_tokenizer.tokenize(text): |
| 175 | for sub_token in self.wordpiece_tokenizer.tokenize(token): |
| 176 | split_tokens.append(sub_token) |
| 177 | |
| 178 | return split_tokens |
| 179 | |
| 180 | def convert_tokens_to_ids(self, tokens): |
| 181 | return convert_by_vocab(self.vocab, tokens) |
| 182 | |
| 183 | def convert_ids_to_tokens(self, ids): |
| 184 | return convert_by_vocab(self.inv_vocab, ids) |
| 185 | |
| 186 | |
| 187 | class BasicTokenizer(object): |