Runs end-to-end tokenization.
| 157 | |
| 158 | |
| 159 | class FullTokenizer(object): |
| 160 | """Runs end-to-end tokenization.""" |
| 161 | |
| 162 | def __init__(self, vocab_file, do_lower_case=True): |
| 163 | self.vocab = load_vocab(vocab_file) |
| 164 | self.inv_vocab = {v: k for k, v in self.vocab.items()} |
| 165 | self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) |
| 166 | self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) |
| 167 | |
| 168 | def tokenize(self, text): |
| 169 | split_tokens = [] |
| 170 | for token in self.basic_tokenizer.tokenize(text): |
| 171 | for sub_token in self.wordpiece_tokenizer.tokenize(token): |
| 172 | split_tokens.append(sub_token) |
| 173 | |
| 174 | return split_tokens |
| 175 | |
| 176 | def convert_tokens_to_ids(self, tokens): |
| 177 | return convert_by_vocab(self.vocab, tokens) |
| 178 | |
| 179 | def convert_ids_to_tokens(self, ids): |
| 180 | return convert_by_vocab(self.inv_vocab, ids) |
| 181 | |
| 182 | @staticmethod |
| 183 | def convert_tokens_to_string(tokens, clean_up_tokenization_spaces=True): |
| 184 | """ Converts a sequence of tokens (string) in a single string. """ |
| 185 | |
| 186 | def clean_up_tokenization(out_string): |
| 187 | """ Clean up a list of simple English tokenization artifacts |
| 188 | like spaces before punctuations and abbreviated forms. |
| 189 | """ |
| 190 | out_string = ( |
| 191 | out_string.replace(' .', '.').replace(' ?', '?').replace( |
| 192 | ' !', '!').replace(' ,', ',').replace(" ' ", "'").replace( |
| 193 | " n't", "n't").replace(" 'm", "'m").replace( |
| 194 | " 's", "'s").replace(" 've", |
| 195 | "'ve").replace(" 're", "'re")) |
| 196 | return out_string |
| 197 | |
| 198 | text = ' '.join(tokens).replace(' ##', '').strip() |
| 199 | if clean_up_tokenization_spaces: |
| 200 | clean_text = clean_up_tokenization(text) |
| 201 | return clean_text |
| 202 | else: |
| 203 | return text |
| 204 | |
| 205 | def vocab_size(self): |
| 206 | return len(self.vocab) |
| 207 | |
| 208 | |
| 209 | class BasicTokenizer(object): |