Original GPT2 BPE tokenizer.
| 180 | |
| 181 | |
| 182 | class _GPT2BPETokenizer(AbstractTokenizer): |
| 183 | """Original GPT2 BPE tokenizer.""" |
| 184 | |
| 185 | def __init__(self, vocab_file, merge_file): |
| 186 | name = "GPT2 BPE" |
| 187 | super().__init__(name) |
| 188 | |
| 189 | self.tokenizer = GPT2Tokenizer( |
| 190 | vocab_file, merge_file, errors="replace", special_tokens=[], max_len=None |
| 191 | ) |
| 192 | self.eod_id = self.tokenizer.encoder["<|endoftext|>"] |
| 193 | |
| 194 | @property |
| 195 | def vocab_size(self): |
| 196 | return len(self.tokenizer.encoder) |
| 197 | |
| 198 | @property |
| 199 | def vocab(self): |
| 200 | return self.tokenizer.encoder |
| 201 | |
| 202 | @property |
| 203 | def inv_vocab(self): |
| 204 | return self.tokenizer.decoder |
| 205 | |
| 206 | def tokenize(self, text): |
| 207 | return self.tokenizer.encode(text) |
| 208 | |
| 209 | def detokenize(self, token_ids): |
| 210 | return self.tokenizer.decode(token_ids) |
| 211 | |
| 212 | @property |
| 213 | def eod(self): |
| 214 | return self.eod_id |
| 215 | |
| 216 | |
| 217 | class HgfTokenizerWrapper(AbstractTokenizer): |