| 173 | # this has poisoned all HF tokenizer configs that use ByteLevel decoder/preprocessor |
| 174 | # as a result we get crazy UTF-8-as-bytes-as-UTF8 in the tokenizer data that we need to convert back |
| 175 | def gpt2_bytes_to_unicode(): |
| 176 | bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) |
| 177 | cs = bs[:] |
| 178 | n = 0 |
| 179 | for b in range(2**8): |
| 180 | if b not in bs: |
| 181 | bs.append(b) |
| 182 | cs.append(2**8+n) |
| 183 | n += 1 |
| 184 | cs = [chr(n) for n in cs] |
| 185 | return dict(zip(bs, cs)) |
| 186 | |
| 187 | def load_tokens(tokenizer_path, vocab_size): |
| 188 | tokens = [""] * vocab_size |