()
| 456 | # Clip tokenizer, taken from https://github.com/openai/CLIP/blob/main/clip/simple_tokenizer.py (MIT license) |
| 457 | @lru_cache() |
| 458 | def default_bpe(): |
| 459 | return Path(__file__).parent.parent / "weights/bpe_simple_vocab_16e6.txt.gz" |
| 460 | |
| 461 | def get_pairs(word): |
| 462 | """Return set of symbol pairs in a word. |