Return set of symbol pairs in a word. Word is represented as tuple of symbols (symbols being variable-length strings).
(word)
| 459 | return Path(__file__).parent.parent / "weights/bpe_simple_vocab_16e6.txt.gz" |
| 460 | |
| 461 | def get_pairs(word): |
| 462 | """Return set of symbol pairs in a word. |
| 463 | Word is represented as tuple of symbols (symbols being variable-length strings). |
| 464 | """ |
| 465 | pairs = set() |
| 466 | prev_char = word[0] |
| 467 | for char in word[1:]: |
| 468 | pairs.add((prev_char, char)) |
| 469 | prev_char = char |
| 470 | return pairs |
| 471 | |
| 472 | def whitespace_clean(text): |
| 473 | text = re.sub(r'\s+', ' ', text) |