Return set of symbol pairs in a word. Word is represented as tuple of symbols (symbols being variable-length strings).
(word)
| 84 | |
| 85 | |
| 86 | def get_pairs(word): |
| 87 | """ |
| 88 | Return set of symbol pairs in a word. |
| 89 | |
| 90 | Word is represented as tuple of symbols (symbols being variable-length strings). |
| 91 | """ |
| 92 | pairs = set() |
| 93 | prev_char = word[0] |
| 94 | for char in word[1:]: |
| 95 | pairs.add((prev_char, char)) |
| 96 | prev_char = char |
| 97 | return pairs |
| 98 | |
| 99 | |
| 100 | class MossTokenizer(PreTrainedTokenizer): |