Return set of symbol pairs in a word. Word is represented as tuple of symbols (symbols being variable-length strings).
(word)
| 79 | |
| 80 | |
| 81 | def get_pairs(word): |
| 82 | """Return set of symbol pairs in a word. |
| 83 | |
| 84 | Word is represented as tuple of symbols (symbols being variable-length strings). |
| 85 | """ |
| 86 | pairs = set() |
| 87 | prev_char = word[0] |
| 88 | for char in word[1:]: |
| 89 | pairs.add((prev_char, char)) |
| 90 | prev_char = char |
| 91 | return pairs |
| 92 | |
| 93 | |
| 94 | class GPT2Tokenizer(object): |