| 55 | self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""") |
| 56 | |
| 57 | def bpe(self, token): |
| 58 | if token in self.cache: |
| 59 | return self.cache[token] |
| 60 | word = tuple(token) |
| 61 | pairs = get_pairs(word) |
| 62 | |
| 63 | if not pairs: |
| 64 | return token |
| 65 | |
| 66 | while True: |
| 67 | bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf'))) |
| 68 | if bigram not in self.bpe_ranks: |
| 69 | break |
| 70 | first, second = bigram |
| 71 | new_word = [] |
| 72 | i = 0 |
| 73 | while i < len(word): |
| 74 | try: |
| 75 | j = word.index(first, i) |
| 76 | new_word.extend(word[i:j]) |
| 77 | i = j |
| 78 | except: |
| 79 | new_word.extend(word[i:]) |
| 80 | break |
| 81 | |
| 82 | if word[i] == first and i < len(word)-1 and word[i+1] == second: |
| 83 | new_word.append(first+second) |
| 84 | i += 2 |
| 85 | else: |
| 86 | new_word.append(word[i]) |
| 87 | i += 1 |
| 88 | new_word = tuple(new_word) |
| 89 | word = new_word |
| 90 | if len(word) == 1: |
| 91 | break |
| 92 | else: |
| 93 | pairs = get_pairs(word) |
| 94 | word = ' '.join(word) |
| 95 | self.cache[token] = word |
| 96 | |
| 97 | |
| 98 | return word |
| 99 | |
| 100 | def encode(self, text): |
| 101 | bpe_tokens = [] |