| 194 | return dict(self.encoder, **self.added_tokens_encoder) |
| 195 | |
| 196 | def bpe(self, token): |
| 197 | if token in self.cache: |
| 198 | return self.cache[token] |
| 199 | word = tuple(token) |
| 200 | pairs = get_pairs(word) |
| 201 | |
| 202 | if not pairs: |
| 203 | return token |
| 204 | |
| 205 | while True: |
| 206 | bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))) |
| 207 | if bigram not in self.bpe_ranks: |
| 208 | break |
| 209 | first, second = bigram |
| 210 | new_word = [] |
| 211 | i = 0 |
| 212 | while i < len(word): |
| 213 | try: |
| 214 | j = word.index(first, i) |
| 215 | except ValueError: |
| 216 | new_word.extend(word[i:]) |
| 217 | break |
| 218 | else: |
| 219 | new_word.extend(word[i:j]) |
| 220 | i = j |
| 221 | |
| 222 | if word[i] == first and i < len(word) - 1 and word[i + 1] == second: |
| 223 | new_word.append(first + second) |
| 224 | i += 2 |
| 225 | else: |
| 226 | new_word.append(word[i]) |
| 227 | i += 1 |
| 228 | new_word = tuple(new_word) |
| 229 | word = new_word |
| 230 | if len(word) == 1: |
| 231 | break |
| 232 | else: |
| 233 | pairs = get_pairs(word) |
| 234 | word = " ".join(word) |
| 235 | self.cache[token] = word |
| 236 | return word |
| 237 | |
| 238 | def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): |
| 239 | if self.add_bos_token: |