| 512 | self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[^\s]+""", re.IGNORECASE) |
| 513 | |
| 514 | def bpe(self, token): |
| 515 | if token in self.cache: |
| 516 | return self.cache[token] |
| 517 | word = tuple(token[:-1]) + ( token[-1] + '</w>',) |
| 518 | pairs = get_pairs(word) |
| 519 | |
| 520 | if not pairs: |
| 521 | return token+'</w>' |
| 522 | |
| 523 | while True: |
| 524 | bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf'))) |
| 525 | if bigram not in self.bpe_ranks: |
| 526 | break |
| 527 | first, second = bigram |
| 528 | new_word = [] |
| 529 | i = 0 |
| 530 | while i < len(word): |
| 531 | try: |
| 532 | j = word.index(first, i) |
| 533 | new_word.extend(word[i:j]) |
| 534 | i = j |
| 535 | except Exception: |
| 536 | new_word.extend(word[i:]) |
| 537 | break |
| 538 | |
| 539 | if word[i] == first and i < len(word)-1 and word[i+1] == second: |
| 540 | new_word.append(first+second) |
| 541 | i += 2 |
| 542 | else: |
| 543 | new_word.append(word[i]) |
| 544 | i += 1 |
| 545 | new_word = tuple(new_word) |
| 546 | word = new_word |
| 547 | if len(word) == 1: |
| 548 | break |
| 549 | pairs = get_pairs(word) |
| 550 | word = ' '.join(word) |
| 551 | self.cache[token] = word |
| 552 | return word |
| 553 | |
| 554 | def encode(self, text): |
| 555 | bpe_tokens = [] |