(self, bpe_path: str = default_bpe())
| 497 | |
| 498 | class ClipTokenizer: |
| 499 | def __init__(self, bpe_path: str = default_bpe()): |
| 500 | self.byte_encoder = bytes_to_unicode() |
| 501 | merges = gzip.open(bpe_path).read().decode("utf-8").split('\n') |
| 502 | merges = merges[1:49152-256-2+1] |
| 503 | merges = [tuple(merge.split()) for merge in merges] |
| 504 | vocab = list(bytes_to_unicode().values()) |
| 505 | vocab = vocab + [v+'</w>' for v in vocab] |
| 506 | for merge in merges: |
| 507 | vocab.append(''.join(merge)) |
| 508 | vocab.extend(['<|startoftext|>', '<|endoftext|>']) |
| 509 | self.encoder = dict(zip(vocab, range(len(vocab)))) |
| 510 | self.bpe_ranks = dict(zip(merges, range(len(merges)))) |
| 511 | self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'} |
| 512 | self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[^\s]+""", re.IGNORECASE) |
| 513 | |
| 514 | def bpe(self, token): |
| 515 | if token in self.cache: |
nothing calls this directly
no test coverage detected