(self, bpe_path: str = default_bpe())
| 61 | |
| 62 | class SimpleTokenizer(object): |
| 63 | def __init__(self, bpe_path: str = default_bpe()): |
| 64 | self.byte_encoder = bytes_to_unicode() |
| 65 | self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} |
| 66 | merges = gzip.open(bpe_path).read().decode("utf-8").split('\n') |
| 67 | merges = merges[1:49152-256-2+1] |
| 68 | merges = [tuple(merge.split()) for merge in merges] |
| 69 | vocab = list(bytes_to_unicode().values()) |
| 70 | vocab = vocab + [v+'</w>' for v in vocab] |
| 71 | for merge in merges: |
| 72 | vocab.append(''.join(merge)) |
| 73 | vocab.extend(['<|startoftext|>', '<|endoftext|>']) |
| 74 | self.encoder = dict(zip(vocab, range(len(vocab)))) |
| 75 | self.decoder = {v: k for k, v in self.encoder.items()} |
| 76 | self.bpe_ranks = dict(zip(merges, range(len(merges)))) |
| 77 | self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'} |
| 78 | self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE) |
| 79 | |
| 80 | def bpe(self, token): |
| 81 | if token in self.cache: |
nothing calls this directly
no test coverage detected