(self, encoder, bpe_merges)
| 33 | |
| 34 | class Encoder: |
| 35 | def __init__(self, encoder, bpe_merges): |
| 36 | self.encoder = encoder |
| 37 | self.decoder = {v: k for k, v in self.encoder.items()} |
| 38 | self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges)))) |
| 39 | self.cache = {} |
| 40 | self.max_len = 0 |
| 41 | |
| 42 | def bpe(self, token): |
| 43 | if token in self.cache: |
nothing calls this directly
no outgoing calls
no test coverage detected