(self, tokens)
| 83 | return [self.encoder.get(token, 1) for token in self.tokenize(text)] |
| 84 | |
| 85 | def decode(self, tokens): |
| 86 | text = ''.join([self.decoder[token] for token in tokens]) |
| 87 | return text |
| 88 | |
| 89 | def tokenize(self, text): |
| 90 | bpe_tokens = [] |
no outgoing calls
no test coverage detected