(self, tokens)
| 124 | return [self.map_token(token) for token in tokens] |
| 125 | |
| 126 | def decode_tokens(self, tokens): |
| 127 | decoded = [self.string(token) for token in tokens] |
| 128 | return [int(x) for x in decoded if not x.startswith("vocab_pad_token")] |
| 129 | |
| 130 | |
| 131 | class CodeTokenizer(object): |