(self)
| 473 | self.fname_tokenizer = fname_tokenizer |
| 474 | |
| 475 | def bpe_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: |
| 476 | reverse_vocab = {id: encoded_tok for encoded_tok, id in self.vocab.items()} |
| 477 | |
| 478 | for i, _ in enumerate(self.vocab): |
| 479 | yield reverse_vocab[i], 0.0, gguf.TokenType.NORMAL |
| 480 | |
| 481 | def added_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: |
| 482 | for text in self.added_tokens_list: |