(self, vocab: Vocab)
| 1046 | return tokens, scores, toktypes |
| 1047 | |
| 1048 | def add_meta_vocab(self, vocab: Vocab) -> None: |
| 1049 | # Handle the tokenizer model |
| 1050 | tokenizer_model = self.handle_tokenizer_model(vocab) |
| 1051 | |
| 1052 | # Ensure that tokenizer_model is added to the GGUF model |
| 1053 | self.gguf.add_tokenizer_model(tokenizer_model) |
| 1054 | |
| 1055 | # Extract model vocabulary for model conversion |
| 1056 | tokens, scores, toktypes = self.extract_vocabulary_from_model(vocab) |
| 1057 | |
| 1058 | # Add extracted token information for model conversion |
| 1059 | self.gguf.add_token_list(tokens) |
| 1060 | self.gguf.add_token_scores(scores) |
| 1061 | self.gguf.add_token_types(toktypes) |
| 1062 | |
| 1063 | def add_meta_special_vocab(self, svocab: gguf.SpecialVocab) -> None: |
| 1064 | svocab.add_to_gguf(self.gguf) |
no test coverage detected