(self, vocab: Vocab)
| 1136 | return tokens, scores, toktypes |
| 1137 | |
| 1138 | def add_meta_vocab(self, vocab: Vocab) -> None: |
| 1139 | # Ensure that tokenizer_model is added to the GGUF model |
| 1140 | self.gguf.add_tokenizer_model(vocab.tokenizer_model) |
| 1141 | |
| 1142 | # Extract model vocabulary for model conversion |
| 1143 | tokens, scores, toktypes = self.extract_vocabulary_from_model(vocab) |
| 1144 | |
| 1145 | # Add extracted token information for model conversion |
| 1146 | self.gguf.add_token_list(tokens) |
| 1147 | self.gguf.add_token_scores(scores) |
| 1148 | self.gguf.add_token_types(toktypes) |
| 1149 | |
| 1150 | def add_meta_special_vocab(self, svocab: gguf.SpecialVocab) -> None: |
| 1151 | svocab.add_to_gguf(self.gguf) |
no test coverage detected