(self, vocab: Vocab)
| 1121 | self.gguf.add_file_type(params.ftype) |
| 1122 | |
| 1123 | def extract_vocabulary_from_model(self, vocab: Vocab) -> tuple[list[bytes], list[float], list[gguf.TokenType]]: |
| 1124 | tokens = [] |
| 1125 | scores = [] |
| 1126 | toktypes = [] |
| 1127 | |
| 1128 | # NOTE: `all_tokens` returns the base vocabulary and added tokens |
| 1129 | for text, score, toktype in vocab.all_tokens(): |
| 1130 | tokens.append(text) |
| 1131 | scores.append(score) |
| 1132 | toktypes.append(toktype) |
| 1133 | |
| 1134 | assert len(tokens) == vocab.vocab_size |
| 1135 | |
| 1136 | return tokens, scores, toktypes |
| 1137 | |
| 1138 | def add_meta_vocab(self, vocab: Vocab) -> None: |
| 1139 | # Ensure that tokenizer_model is added to the GGUF model |
no test coverage detected