| 1017 | self.gguf.add_file_type(params.ftype) |
| 1018 | |
| 1019 | def handle_tokenizer_model(self, vocab: Vocab) -> str: |
| 1020 | # Map the vocab types to the supported tokenizer models |
| 1021 | tokenizer_model = { |
| 1022 | SentencePieceVocab: "llama", |
| 1023 | HfVocab: "llama", |
| 1024 | BpeVocab: "gpt2", |
| 1025 | }.get(type(vocab)) |
| 1026 | |
| 1027 | # Block if vocab type is not predefined |
| 1028 | if tokenizer_model is None: |
| 1029 | raise ValueError("Unknown vocab type: Not supported") |
| 1030 | |
| 1031 | return tokenizer_model |
| 1032 | |
| 1033 | def extract_vocabulary_from_model(self, vocab: Vocab) -> tuple[list[bytes], list[float], list[gguf.TokenType]]: |
| 1034 | tokens = [] |