| 1429 | self.path = path |
| 1430 | |
| 1431 | def _create_special_vocab(self, vocab: BaseVocab, model_parent_path: Path) -> gguf.SpecialVocab: |
| 1432 | load_merges = vocab.name == "bpe" |
| 1433 | n_vocab = vocab.vocab_size if isinstance(vocab, Vocab) else None |
| 1434 | return gguf.SpecialVocab( |
| 1435 | model_parent_path, |
| 1436 | load_merges=load_merges, |
| 1437 | special_token_types=None, # Predetermined or passed as a parameter |
| 1438 | n_vocab=n_vocab, |
| 1439 | ) |
| 1440 | |
| 1441 | def _create_vocab_by_path(self, vocab_types: list[str]) -> Vocab: |
| 1442 | vocab_classes: dict[str, type[Vocab]] = {cls.name: cls for cls in self._VOCAB_CLASSES} |