| 1439 | ) |
| 1440 | |
| 1441 | def _create_vocab_by_path(self, vocab_types: list[str]) -> Vocab: |
| 1442 | vocab_classes: dict[str, type[Vocab]] = {cls.name: cls for cls in self._VOCAB_CLASSES} |
| 1443 | selected_vocabs: dict[str, type[Vocab]] = {} |
| 1444 | for vtype in vocab_types: |
| 1445 | try: |
| 1446 | selected_vocabs[vtype] = vocab_classes[vtype] |
| 1447 | except KeyError: |
| 1448 | raise ValueError(f"Unsupported vocabulary type {vtype}") from None |
| 1449 | |
| 1450 | for vtype, cls in selected_vocabs.items(): |
| 1451 | try: |
| 1452 | vocab = cls(self.path) |
| 1453 | break |
| 1454 | except FileNotFoundError: |
| 1455 | pass # ignore unavailable tokenizers |
| 1456 | else: |
| 1457 | raise FileNotFoundError(f"Could not find a tokenizer matching any of {vocab_types}") |
| 1458 | |
| 1459 | logger.info(f"Loaded vocab file {vocab.fname_tokenizer!r}, type {vocab.name!r}") |
| 1460 | return vocab |
| 1461 | |
| 1462 | def load_vocab(self, vocab_types: list[str] | None, model_parent_path: Path) -> tuple[BaseVocab, gguf.SpecialVocab]: |
| 1463 | vocab: BaseVocab |