(path: Path, vocabtype: str | None)
| 1069 | |
| 1070 | |
| 1071 | def load_vocab(path: Path, vocabtype: str | None) -> Vocab: |
| 1072 | # Be extra-friendly and accept either a file or a directory. Also, if it's |
| 1073 | # a directory, it might be the model directory, and tokenizer.model might |
| 1074 | # be in the parent of that. |
| 1075 | if path.is_dir(): |
| 1076 | vocab_file = "tokenizer.model" |
| 1077 | if vocabtype == 'bpe': |
| 1078 | vocab_file = "vocab.json" |
| 1079 | path2 = path / vocab_file |
| 1080 | # Use `.parent` instead of /.. to handle the symlink case better. |
| 1081 | path3 = path.parent / vocab_file |
| 1082 | if path2.exists(): |
| 1083 | path = path2 |
| 1084 | elif path3.exists(): |
| 1085 | path = path3 |
| 1086 | else: |
| 1087 | raise FileNotFoundError( |
| 1088 | f"Could not find {vocab_file} in {path} or its parent; " |
| 1089 | "if it's in another directory, pass the directory as --vocab-dir") |
| 1090 | |
| 1091 | print(f"Loading vocab file '{path}', type '{vocabtype}'") |
| 1092 | |
| 1093 | added_tokens_path = path.parent / "added_tokens.json" |
| 1094 | if vocabtype == "bpe": |
| 1095 | return BpeVocab(path, added_tokens_path if added_tokens_path.exists() else None) |
| 1096 | elif vocabtype == "spm": |
| 1097 | return SentencePieceVocab(path, added_tokens_path if added_tokens_path.exists() else None) |
| 1098 | else: |
| 1099 | raise ValueError(f"Unsupported vocabulary type {vocabtype}") |
| 1100 | |
| 1101 | |
| 1102 | def default_outfile(model_paths: list[Path], file_type: GGMLFileType) -> Path: |
no test coverage detected