(path: Path, vocabtype: str | None)
| 1135 | |
| 1136 | |
| 1137 | def load_vocab(path: Path, vocabtype: str | None) -> Vocab: |
| 1138 | # Be extra-friendly and accept either a file or a directory. Also, if it's |
| 1139 | # a directory, it might be the model directory, and tokenizer.model might |
| 1140 | # be in the parent of that. |
| 1141 | if path.is_dir(): |
| 1142 | vocab_file = "tokenizer.model" |
| 1143 | if vocabtype == 'bpe': |
| 1144 | vocab_file = "vocab.json" |
| 1145 | path2 = path / vocab_file |
| 1146 | # Use `.parent` instead of /.. to handle the symlink case better. |
| 1147 | path3 = path.parent / vocab_file |
| 1148 | if path2.exists(): |
| 1149 | path = path2 |
| 1150 | elif path3.exists(): |
| 1151 | path = path3 |
| 1152 | else: |
| 1153 | raise FileNotFoundError( |
| 1154 | f"Could not find {vocab_file} in {path} or its parent; " |
| 1155 | "if it's in another directory, pass the directory as --vocab-dir") |
| 1156 | |
| 1157 | print(f"Loading vocab file '{path}', type '{vocabtype}'") |
| 1158 | |
| 1159 | added_tokens_path = path.parent / "added_tokens.json" |
| 1160 | if vocabtype == "bpe": |
| 1161 | return BpeVocab(path, added_tokens_path if added_tokens_path.exists() else None) |
| 1162 | elif vocabtype == "spm": |
| 1163 | return SentencePieceVocab(path, added_tokens_path if added_tokens_path.exists() else None) |
| 1164 | else: |
| 1165 | raise ValueError(f"Unsupported vocabulary type {vocabtype}") |
| 1166 | |
| 1167 | |
| 1168 | def default_outfile(model_paths: list[Path], file_type: GGMLFileType) -> Path: |
no test coverage detected