(args_in: list[str] | None = None)
| 1123 | |
| 1124 | |
| 1125 | def main(args_in: list[str] | None = None) -> None: |
| 1126 | output_choices = ["f32", "f16"] |
| 1127 | if np.uint32(1) == np.uint32(1).newbyteorder("<"): |
| 1128 | # We currently only support Q8_0 output on little endian systems. |
| 1129 | output_choices.append("q8_0") |
| 1130 | parser = argparse.ArgumentParser(description="Convert a LLaMa model to a GGML compatible file") |
| 1131 | parser.add_argument("--dump", action="store_true", help="don't convert, just show what's in the model") |
| 1132 | parser.add_argument("--dump-single", action="store_true", help="don't convert, just show what's in a single model file") |
| 1133 | parser.add_argument("--vocab-only", action="store_true", help="extract only the vocab") |
| 1134 | parser.add_argument("--outtype", choices=output_choices, help="output format - note: q8_0 may be very slow (default: f16 or f32 based on input)") |
| 1135 | parser.add_argument("--vocab-dir", type=Path, help="directory containing tokenizer.model, if separate from model file") |
| 1136 | parser.add_argument("--outfile", type=Path, help="path to write to; default: based on input") |
| 1137 | parser.add_argument("model", type=Path, help="directory containing model file, or model file itself (*.pth, *.pt, *.bin, *.safetensors)") |
| 1138 | parser.add_argument("--vocabtype", choices=["spm", "bpe"], help="vocab format (default: spm)", default="spm") |
| 1139 | parser.add_argument("--ctx", type=int, help="model training context (default: based on input)") |
| 1140 | parser.add_argument("--concurrency", type=int, help=f"concurrency used for conversion (default: {DEFAULT_CONCURRENCY})", default = DEFAULT_CONCURRENCY) |
| 1141 | parser.add_argument("--bigendian", action="store_true", help="model is executed on big endian machine") |
| 1142 | |
| 1143 | args = parser.parse_args(args_in) |
| 1144 | |
| 1145 | if args.dump_single: |
| 1146 | model_plus = lazy_load_file(args.model) |
| 1147 | do_dump_model(model_plus) |
| 1148 | return |
| 1149 | |
| 1150 | if not args.vocab_only: |
| 1151 | model_plus = load_some_model(args.model) |
| 1152 | else: |
| 1153 | model_plus = ModelPlus(model = {}, paths = [args.model / 'dummy'], format = 'none', vocab = None) |
| 1154 | |
| 1155 | if args.dump: |
| 1156 | do_dump_model(model_plus) |
| 1157 | return |
| 1158 | endianess = gguf.GGUFEndian.LITTLE |
| 1159 | if args.bigendian: |
| 1160 | endianess = gguf.GGUFEndian.BIG |
| 1161 | |
| 1162 | params = Params.load(model_plus) |
| 1163 | if params.n_ctx == -1: |
| 1164 | if args.ctx is None: |
| 1165 | raise Exception("The model doesn't have a context size, and you didn't specify one with --ctx\n" |
| 1166 | "Please specify one with --ctx:\n" |
| 1167 | " - LLaMA v1: --ctx 2048\n" |
| 1168 | " - LLaMA v2: --ctx 4096\n") |
| 1169 | params.n_ctx = args.ctx |
| 1170 | |
| 1171 | if args.outtype: |
| 1172 | params.ftype = { |
| 1173 | "f32": GGMLFileType.AllF32, |
| 1174 | "f16": GGMLFileType.MostlyF16, |
| 1175 | "q8_0": GGMLFileType.MostlyQ8_0, |
| 1176 | }[args.outtype] |
| 1177 | |
| 1178 | print(f"params = {params}") |
| 1179 | |
| 1180 | vocab: Vocab |
| 1181 | if args.vocab_only: |
| 1182 | if not args.outfile: |
no test coverage detected