(args_in: list[str] | None = None)
| 1498 | |
| 1499 | |
| 1500 | def main(args_in: list[str] | None = None) -> None: |
| 1501 | output_choices = ["f32", "f16", "i2"] |
| 1502 | if sys.byteorder == "little": |
| 1503 | # We currently only support Q8_0 output on little endian systems. |
| 1504 | output_choices.append("q8_0") |
| 1505 | parser = argparse.ArgumentParser(description="Convert a LLaMA model to a GGML compatible file") |
| 1506 | parser.add_argument("--dump", action="store_true", help="don't convert, just show what's in the model") |
| 1507 | parser.add_argument("--dump-single", action="store_true", help="don't convert, just show what's in a single model file") |
| 1508 | parser.add_argument("--vocab-only", action="store_true", help="extract only the vocab") |
| 1509 | parser.add_argument("--no-vocab", action="store_true", help="store model without the vocab") |
| 1510 | parser.add_argument("--outtype", choices=output_choices, help="output format - note: q8_0 may be very slow (default: f16 or f32 based on input)") |
| 1511 | parser.add_argument("--vocab-dir", type=Path, help="directory containing tokenizer.model, if separate from model file") |
| 1512 | parser.add_argument("--vocab-type", help="vocab types to try in order, choose from 'spm', 'bpe', 'hfft' (default: spm,hfft)", default="spm,hfft") |
| 1513 | parser.add_argument("--outfile", type=Path, help="path to write to; default: based on input") |
| 1514 | parser.add_argument("model", type=Path, help="directory containing model file, or model file itself (*.pth, *.pt, *.bin)") |
| 1515 | parser.add_argument("--ctx", type=int, help="model training context (default: based on input)") |
| 1516 | parser.add_argument("--concurrency", type=int, help=f"concurrency used for conversion (default: {DEFAULT_CONCURRENCY})", default=DEFAULT_CONCURRENCY) |
| 1517 | parser.add_argument("--big-endian", action="store_true", help="model is executed on big endian machine") |
| 1518 | parser.add_argument("--pad-vocab", action="store_true", help="add pad tokens when model vocab expects more than tokenizer metadata provides") |
| 1519 | parser.add_argument("--skip-unknown", action="store_true", help="skip unknown tensor names instead of failing") |
| 1520 | parser.add_argument("--verbose", action="store_true", help="increase output verbosity") |
| 1521 | |
| 1522 | args = parser.parse_args(args_in) |
| 1523 | |
| 1524 | if args.verbose: |
| 1525 | logging.basicConfig(level=logging.DEBUG) |
| 1526 | elif args.dump_single or args.dump: |
| 1527 | # Avoid printing anything besides the dump output |
| 1528 | logging.basicConfig(level=logging.WARNING) |
| 1529 | else: |
| 1530 | logging.basicConfig(level=logging.INFO) |
| 1531 | |
| 1532 | if args.no_vocab and args.vocab_only: |
| 1533 | raise ValueError("--vocab-only does not make sense with --no-vocab") |
| 1534 | |
| 1535 | if args.dump_single: |
| 1536 | model_plus = lazy_load_file(args.model) |
| 1537 | do_dump_model(model_plus) |
| 1538 | return |
| 1539 | |
| 1540 | if not args.vocab_only: |
| 1541 | model_plus = load_some_model(args.model) |
| 1542 | else: |
| 1543 | model_plus = ModelPlus(model = {}, paths = [args.model / 'dummy'], format = 'none', vocab = None) |
| 1544 | |
| 1545 | if args.dump: |
| 1546 | do_dump_model(model_plus) |
| 1547 | return |
| 1548 | |
| 1549 | endianess = gguf.GGUFEndian.LITTLE |
| 1550 | if args.big_endian: |
| 1551 | endianess = gguf.GGUFEndian.BIG |
| 1552 | |
| 1553 | params = Params.load(model_plus) |
| 1554 | if params.n_ctx == -1: |
| 1555 | if args.ctx is None: |
| 1556 | msg = """\ |
| 1557 | The model doesn't have a context size, and you didn't specify one with --ctx |
no test coverage detected