Function main

convert.py:1191–1304

(args_in: list[str] | None = None)

def main(args_in: list[str] | None = None) -> None:
    output_choices = ["f32", "f16"]
    if np.uint32(1) == np.uint32(1).newbyteorder("<"):
        # We currently only support Q8_0 output on little endian systems.
        output_choices.append("q8_0")
    parser = argparse.ArgumentParser(description="Convert a LLaMa model to a GGML compatible file")
    parser.add_argument("--dump",        action="store_true",    help="don't convert, just show what's in the model")
    parser.add_argument("--dump-single", action="store_true",    help="don't convert, just show what's in a single model file")
    parser.add_argument("--vocab-only",  action="store_true",    help="extract only the vocab")
    parser.add_argument("--outtype",     choices=output_choices, help="output format - note: q8_0 may be very slow (default: f16 or f32 based on input)", default="f16")
    parser.add_argument("--vocab-dir",   type=Path,              help="directory containing tokenizer.model, if separate from model file")
    parser.add_argument("--outfile",     type=Path,              help="path to write to; default: based on input")
    parser.add_argument("--ctx",         type=int,               help="model training context (default: based on input)")
    parser.add_argument("--concurrency", type=int,               help=f"concurrency used for conversion (default: {DEFAULT_CONCURRENCY})", default = DEFAULT_CONCURRENCY)
    parser.add_argument("--bigendian",   action="store_true",    help="model is executed on big endian machine")
    parser.add_argument("--vocabtype",   choices=["spm", "bpe"], help="vocab format (default: spm)", default="spm")
    parser.add_argument("model",         type=Path,              help="directory containing model file, or model file itself (*.pth, *.pt, *.bin, *.safetensors)")
    parser.add_argument("sparse_predictor",     type=Path,              help="predictors for sparse FFN inference")

    args = parser.parse_args(args_in)

    try:
        with open(args.model / "config.json", "r", encoding="utf-8") as f:
            hf_config = json.load(f)
        if model_type := hf_config.get("model_type") not in ("llama", "bamboo"):
            # invoke another script to convert other models
            print(f"Model architecture {model_type} is not supported by this `convert.py`. Trying with `convert-hf-to-powerinfer-gguf.py`...")
            script_path = Path(__file__).resolve().parent / "convert-hf-to-powerinfer-gguf.py"
            subprocess.run(["python3", str(script_path.absolute())] + sys.argv[1:])
            return
    except FileNotFoundError:
        print("Could not find config.json under the original model directory. ", file=sys.stderr)
        sys.exit(1)

    if args.dump_single:
        model_plus = lazy_load_file(args.model)
        do_dump_model(model_plus)
        return

    if not args.vocab_only:
        model_plus = load_some_model(args.model)
        params = Params.load(model_plus)
        mlp_predictor_plus = load_predictor_model(args.sparse_predictor)
        params.predictor_params = PredictorParams.load(mlp_predictor_plus)
        model_plus = merge_multifile_models([model_plus, mlp_predictor_plus])
    else:
        model_plus = ModelPlus(model = {}, paths = [args.model / 'dummy'], format = 'none', vocab = None)
        params = Params.load(model_plus)

    if args.dump:
        do_dump_model(model_plus)
        return
    endianess = gguf.GGUFEndian.LITTLE
    if args.bigendian:
        endianess = gguf.GGUFEndian.BIG

    if params.n_ctx == -1:
        if args.ctx is None: