(args)
| 66 | |
| 67 | # load model and tokenizer |
| 68 | def load(args): |
| 69 | |
| 70 | # define the model |
| 71 | misc.init_distributed_mode(args) |
| 72 | fs_init.initialize_model_parallel(args.model_parallel_size) |
| 73 | model = MetaModel(args.llama_type, args.llama_config, args.tokenizer_path, with_visual=True) |
| 74 | print(f"load pretrained from {args.pretrained_path}") |
| 75 | load_tensor_parallel_model_list(model, args.pretrained_path) |
| 76 | |
| 77 | if args.quant: |
| 78 | print("Quantizing model to 4bit!") |
| 79 | |
| 80 | from transformers.utils.quantization_config import BitsAndBytesConfig |
| 81 | quantization_config = BitsAndBytesConfig.from_dict( |
| 82 | config_dict={ |
| 83 | "load_in_8bit": False, |
| 84 | "load_in_4bit": True, |
| 85 | "bnb_4bit_quant_type": "nf4", |
| 86 | }, |
| 87 | return_unused_kwargs=False, |
| 88 | ) |
| 89 | quantize(model, quantization_config) |
| 90 | |
| 91 | #print("Model = %s" % str(model)) |
| 92 | model.bfloat16().cuda() |
| 93 | return model |
| 94 | |
| 95 | def extract_ans(ans, mode): |
| 96 | ans_line = re.split("Q:", ans, flags=re.IGNORECASE)[0] |
no test coverage detected