(args)
| 70 | |
| 71 | |
| 72 | def eval_model(args): |
| 73 | # Model |
| 74 | disable_torch_init() |
| 75 | model_path = os.path.expanduser(args.model_path) |
| 76 | model_name = get_model_name_from_path(model_path) |
| 77 | tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, args.model_base, model_name) |
| 78 | |
| 79 | questions = [json.loads(q) for q in open(os.path.expanduser(args.question_file), "r")] |
| 80 | questions = get_chunk(questions, args.num_chunks, args.chunk_idx) |
| 81 | answers_file = os.path.expanduser(args.answers_file) |
| 82 | os.makedirs(os.path.dirname(answers_file), exist_ok=True) |
| 83 | ans_file = open(answers_file, "w") |
| 84 | |
| 85 | if 'plain' in model_name and 'finetune' not in model_name.lower() and 'mmtag' not in args.conv_mode: |
| 86 | args.conv_mode = args.conv_mode + '_mmtag' |
| 87 | print(f'It seems that this is a plain model, but it is not using a mmtag prompt, auto switching to {args.conv_mode}.') |
| 88 | |
| 89 | data_loader = create_data_loader(questions, args.image_folder, tokenizer, image_processor, model.config) |
| 90 | |
| 91 | for (input_ids, image_tensor), line in tqdm(zip(data_loader, questions), total=len(questions)): |
| 92 | idx = line["question_id"] |
| 93 | cur_prompt = line["text"] |
| 94 | |
| 95 | stop_str = conv_templates[args.conv_mode].sep if conv_templates[args.conv_mode].sep_style != SeparatorStyle.TWO else conv_templates[args.conv_mode].sep2 |
| 96 | input_ids = input_ids.to(device='cuda', non_blocking=True) |
| 97 | |
| 98 | with torch.inference_mode(): |
| 99 | output_ids = model.generate( |
| 100 | input_ids, |
| 101 | images=image_tensor.to(dtype=torch.float16, device='cuda', non_blocking=True), |
| 102 | do_sample=True if args.temperature > 0 else False, |
| 103 | temperature=args.temperature, |
| 104 | top_p=args.top_p, |
| 105 | num_beams=args.num_beams, |
| 106 | max_new_tokens=128, |
| 107 | use_cache=True) |
| 108 | |
| 109 | input_token_len = input_ids.shape[1] |
| 110 | n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item() |
| 111 | if n_diff_input_output > 0: |
| 112 | print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids') |
| 113 | outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0] |
| 114 | outputs = outputs.strip() |
| 115 | if outputs.endswith(stop_str): |
| 116 | outputs = outputs[:-len(stop_str)] |
| 117 | outputs = outputs.strip() |
| 118 | |
| 119 | ans_id = shortuuid.uuid() |
| 120 | ans_file.write(json.dumps({"question_id": idx, |
| 121 | "prompt": cur_prompt, |
| 122 | "text": outputs, |
| 123 | "answer_id": ans_id, |
| 124 | "model_id": model_name, |
| 125 | "metadata": {}}) + "\n") |
| 126 | # ans_file.flush() |
| 127 | ans_file.close() |
| 128 | |
| 129 | if __name__ == "__main__": |
no test coverage detected