(args)
| 52 | |
| 53 | |
| 54 | def eval_model(args): |
| 55 | # Model |
| 56 | disable_torch_init() |
| 57 | model_path = os.path.expanduser(args.model_path) |
| 58 | model_name = get_model_name_from_path(model_path) |
| 59 | tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, args.model_base, model_name) |
| 60 | |
| 61 | questions = pd.read_table(os.path.expanduser(args.question_file)) |
| 62 | questions = get_chunk(questions, args.num_chunks, args.chunk_idx) |
| 63 | answers_file = os.path.expanduser(args.answers_file) |
| 64 | os.makedirs(os.path.dirname(answers_file), exist_ok=True) |
| 65 | ans_file = open(answers_file, "w") |
| 66 | |
| 67 | if 'plain' in model_name and 'finetune' not in model_name.lower() and 'mmtag' not in args.conv_mode: |
| 68 | args.conv_mode = args.conv_mode + '_mmtag' |
| 69 | print(f'It seems that this is a plain model, but it is not using a mmtag prompt, auto switching to {args.conv_mode}.') |
| 70 | |
| 71 | for index, row in tqdm(questions.iterrows(), total=len(questions)): |
| 72 | options = get_options(row, all_options) |
| 73 | cur_option_char = all_options[:len(options)] |
| 74 | |
| 75 | if args.all_rounds: |
| 76 | num_rounds = len(options) |
| 77 | else: |
| 78 | num_rounds = 1 |
| 79 | |
| 80 | for round_idx in range(num_rounds): |
| 81 | idx = row['index'] |
| 82 | question = row['question'] |
| 83 | hint = row['hint'] |
| 84 | image = load_image_from_base64(row['image']) |
| 85 | if not is_none(hint): |
| 86 | question = hint + '\n' + question |
| 87 | for option_char, option in zip(all_options[:len(options)], options): |
| 88 | question = question + '\n' + option_char + '. ' + option |
| 89 | qs = cur_prompt = question |
| 90 | if model.config.mm_use_im_start_end: |
| 91 | qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs |
| 92 | else: |
| 93 | qs = DEFAULT_IMAGE_TOKEN + '\n' + qs |
| 94 | |
| 95 | if args.single_pred_prompt: |
| 96 | if args.lang == 'cn': |
| 97 | qs = qs + '\n' + "请直接回答选项字母。" |
| 98 | else: |
| 99 | qs = qs + '\n' + "Answer with the option's letter from the given choices directly." |
| 100 | |
| 101 | conv = conv_templates[args.conv_mode].copy() |
| 102 | conv.append_message(conv.roles[0], qs) |
| 103 | conv.append_message(conv.roles[1], None) |
| 104 | prompt = conv.get_prompt() |
| 105 | |
| 106 | input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda() |
| 107 | |
| 108 | image_tensor = process_images([image], image_processor, model.config)[0] |
| 109 | |
| 110 | with torch.inference_mode(): |
| 111 | output_ids = model.generate( |
no test coverage detected