(args)
| 217 | |
| 218 | |
| 219 | def eval_gpt4(args): |
| 220 | print("evaluating with GPT4 now...") |
| 221 | openai.api_key = args.openai_key |
| 222 | os.makedirs(f"../LLaVA_benchmark/{args.model_name}", exist_ok=True) |
| 223 | f_q = open(os.path.expanduser(args.question_file)) |
| 224 | f_ans1 = open(os.path.expanduser(args.answer_list[0])) |
| 225 | f_ans2 = open(os.path.expanduser(args.answer_list[1])) |
| 226 | rule_dict = json.load(open(os.path.expanduser(args.rule), 'r')) |
| 227 | |
| 228 | if os.path.isfile(os.path.expanduser(args.output)): |
| 229 | cur_reviews = [json.loads(line) for line in open(os.path.expanduser(args.output))] |
| 230 | else: |
| 231 | cur_reviews = [] |
| 232 | |
| 233 | review_file = open(f'{args.output}', 'a') |
| 234 | |
| 235 | context_list = [json.loads(line) for line in open(os.path.expanduser(args.context))] |
| 236 | image_to_context = {context['image']: context for context in context_list} |
| 237 | |
| 238 | handles = [] |
| 239 | idx = 0 |
| 240 | for ques_js, ans1_js, ans2_js in zip(f_q, f_ans1, f_ans2): |
| 241 | ques = json.loads(ques_js) |
| 242 | ans1 = json.loads(ans1_js) |
| 243 | ans2 = json.loads(ans2_js) |
| 244 | |
| 245 | inst = image_to_context[ques['image']] |
| 246 | cap_str = '\n'.join(inst['caption']) |
| 247 | #box_str = '\n'.join([f'{instance["category"]}: {instance["bbox"]}' for instance in inst['instances']]) |
| 248 | |
| 249 | category = json.loads(ques_js)['category'] |
| 250 | if category in rule_dict: |
| 251 | rule = rule_dict[category] |
| 252 | else: |
| 253 | assert False, f"Visual QA category not found in rule file: {category}." |
| 254 | prompt = rule['prompt'] |
| 255 | role = rule['role'] |
| 256 | content = (f'[Context]\n{cap_str}\n\n' |
| 257 | f'[Question]\n{ques["text"]}\n\n' |
| 258 | f'[{role} 1]\n{ans1["text"]}\n\n[End of {role} 1]\n\n' |
| 259 | f'[{role} 2]\n{ans2["text"]}\n\n[End of {role} 2]\n\n' |
| 260 | f'[System]\n{prompt}\n\n') |
| 261 | cur_js = { |
| 262 | 'id': idx+1, |
| 263 | 'question_id': ques['question_id'], |
| 264 | 'answer1_id': ans1.get('answer_id', ans1['question_id']), |
| 265 | 'answer2_id': ans2.get('answer_id', ans2['answer_id']), |
| 266 | 'category': category |
| 267 | } |
| 268 | if idx >= len(cur_reviews): |
| 269 | review = get_eval(content, args.max_tokens) |
| 270 | scores = parse_score(review) |
| 271 | cur_js['content'] = review |
| 272 | cur_js['tuple'] = scores |
| 273 | review_file.write(json.dumps(cur_js) + '\n') |
| 274 | review_file.flush() |
| 275 | else: |
| 276 | print(f'Skipping {idx} as we already have it.') |
no test coverage detected