(model, args)
| 146 | |
| 147 | |
| 148 | def eval_llava_benchmark(model, args): |
| 149 | model_name = args.model_name |
| 150 | questions = [json.loads(q) for q in open(os.path.expanduser(args.question_file), "r")] |
| 151 | questions = get_chunk(questions, args.num_chunks, args.chunk_idx) |
| 152 | answers_file = os.path.expanduser(args.answers_file) |
| 153 | os.makedirs(os.path.dirname(answers_file), exist_ok=True) |
| 154 | ans_file = open(answers_file, "w") |
| 155 | for line in tqdm(questions): |
| 156 | idx = line["question_id"] |
| 157 | image_file = line["image"] |
| 158 | qs = line["text"] |
| 159 | cur_prompt = qs |
| 160 | image_path = os.path.join(args.image_folder, image_file) |
| 161 | prompt = qs |
| 162 | times=0 |
| 163 | output_text = generate_output( |
| 164 | model = model, |
| 165 | img_path=image_path, |
| 166 | prompt=prompt) |
| 167 | ans_id = shortuuid.uuid() |
| 168 | ans_file.write(json.dumps({"question_id": idx, |
| 169 | "prompt": cur_prompt, |
| 170 | "text": output_text, |
| 171 | "answer_id": ans_id, |
| 172 | "model_id": model_name, |
| 173 | "metadata": {}}) + "\n") |
| 174 | ans_file.flush() |
| 175 | ans_file.close() |
| 176 | |
| 177 | |
| 178 | def get_eval(content: str, max_tokens: int): |
no test coverage detected