| 10 | |
| 11 | |
| 12 | class OpenEvaluator(BaseEvaluator): |
| 13 | def __init__( |
| 14 | self, |
| 15 | prediction_key: str = "model_prediction", |
| 16 | score_model: Optional[Any] = None, |
| 17 | ): |
| 18 | super().__init__(prediction_key) |
| 19 | # 支持自定义评分模型，默认 GPT4o |
| 20 | self.score_model = score_model or GPT4o() |
| 21 | |
| 22 | def _build_prompt(self, item: Dict) -> Any: |
| 23 | """ |
| 24 | 为被测模型构建解题 prompt。 |
| 25 | """ |
| 26 | question = item.get("question", "") |
| 27 | images = normalize_image_paths(item.get("image_path")) |
| 28 | text_content = f"Question: {question}\nPlease answer the question." |
| 29 | if images: |
| 30 | assert all( |
| 31 | isinstance(p, str) for p in images |
| 32 | ), f"images should be List[str], got {images}" |
| 33 | return {"text": text_content, "images": prepare_images_for_prompt(images)} |
| 34 | else: |
| 35 | return text_content |
| 36 | |
| 37 | def _build_score_prompt(self, item: Dict, model_output: Any) -> Any: |
| 38 | """ |
| 39 | 构建评分 prompt，包含评分准则。 |
| 40 | """ |
| 41 | question = item.get("question", "") |
| 42 | images = normalize_image_paths(item.get("image_path")) |
| 43 | reference_answer = item.get("answer", "") |
| 44 | # 支持图片型参考答案 |
| 45 | reference_images = [] |
| 46 | if isinstance(reference_answer, str) and reference_answer.lower().endswith( |
| 47 | (".png", ".jpg", ".jpeg", ".bmp", ".gif", ".webp") |
| 48 | ): |
| 49 | reference_images = [reference_answer] |
| 50 | reference_answer_text = "[See reference image]" |
| 51 | else: |
| 52 | reference_answer_text = reference_answer |
| 53 | # 支持图片型模型输出 |
| 54 | model_output_images = [] |
| 55 | if isinstance(model_output, str) and model_output.lower().endswith( |
| 56 | (".png", ".jpg", ".jpeg", ".bmp", ".gif", ".webp") |
| 57 | ): |
| 58 | model_output_images = [model_output] |
| 59 | model_output_text = "[See model output image]" |
| 60 | else: |
| 61 | model_output_text = model_output |
| 62 | # 评分准则 |
| 63 | prompt_lines = [ |
| 64 | "You are an expert evaluator. Given the following question, reference answer, and model answer, please rate the model answer on a scale of 0 to 1, and explain your reasoning.", |
| 65 | "Scoring rules:", |
| 66 | "- If the reference answer is an image but the model output does not contain an image, score 0.", |
| 67 | "- If the reference answer is text but the model output does not contain text, score 0.", |
| 68 | "- Otherwise, score based on the similarity and correctness of the model output compared to the reference answer.", |
| 69 | "- If both text and image are present, consider both in your evaluation.", |