| 30 | return json.dumps(dumped_dict) |
| 31 | |
| 32 | class EvalOutput(BaseModel): |
| 33 | # NOTE: User-specified |
| 34 | instance_id: str |
| 35 | # output of the evaluation |
| 36 | # store anything that is needed for the score calculation |
| 37 | test_result: dict[str, Any] |
| 38 | |
| 39 | instruction: str | None = None |
| 40 | |
| 41 | # Interaction info |
| 42 | metadata: EvalMetadata | None = None |
| 43 | # list[tuple[dict[str, Any], dict[str, Any]]] - for compatibility with the old format |
| 44 | messages: List | None = None |
| 45 | error: str | None = None |
| 46 | |
| 47 | # Optionally save the input test instance |
| 48 | instance: dict[str, Any] | None = None |
| 49 | |
| 50 | def model_dump(self, *args, **kwargs): |
| 51 | dumped_dict = super().model_dump(*args, **kwargs) |
| 52 | # Remove None values |
| 53 | dumped_dict = {k: v for k, v in dumped_dict.items() if v is not None} |
| 54 | # Apply custom serialization for metadata (to avoid leaking sensitive information) |
| 55 | if self.metadata is not None: |
| 56 | dumped_dict['metadata'] = self.metadata.model_dump() |
| 57 | return dumped_dict |
| 58 | |
| 59 | def model_dump_json(self, *args, **kwargs): |
| 60 | dumped = super().model_dump_json(*args, **kwargs) |
| 61 | dumped_dict = json.loads(dumped) |
| 62 | # Apply custom serialization for metadata (to avoid leaking sensitive information) |
| 63 | if 'metadata' in dumped_dict: |
| 64 | dumped_dict['metadata'] = json.loads(self.metadata.model_dump_json()) |
| 65 | return json.dumps(dumped_dict) |