(self)
| 56 | return messages |
| 57 | |
| 58 | def run(self) -> List[Experience]: |
| 59 | messages = self.format_messages() |
| 60 | |
| 61 | responses: List[Experience] = self.model.chat(messages, **self.eval_gen_args) |
| 62 | |
| 63 | for response in responses: |
| 64 | if response.response_text is None or self.task.truth is None: |
| 65 | continue |
| 66 | |
| 67 | accuracy, _ = verify_math_answer( |
| 68 | response_text=response.response_text, ground_truth=self.task.truth |
| 69 | ) |
| 70 | |
| 71 | acc_metrics = {"accuracy": accuracy} |
| 72 | if response.metrics is None: |
| 73 | response.metrics = {} |
| 74 | response.metrics.update(acc_metrics) |
| 75 | |
| 76 | return responses |
| 77 | |
| 78 | |
| 79 | class AsyncMathEvalWorkflow(MathEvalWorkflow): |
nothing calls this directly
no test coverage detected