Evaluate a single QA pair with the given tools.
(
client: Anthropic,
model: str,
qa_pair: dict[str, Any],
tools: list[dict[str, Any]],
connection: Any,
task_index: int,
)
| 152 | |
| 153 | |
| 154 | async def evaluate_single_task( |
| 155 | client: Anthropic, |
| 156 | model: str, |
| 157 | qa_pair: dict[str, Any], |
| 158 | tools: list[dict[str, Any]], |
| 159 | connection: Any, |
| 160 | task_index: int, |
| 161 | ) -> dict[str, Any]: |
| 162 | """Evaluate a single QA pair with the given tools.""" |
| 163 | start_time = time.time() |
| 164 | |
| 165 | print(f"Task {task_index + 1}: Running task with question: {qa_pair['question']}") |
| 166 | response, tool_metrics = await agent_loop(client, model, qa_pair["question"], tools, connection) |
| 167 | |
| 168 | response_value = extract_xml_content(response, "response") |
| 169 | summary = extract_xml_content(response, "summary") |
| 170 | feedback = extract_xml_content(response, "feedback") |
| 171 | |
| 172 | duration_seconds = time.time() - start_time |
| 173 | |
| 174 | return { |
| 175 | "question": qa_pair["question"], |
| 176 | "expected": qa_pair["answer"], |
| 177 | "actual": response_value, |
| 178 | "score": int(response_value == qa_pair["answer"]) if response_value else 0, |
| 179 | "total_duration": duration_seconds, |
| 180 | "tool_calls": tool_metrics, |
| 181 | "num_tool_calls": sum(len(metrics["durations"]) for metrics in tool_metrics.values()), |
| 182 | "summary": summary, |
| 183 | "feedback": feedback, |
| 184 | } |
| 185 | |
| 186 | |
| 187 | REPORT_HEADER = """ |
no test coverage detected