Runs an eval given the details in the eval request.
(
app_name: str, eval_set_id: str, req: RunEvalRequest
)
| 983 | tags=[TAG_EVALUATION], |
| 984 | ) |
| 985 | async def run_eval( |
| 986 | app_name: str, eval_set_id: str, req: RunEvalRequest |
| 987 | ) -> RunEvalResponse: |
| 988 | """Runs an eval given the details in the eval request.""" |
| 989 | # Create a mapping from eval set file to all the evals that needed to be |
| 990 | # run. |
| 991 | try: |
| 992 | from ..evaluation.local_eval_service import LocalEvalService |
| 993 | from .cli_eval import _collect_eval_results |
| 994 | from .cli_eval import _collect_inferences |
| 995 | |
| 996 | eval_set = self.eval_sets_manager.get_eval_set(app_name, eval_set_id) |
| 997 | |
| 998 | if not eval_set: |
| 999 | raise HTTPException( |
| 1000 | status_code=400, detail=f"Eval set `{eval_set_id}` not found." |
| 1001 | ) |
| 1002 | |
| 1003 | agent_or_app = self.agent_loader.load_agent(app_name) |
| 1004 | root_agent = self._get_root_agent(agent_or_app) |
| 1005 | |
| 1006 | eval_case_results = [] |
| 1007 | |
| 1008 | eval_service = LocalEvalService( |
| 1009 | root_agent=root_agent, |
| 1010 | eval_sets_manager=self.eval_sets_manager, |
| 1011 | eval_set_results_manager=self.eval_set_results_manager, |
| 1012 | session_service=self.session_service, |
| 1013 | artifact_service=self.artifact_service, |
| 1014 | ) |
| 1015 | inference_request = InferenceRequest( |
| 1016 | app_name=app_name, |
| 1017 | eval_set_id=eval_set.eval_set_id, |
| 1018 | eval_case_ids=req.eval_case_ids or req.eval_ids, |
| 1019 | inference_config=InferenceConfig(), |
| 1020 | ) |
| 1021 | inference_results = await _collect_inferences( |
| 1022 | inference_requests=[inference_request], eval_service=eval_service |
| 1023 | ) |
| 1024 | |
| 1025 | eval_case_results = await _collect_eval_results( |
| 1026 | inference_results=inference_results, |
| 1027 | eval_service=eval_service, |
| 1028 | eval_metrics=req.eval_metrics, |
| 1029 | ) |
| 1030 | except ModuleNotFoundError as e: |
| 1031 | logger.exception("%s", e) |
| 1032 | raise HTTPException( |
| 1033 | status_code=400, detail=MISSING_EVAL_DEPENDENCIES_MESSAGE |
| 1034 | ) from e |
| 1035 | |
| 1036 | run_eval_results = [] |
| 1037 | for eval_case_result in eval_case_results: |
| 1038 | run_eval_results.append( |
| 1039 | RunEvalResult( |
| 1040 | eval_set_file=eval_case_result.eval_set_file, |
| 1041 | eval_set_id=eval_set_id, |
| 1042 | eval_id=eval_case_result.eval_id, |
no test coverage detected