Ask AMP to run the project's deployment once per model and compare them. AMP finds the deployment by PROJECT_ID (`[tool.crewai].project_id`) unless DEPLOYMENT_ID names one. MODELS are `provider/model` strings; the deployment's own models are the baseline, so they are not sen
(
self,
models: list[str],
*,
project_id: str,
eval_config: str | None = None,
deployment_id: str | None = None,
)
| 71 | ) |
| 72 | |
| 73 | def create_models_evaluation( |
| 74 | self, |
| 75 | models: list[str], |
| 76 | *, |
| 77 | project_id: str, |
| 78 | eval_config: str | None = None, |
| 79 | deployment_id: str | None = None, |
| 80 | ) -> httpx.Response: |
| 81 | """Ask AMP to run the project's deployment once per model and compare them. |
| 82 | |
| 83 | AMP finds the deployment by PROJECT_ID (`[tool.crewai].project_id`) |
| 84 | unless DEPLOYMENT_ID names one. MODELS are `provider/model` strings; |
| 85 | the deployment's own models are the baseline, so they are not sent. |
| 86 | EVAL_CONFIG is the project's `eval.jsonc`, as `create_evaluation` sends it. |
| 87 | """ |
| 88 | body: dict[str, Any] = { |
| 89 | "kind": "models", |
| 90 | "project_id": project_id, |
| 91 | "models": list(models), |
| 92 | } |
| 93 | if eval_config: |
| 94 | body["eval_config"] = eval_config |
| 95 | if deployment_id: |
| 96 | body["deployment_id"] = deployment_id |
| 97 | return self._make_request( |
| 98 | "POST", |
| 99 | self.EVALUATIONS_RESOURCE, |
| 100 | json=body, |
| 101 | timeout=self.EVALUATION_START_TIMEOUT, |
| 102 | ) |
| 103 | |
| 104 | def get_evaluation(self, evaluation_id: str) -> httpx.Response: |
| 105 | """The evaluation's status and, once done, its verdict. |
no test coverage detected