MCPcopy Create free account
hub / github.com/crewAIInc/crewAI / create_models_evaluation

Method create_models_evaluation

lib/cli/src/crewai_cli/plus_api.py:73–102  ·  view source on GitHub ↗

Ask AMP to run the project's deployment once per model and compare them. AMP finds the deployment by PROJECT_ID (`[tool.crewai].project_id`) unless DEPLOYMENT_ID names one. MODELS are `provider/model` strings; the deployment's own models are the baseline, so they are not sen

(
        self,
        models: list[str],
        *,
        project_id: str,
        eval_config: str | None = None,
        deployment_id: str | None = None,
    )

Source from the content-addressed store, hash-verified

71 )
72
73 def create_models_evaluation(
74 self,
75 models: list[str],
76 *,
77 project_id: str,
78 eval_config: str | None = None,
79 deployment_id: str | None = None,
80 ) -> httpx.Response:
81 """Ask AMP to run the project's deployment once per model and compare them.
82
83 AMP finds the deployment by PROJECT_ID (`[tool.crewai].project_id`)
84 unless DEPLOYMENT_ID names one. MODELS are `provider/model` strings;
85 the deployment's own models are the baseline, so they are not sent.
86 EVAL_CONFIG is the project's `eval.jsonc`, as `create_evaluation` sends it.
87 """
88 body: dict[str, Any] = {
89 "kind": "models",
90 "project_id": project_id,
91 "models": list(models),
92 }
93 if eval_config:
94 body["eval_config"] = eval_config
95 if deployment_id:
96 body["deployment_id"] = deployment_id
97 return self._make_request(
98 "POST",
99 self.EVALUATIONS_RESOURCE,
100 json=body,
101 timeout=self.EVALUATION_START_TIMEOUT,
102 )
103
104 def get_evaluation(self, evaluation_id: str) -> httpx.Response:
105 """The evaluation's status and, once done, its verdict.

Callers 1

eval_modelsFunction · 0.45

Calls 1

_make_requestMethod · 0.95

Tested by

no test coverage detected