Contains configurations needed to run evaluations.
| 32 | |
| 33 | |
| 34 | class EvaluateConfig(BaseModel): |
| 35 | """Contains configurations needed to run evaluations.""" |
| 36 | |
| 37 | model_config = ConfigDict( |
| 38 | alias_generator=alias_generators.to_camel, |
| 39 | populate_by_name=True, |
| 40 | ) |
| 41 | |
| 42 | eval_metrics: list[EvalMetric] = Field( |
| 43 | description="""The list of metrics to be used in Eval.""", |
| 44 | ) |
| 45 | |
| 46 | parallelism: int = Field( |
| 47 | default=4, |
| 48 | description="""Number of parallel evaluations to run during an Eval. Few |
| 49 | factors to consider while changing this value: |
| 50 | |
| 51 | 1) Your available quota with the model, especially for those metrics that use |
| 52 | a model as a judge. Models tend to enforce per-minute or per-second SLAs. Using |
| 53 | a larger value could result in the eval quickly consuming the quota. |
| 54 | """, |
| 55 | ) |
| 56 | |
| 57 | |
| 58 | class InferenceConfig(BaseModel): |
no outgoing calls