Returns an instance of EvalSetsManager.
(
eval_storage_uri: Optional[str], agents_dir: str
)
| 296 | |
| 297 | |
| 298 | def get_eval_sets_manager( |
| 299 | eval_storage_uri: Optional[str], agents_dir: str |
| 300 | ) -> EvalSetsManager: |
| 301 | """Returns an instance of EvalSetsManager.""" |
| 302 | try: |
| 303 | from ..evaluation.local_eval_sets_manager import LocalEvalSetsManager |
| 304 | from .utils import evals |
| 305 | except ModuleNotFoundError as mnf: |
| 306 | raise click.ClickException(MISSING_EVAL_DEPENDENCIES_MESSAGE) from mnf |
| 307 | |
| 308 | if eval_storage_uri: |
| 309 | gcs_eval_managers = evals.create_gcs_eval_managers_from_uri( |
| 310 | eval_storage_uri |
| 311 | ) |
| 312 | return gcs_eval_managers.eval_sets_manager |
| 313 | else: |
| 314 | return LocalEvalSetsManager(agents_dir=agents_dir) |