Returns a dictionary of eval set info to evals that should be run. Args: evals_to_run_info: While the structure is quite simple, a list of string, each string actually is formatted with the following convention: :[comma separated eval case ids]
(
evals_to_run_info: list[str],
)
| 101 | |
| 102 | |
| 103 | def parse_and_get_evals_to_run( |
| 104 | evals_to_run_info: list[str], |
| 105 | ) -> dict[str, list[str]]: |
| 106 | """Returns a dictionary of eval set info to evals that should be run. |
| 107 | |
| 108 | Args: |
| 109 | evals_to_run_info: While the structure is quite simple, a list of string, |
| 110 | each string actually is formatted with the following convention: |
| 111 | <eval_set_file_path | eval_set_id>:[comma separated eval case ids] |
| 112 | """ |
| 113 | eval_set_to_evals = {} |
| 114 | for input_eval_set in evals_to_run_info: |
| 115 | evals = [] |
| 116 | if ":" not in input_eval_set: |
| 117 | # We don't have any eval cases specified. This would be the case where the |
| 118 | # the user wants to run all eval cases in the eval set. |
| 119 | eval_set = input_eval_set |
| 120 | else: |
| 121 | # There are eval cases that we need to parse. The user wants to run |
| 122 | # specific eval cases from the eval set. |
| 123 | eval_set = input_eval_set.split(":")[0] |
| 124 | evals = input_eval_set.split(":")[1].split(",") |
| 125 | evals = [s for s in evals if s.strip()] |
| 126 | |
| 127 | if eval_set not in eval_set_to_evals: |
| 128 | eval_set_to_evals[eval_set] = [] |
| 129 | |
| 130 | eval_set_to_evals[eval_set].extend(evals) |
| 131 | |
| 132 | return eval_set_to_evals |
| 133 | |
| 134 | |
| 135 | async def _collect_inferences( |