Evaluates candidate agents with the ADK's LocalEvalService.
| 145 | |
| 146 | |
| 147 | class LocalEvalSampler(Sampler[UnstructuredSamplingResult]): |
| 148 | """Evaluates candidate agents with the ADK's LocalEvalService.""" |
| 149 | |
| 150 | def __init__( |
| 151 | self, |
| 152 | config: LocalEvalSamplerConfig, |
| 153 | eval_sets_manager: EvalSetsManager, |
| 154 | ): |
| 155 | self._config = config |
| 156 | self._eval_sets_manager = eval_sets_manager |
| 157 | |
| 158 | self._train_eval_set = self._config.train_eval_set |
| 159 | self._train_eval_case_ids = ( |
| 160 | self._config.train_eval_case_ids |
| 161 | or self._get_eval_case_ids(self._train_eval_set) |
| 162 | ) |
| 163 | |
| 164 | self._validation_eval_set = ( |
| 165 | self._config.validation_eval_set or self._train_eval_set |
| 166 | ) |
| 167 | if self._config.validation_eval_case_ids: |
| 168 | self._validation_eval_case_ids = self._config.validation_eval_case_ids |
| 169 | elif self._config.validation_eval_set: |
| 170 | self._validation_eval_case_ids = self._get_eval_case_ids( |
| 171 | self._validation_eval_set |
| 172 | ) |
| 173 | else: |
| 174 | self._validation_eval_case_ids = self._train_eval_case_ids |
| 175 | |
| 176 | def _get_selected_example_set_id( |
| 177 | self, example_set: Literal[Sampler.TRAIN_SET, Sampler.VALIDATION_SET] |
| 178 | ) -> str: |
| 179 | """Returns the ID of the selected example set.""" |
| 180 | return { |
| 181 | Sampler.TRAIN_SET: self._train_eval_set, |
| 182 | Sampler.VALIDATION_SET: self._validation_eval_set, |
| 183 | }[example_set] |
| 184 | |
| 185 | def _get_all_example_ids( |
| 186 | self, example_set: Literal[Sampler.TRAIN_SET, Sampler.VALIDATION_SET] |
| 187 | ) -> list[str]: |
| 188 | """Returns the IDs of all examples in the selected example set.""" |
| 189 | return { |
| 190 | Sampler.TRAIN_SET: self._train_eval_case_ids, |
| 191 | Sampler.VALIDATION_SET: self._validation_eval_case_ids, |
| 192 | }[example_set] |
| 193 | |
| 194 | def _get_eval_case_ids(self, eval_set_id: str) -> list[str]: |
| 195 | """Returns the ids of eval cases in the given eval set.""" |
| 196 | eval_set = self._eval_sets_manager.get_eval_set( |
| 197 | app_name=self._config.app_name, |
| 198 | eval_set_id=eval_set_id, |
| 199 | ) |
| 200 | if eval_set: |
| 201 | return [eval_case.eval_id for eval_case in eval_set.eval_cases] |
| 202 | else: |
| 203 | raise ValueError( |
| 204 | f"Eval set `{eval_set_id}` does not exist for app" |
no outgoing calls