(self, testset: List[Dict[str, Any]])
| 67 | set_model(model) |
| 68 | |
| 69 | async def gather_results(self, testset: List[Dict[str, Any]]) -> List[Any]: |
| 70 | # Collect experiment results with concurrency limit |
| 71 | set_module(self.config.module_type) |
| 72 | |
| 73 | question_key = self.config.question_key |
| 74 | semaphore = asyncio.Semaphore(self.max_concurrent) |
| 75 | |
| 76 | async def limited_atom(question, context=None): |
| 77 | async with semaphore: |
| 78 | if context is not None: |
| 79 | return await atom(question, context) |
| 80 | else: |
| 81 | return await atom(question) |
| 82 | |
| 83 | tasks = [] |
| 84 | |
| 85 | if self.config.requires_context(): |
| 86 | from experiment.prompter.multihop import contexts |
| 87 | # Handle case where question_key is a list |
| 88 | if isinstance(question_key, list): |
| 89 | formatted_questions = [self._format_question_from_keys(item, question_key) for item in testset] |
| 90 | tasks = [limited_atom(question, contexts(item, self.dataset)) |
| 91 | for question, item in zip(formatted_questions, testset)] |
| 92 | else: |
| 93 | tasks = [limited_atom(item[question_key], contexts(item, self.dataset)) for item in testset] |
| 94 | else: |
| 95 | # Handle case where question_key is a list |
| 96 | if isinstance(question_key, list): |
| 97 | tasks = [limited_atom(self._format_question_from_keys(item, question_key)) for item in testset] |
| 98 | else: |
| 99 | tasks = [limited_atom(item[question_key]) for item in testset] |
| 100 | |
| 101 | return await tqdm.gather(*tasks, desc=f"Processing {self.dataset} tasks") |
| 102 | |
| 103 | def _format_question_from_keys(self, item: Dict[str, Any], keys: List[str]) -> str: |
| 104 | # When question_key is a list, concatenate values from multiple keys into a single question |
no test coverage detected