| 50 | |
| 51 | |
| 52 | class ExperimentRunner: |
| 53 | def __init__(self, dataset: str, model: str, start: int = 0, end: int = -1, mode: str = "atom", max_concurrent: int = 10): |
| 54 | # Initialize experiment runner |
| 55 | self.dataset = dataset |
| 56 | self.start = start |
| 57 | self.end = None if end == -1 else end |
| 58 | self.interval = "full" if self.end is None else f"{start}-{end}" |
| 59 | self.timestamp = time.time() |
| 60 | self.mode = mode |
| 61 | self.max_concurrent = max_concurrent # Maximum concurrent tasks |
| 62 | # Validate dataset support |
| 63 | if dataset not in DATASET_CONFIGS: |
| 64 | raise ValueError(f"Unsupported dataset: {dataset}") |
| 65 | |
| 66 | self.config = DATASET_CONFIGS[dataset] |
| 67 | set_model(model) |
| 68 | |
| 69 | async def gather_results(self, testset: List[Dict[str, Any]]) -> List[Any]: |
| 70 | # Collect experiment results with concurrency limit |
| 71 | set_module(self.config.module_type) |
| 72 | |
| 73 | question_key = self.config.question_key |
| 74 | semaphore = asyncio.Semaphore(self.max_concurrent) |
| 75 | |
| 76 | async def limited_atom(question, context=None): |
| 77 | async with semaphore: |
| 78 | if context is not None: |
| 79 | return await atom(question, context) |
| 80 | else: |
| 81 | return await atom(question) |
| 82 | |
| 83 | tasks = [] |
| 84 | |
| 85 | if self.config.requires_context(): |
| 86 | from experiment.prompter.multihop import contexts |
| 87 | # Handle case where question_key is a list |
| 88 | if isinstance(question_key, list): |
| 89 | formatted_questions = [self._format_question_from_keys(item, question_key) for item in testset] |
| 90 | tasks = [limited_atom(question, contexts(item, self.dataset)) |
| 91 | for question, item in zip(formatted_questions, testset)] |
| 92 | else: |
| 93 | tasks = [limited_atom(item[question_key], contexts(item, self.dataset)) for item in testset] |
| 94 | else: |
| 95 | # Handle case where question_key is a list |
| 96 | if isinstance(question_key, list): |
| 97 | tasks = [limited_atom(self._format_question_from_keys(item, question_key)) for item in testset] |
| 98 | else: |
| 99 | tasks = [limited_atom(item[question_key]) for item in testset] |
| 100 | |
| 101 | return await tqdm.gather(*tasks, desc=f"Processing {self.dataset} tasks") |
| 102 | |
| 103 | def _format_question_from_keys(self, item: Dict[str, Any], keys: List[str]) -> str: |
| 104 | # When question_key is a list, concatenate values from multiple keys into a single question |
| 105 | parts = [] |
| 106 | for key in keys: |
| 107 | if key in item: |
| 108 | parts.append(f"{key}: {item[key]}") |
| 109 | return "\n".join(parts) |